├── .gitignore ├── .gitmodules ├── README.md ├── bins ├── bgpdump ├── bgpdump-amd64 ├── bgpdump-x86 ├── bgpdump.exe ├── cygwin1.dll ├── libbz2.so-amd64 └── libbz2.so-x86 ├── html ├── base.html ├── data.html ├── help.html ├── map.html ├── search_res.html ├── tools.html ├── view_cable.html ├── view_country.html ├── view_fac.html ├── view_ix.html ├── view_landing.html ├── view_net.html └── view_world.html ├── pwui ├── pwui.py ├── pwui_colors.py ├── pwui_jinja.py ├── pwui_pages.py ├── pwui_routes.py └── pwui_tools.py ├── requirements.txt ├── res ├── Flaticon.eot ├── Flaticon.svg ├── Flaticon.ttf ├── Flaticon.woff ├── blank.gif ├── bootstrap.min.css ├── bootstrap.min.js ├── d3.v3.min.js ├── datamaps.world.hires.min.js ├── datamaps.world.min.js ├── favicon.png ├── flags-24.png ├── flags-32.png ├── flags.min.css ├── flaticon.css ├── flaticon.html ├── jquery-3.2.1.min.js ├── popper.min.js ├── sprintf.min.js └── topojson.v1.min.js ├── screenshot.png ├── update.sh └── wnm ├── wnm_asnames.py ├── wnm_bgp.py ├── wnm_geo.py ├── wnm_merge.py ├── wnm_nics.py ├── wnm_scd.py ├── wnm_utils.py └── wnm_world.py /.gitignore: -------------------------------------------------------------------------------- 1 | venv 2 | pwui/*.pyc 3 | wnm/*.pyc 4 | 5 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "data"] 2 | path = data 3 | url = https://github.com/ixty/wnm_data.git 4 | [submodule "data-raw"] 5 | path = data-raw 6 | url = https://github.com/ixty/wnm_data_raw.git 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # World Net Map 2 | 2017-2018 - Martin Balch - contact 3 | 4 | ## intro 5 | 6 | World Net Map (WNM) is a project to visualize geographically the internet infrastructure, networks and connectivity. 7 | It uses data from multiple sources to display information on networks, countries, facilities, exchanges, submarine cables, ... and represent everything in a single interactive map. 8 | This repository hosts the code used to download, agregate, consolidate & format the data (`wnm_*`) in addition to the code of a web server that displays the data (`pwui_*`). 9 | 10 | See it in action here: [http://wnm.ixty.net/](http://wnm.ixty.net/) 11 | 12 | ![World Net Map Screenshot](https://github.com/ixty/wnm/raw/master/screenshot.png "World Net Map Screenshot") 13 | 14 | ## installation 15 | 16 | ```shell 17 | # pre-requisites 18 | $ sudo apt-get install git virtualenv unzip 19 | 20 | # install with data (recommended) 21 | $ git clone --recursive https://github.com/ixty/wnm 22 | 23 | # or install without data (you need to regenerate databases) 24 | $ # git clone https://github.com/ixty/wnm 25 | 26 | # python requirements 27 | $ cd wnm 28 | $ virtualenv venv 29 | $ source venv/bin/activate 30 | $ pip install -r requirements.txt 31 | ``` 32 | 33 | ## run server 34 | 35 | In debug mode (default), the default bound address is `127.0.0.1:5000`, flask is used as a webserver, templates are automatically reloaded and there is no cache on pages. 36 | 37 | ```shell 38 | $ source venv/bin/activate 39 | $ ./pwui/pwui.py 40 | ``` 41 | 42 | The default bound address is `0.0.0.0:80` in release mode, which may requires root acess. 43 | 44 | ```shell 45 | $ source venv/bin/activate 46 | $ ./pwui/pwui.py release 47 | ``` 48 | 49 | ## regenerate databases 50 | 51 | If you didnt clone the repository recursively, you need to regenerate the database. 52 | 53 | ### automated update 54 | If you want to generate or update the database used by WNM, you can use the following command: 55 | ```shell 56 | $ source venv/bin/activate 57 | $ ./update.py 58 | ``` 59 | This operation can take a long time based on your bandwith & cpu. 60 | 61 | ### manual sub-database updates 62 | You can also update specific databases by using the following scripts: 63 | ```shell 64 | $ ./wnm/wnm_asnames.py 65 | $ ./wnm/wnm_bgp.py 66 | $ ./wnm/wnm_geo.py 67 | $ ./wnm/wnm_nics.py 68 | $ ./wnm/wnm_scd.py 69 | $ ./wnm/wnm_world.py 70 | ``` 71 | They all have the same usage: `./script ` 72 | - `download` will download the raw source data 73 | - `rebuild` will update our local database (./data/*) with the previously downloaded source data. 74 | - `update` is equivalent to running `download` and `rebuild` 75 | 76 | ### geo-location 77 | To get accurate positioning for facilities, you will need to get an OpenCageData API key from [OpenCageData.com](https://geocoder.opencagedata.com/api) 78 | ```shell 79 | # for a first build 80 | $ OCD_KEY=01234567890123456789012345678901 ./update.py 81 | 82 | # or just regenerate WNM data 83 | $ OCD_KEY=01234567890123456789012345678901 ./wnm/wnm_merge.py 84 | ``` 85 | 86 | ### update log 87 |
88 | Example output of the full updating process 89 | 90 | ```shell 91 | ======================================== 92 | > updating AS names database 93 | ======================================== 94 | > ftp://ftp.radb.net/radb/dbase/radb.db.gz 95 | > downloaded ./data-raw/as-radb.db.gz in 3.2 secs 96 | > ftp://ftp.arin.net/pub/rr/arin.db 97 | > downloaded ./data-raw/as-arin.db in 5.4 secs 98 | > ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz 99 | > downloaded ./data-raw/as-ripe.db.gz in 2.2 secs 100 | > ftp://ftp.afrinic.net/pub/dbase/afrinic.db.gz 101 | > downloaded ./data-raw/as-afrinic.db.gz in 4.4 secs 102 | > ftp://rr.level3.net/pub/rr/level3.db.gz 103 | > downloaded ./data-raw/as-level3.db.gz in 3.1 secs 104 | > https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz 105 | > downloaded ./data-raw/as-apnic.db.gz in 6.6 secs 106 | ======================================== 107 | > done. 108 | > parsing as-radb.db.gz 109 | > parsing as-arin.db.gz 110 | > parsing as-ripe.db.gz 111 | > parsing as-afrinic.db.gz 112 | > parsing as-level3.db.gz 113 | > parsing as-apnic.db.gz 114 | > saving to ./data/asnames.json.gz (52112 items) 115 | 116 | ======================================== 117 | > updating BGP database 118 | ======================================== 119 | > http://routeviews.org/route-views.linx/bgpdata/2018.01/RIBS/rib.20180129.0000.bz2 120 | > downloaded ./data-raw/bgp-linx.bz2 in 9.3 secs 121 | > downloaded [linx] bgp database. 122 | > http://routeviews.org/route-views.eqix/bgpdata/2018.01/RIBS/rib.20180129.0000.bz2 123 | > downloaded ./data-raw/bgp-eqix.bz2 in 4.7 secs 124 | > downloaded [eqix] bgp database. 125 | > http://routeviews.org/route-views.jinx/bgpdata/2018.01/RIBS/rib.20180129.0000.bz2 126 | > downloaded ./data-raw/bgp-jinx.bz2 in 2.3 secs 127 | > downloaded [jinx] bgp database. 128 | > http://routeviews.org/route-views.saopaulo/bgpdata/2018.01/RIBS/rib.20180129.0000.bz2 129 | > downloaded ./data-raw/bgp-saopaulo.bz2 in 4.1 secs 130 | > downloaded [saopaulo] bgp database. 131 | > http://routeviews.org/route-views.sg/bgpdata/2018.01/RIBS/rib.20180129.0000.bz2 132 | > downloaded ./data-raw/bgp-sg.bz2 in 3.3 secs 133 | > downloaded [sg] bgp database. 134 | > done 5/5 135 | > loading ./data-raw/bgp-linx.bz2 136 | > processing ./data-raw/bgp-linx.bz2 137 | 100%|████████████████████████████████████████████| 16948716/16948716 [32:26<00:00, 8707.87it/s] 138 | > loading ./data-raw/bgp-eqix.bz2 139 | > processing ./data-raw/bgp-eqix.bz2 140 | 100%|██████████████████████████████████████████████| 8601056/8601056 [18:27<00:00, 7766.23it/s] 141 | > loading ./data-raw/bgp-jinx.bz2 142 | > processing ./data-raw/bgp-jinx.bz2 143 | 100%|██████████████████████████████████████████████| 1157353/1157353 [02:29<00:00, 7748.20it/s] 144 | > loading ./data-raw/bgp-saopaulo.bz2 145 | > processing ./data-raw/bgp-saopaulo.bz2 146 | 100%|██████████████████████████████████████████████| 9463089/9463089 [24:37<00:00, 6404.86it/s] 147 | > loading ./data-raw/bgp-sg.bz2 148 | > processing ./data-raw/bgp-sg.bz2 149 | 100%|██████████████████████████████████████████████| 4093937/4093937 [09:39<00:00, 7061.49it/s] 150 | > saving to ./data/bgp.json.gz (60826 items) 151 | 152 | ======================================== 153 | > updating world cities database 154 | ======================================== 155 | > http://download.maxmind.com/download/worldcities/worldcitiespop.txt.gz 156 | > downloaded ./data-raw/worldcitiespop.txt.gz in 3.4 secs 157 | > done 158 | > processing ./data-raw/worldcitiespop.txt.gz 159 | > saving to ./data/cities.json.gz (234 items) 160 | 161 | ======================================== 162 | > updating nics database 163 | ======================================== 164 | > ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest 165 | > downloaded ./data-raw/nic-lacnic.txt in 6.3 secs 166 | > ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest 167 | > downloaded ./data-raw/nic-apnic.txt in 10.1 secs 168 | > ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest 169 | > downloaded ./data-raw/nic-afrinic.txt in 3.1 secs 170 | > ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest 171 | > downloaded ./data-raw/nic-arin.txt in 16.6 secs 172 | > ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest 173 | > downloaded ./data-raw/nic-ripe.txt in 1.5 secs 174 | ======================================== 175 | > consolidating database.. 176 | [ lacnic] date: 20180126 records 29128 177 | [ apnic] date: 20180129 records 54420 178 | [afrinic] date: 20180129 records 5652 179 | [ arin] date: 20180129 records 137492 180 | [ripencc] date: 20180128 records 114677 181 | > total rirs info [asn: 85628, ipv4: 183252, ipv6: 81081] 182 | > saving to ./data/nics.json.gz (4 items) 183 | 184 | ======================================== 185 | > updating submarine cable database 186 | ======================================== 187 | > https://github.com/telegeography/www.submarinecablemap.com/archive/master.zip 188 | > error downloading ./data-raw/cables.zip 189 | > https://github.com/telegeography/www.submarinecablemap.com/archive/master.zip 190 | > downloaded ./data-raw/cables.zip in 1.3 secs 191 | > unknown landing 9595 for country ID 192 | > unknown landing 9595 for cable 1895 193 | > saving to ./data/scdb.json.gz (3 items) 194 | 195 | ======================================== 196 | > updating country database 197 | ======================================== 198 | > http://download.geonames.org/export/dump/countryInfo.txt 199 | > downloaded ./data-raw/worldinfo.txt in 0.1 secs 200 | > saving to ./data/worldinfo.json.gz (252 items) 201 | 202 | ======================================== 203 | > updating PeeringDB 204 | ======================================== 205 | Operations to perform: 206 | Synchronize unmigrated apps: django_peeringdb 207 | Apply all migrations: (none) 208 | Synchronizing apps without migrations: 209 | Creating tables... 210 | Running deferred SQL... 211 | Installing custom SQL... 212 | Running migrations: 213 | No migrations to apply. 214 | org last update 1517240845 0 changed 215 | data to be processed 0 216 | fac last update 1517069653 0 changed 217 | data to be processed 0 218 | net last update 1517244409 3 changed 219 | data to be processed 3 220 | ix last update 1517005124 0 changed 221 | data to be processed 0 222 | ixfac last update 1516870316 0 changed 223 | data to be processed 0 224 | ixlan last update 1516870415 0 changed 225 | data to be processed 0 226 | ixpfx last update 1516707274 0 changed 227 | data to be processed 0 228 | poc last update 1517066523 5 changed 229 | data to be processed 5 230 | netfac last update 1517235749 0 changed 231 | data to be processed 0 232 | netixlan last update 1517242057 3 changed 233 | data to be processed 3 234 | 235 | ======================================== 236 | > Merging databases 237 | ======================================== 238 | > loading data from data/nics.json.gz 239 | > loading data from data/bgp.json.gz 240 | > loading data from data/scdb.json.gz 241 | > loading data from data/asnames.json.gz 242 | > loading data from data/cities.json.gz 243 | > loading data from data/worldinfo.json.gz 244 | > loading peering db .. 245 | > merging network info .. 246 | > enriching networks .. 247 | > enriching facilities .. 248 | > loading data from data/geofacs.json.gz 249 | > error loading data/geofacs.json.gz 250 | > warn: set OpenCageData API key (export OCD_KEY=...) for accurate geolocation 251 | > saving to data/geofacs.json.gz (0 items) 252 | 253 | > enriching IXs .. 254 | > building country stats .. 255 | > saving to data/final.json.gz (6 items) 256 | 257 | > all done :) (102m 4s) 258 | ``` 259 | 260 |
261 | 262 | 263 | ## data sources 264 | This is the list of all "data-raw" files that we use as data sources for WNM. 265 | 266 | ### RIRs 267 | ```shell 268 | ./data-raw/nic-afrinic.txt # ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest 269 | ./data-raw/nic-apnic.txt # ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest 270 | ./data-raw/nic-arin.txt # ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest 271 | ./data-raw/nic-lacnic.txt # ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest 272 | ./data-raw/nic-ripe.txt # ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest 273 | ``` 274 | Info is consolidated to `./data/nics.json.gz` 275 | 276 | ### peeringdb (peeringdb.org) 277 | ```shell 278 | ./data-raw/peeringdb.sqlite # https://peeringdb.github.io/peeringdb-py/cli/ 279 | ``` 280 | Used by `wnm_merge.py` as one of the most important databases 281 | 282 | ### telegeography (telegeography.com) 283 | ```shell 284 | ./data-raw/cables/* # https://github.com/telegeography/www.submarinecablemap.com 285 | ``` 286 | Info is consolidated to `./data/scdb.json.gz` 287 | 288 | ### BGP views (routeviews.org) 289 | ```shell 290 | ./data-raw/bgp-eqix.bz2 # http://routeviews.org/route-views.eqix/bgpdata/YYYY.MM/RIBS/rib.YYYYMMDD.HHMM.bz2 291 | ./data-raw/bgp-jinx.bz2 # http://routeviews.org/route-views.jinx/bgpdata/YYYY.MM/RIBS/rib.YYYYMMDD.HHMM.bz2 292 | ./data-raw/bgp-linx.bz2 # http://routeviews.org/route-views.linx/bgpdata/YYYY.MM/RIBS/rib.YYYYMMDD.HHMM.bz2 293 | ./data-raw/bgp-saopaulo.bz2 # http://routeviews.org/route-views.saopaulo/bgpdata/YYYY.MM/RIBS/rib.YYYYMMDD.HHMM.bz2 294 | ./data-raw/bgp-sg.bz2 # http://routeviews.org/route-views.sg/bgpdata/YYYY.MM/RIBS/rib.YYYYMMDD.HHMM.bz2 295 | ``` 296 | Info is consolidated to `./data/bgp.json.gz` 297 | 298 | ### world info (geonames.org) 299 | ```shell 300 | ./data-raw/worldinfo.txt # http://download.geonames.org/export/dump/countryInfo.txt 301 | ``` 302 | Info is consolidated to `./data/worldinfo.json.gz` 303 | 304 | ### world cities (maxmind.org) 305 | ```shell 306 | ./data-raw/worldcitiespop.txt.gz # http://download.maxmind.com/download/worldcities/worldcitiespop.txt.gz 307 | ``` 308 | Info is consolidated to `./data/worldcitiespop.txt.gz` 309 | 310 | ### AS names 311 | ```shell 312 | ./data-raw/as-afrinic.db.gz # ftp://ftp.afrinic.net/pub/dbase/afrinic.db.gz 313 | ./data-raw/as-apnic.db.gz # https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz 314 | ./data-raw/as-arin.db.gz # ftp://ftp.arin.net/pub/rr/arin.db 315 | ./data-raw/as-level3.db.gz # ftp://rr.level3.net/pub/rr/level3.db.gz 316 | ./data-raw/as-radb.db.gz # ftp://ftp.radb.net/radb/dbase/radb.db.gz 317 | ./data-raw/as-ripe.db.gz # ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz 318 | ``` 319 | Info is consolidated to `./data/asnames.json.gz` 320 | 321 | 322 | ## Consolidated data 323 | 324 | All our consolidated data is stored in `./wnm/data/`. 325 | The only file directly used by the web interface is `final.json.gz` which is the consolidation of all the others in the folder. 326 | 327 | Here is the list of all WNM databases: 328 | - final.json.gz # merged database directly usable by pwui 329 | 330 | - asnames.json.gz # AS number to AS name 331 | - bgp.json.gz # AS bgp routes, associated prefix & routing chains 332 | - cities.json.gz # world cities with country & gps coordinates 333 | - geofacs.json.gz # gps coordinates for all peeringdb facilities 334 | - nics.json.gz # RIR info: ASN, IPv4 & IPv6 association to countries 335 | - scdb.json.gz # extract of submarinecablemap.org database in our format 336 | - worldinfo.json.gz # countries surface, population, etc. 337 | 338 | ## display / pwui.py 339 | 340 | To display the map, the `pwui` python flask app was created. 341 | It uses [datamaps.js](http://datamaps.github.io/) (which itself uses [d3.js](https://d3js.org/)) for the world map. 342 | It uses [bootstrap](https://getbootstrap.com/) for the layout. 343 | 344 | ## repository tree overview 345 | 346 | ```shell 347 | wnm 348 | ╷ 349 | ├── bins # bgpdump utility 350 | ├── wnm # data fetching, consolidation & processing python scripts 351 | │ 352 | ├── data # our consolidated databases in json format 353 | │ # (git clone https://github.com/ixty/wnm_data) 354 | ├── data-raw # raw data from open sources 355 | │ # (git clone https://github.com/ixty/wnm_data-raw) 356 | │ 357 | ├── update.sh # script to update all our local databases 358 | │ 359 | ├── html # web interface flask templates 360 | ├── pwui # web interface python code 361 | ├── res # web interface static files 362 | │ 363 | ├── requirements.txt # python dependencies 364 | └── README.md # documentation 365 | ``` 366 | 367 | -------------------------------------------------------------------------------- /bins/bgpdump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ixty/wnm/af5bf95178714e7acdcb00c9d6bbbd2c614f64e8/bins/bgpdump -------------------------------------------------------------------------------- /bins/bgpdump-amd64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ixty/wnm/af5bf95178714e7acdcb00c9d6bbbd2c614f64e8/bins/bgpdump-amd64 -------------------------------------------------------------------------------- /bins/bgpdump-x86: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ixty/wnm/af5bf95178714e7acdcb00c9d6bbbd2c614f64e8/bins/bgpdump-x86 -------------------------------------------------------------------------------- /bins/bgpdump.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ixty/wnm/af5bf95178714e7acdcb00c9d6bbbd2c614f64e8/bins/bgpdump.exe -------------------------------------------------------------------------------- /bins/cygwin1.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ixty/wnm/af5bf95178714e7acdcb00c9d6bbbd2c614f64e8/bins/cygwin1.dll -------------------------------------------------------------------------------- /bins/libbz2.so-amd64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ixty/wnm/af5bf95178714e7acdcb00c9d6bbbd2c614f64e8/bins/libbz2.so-amd64 -------------------------------------------------------------------------------- /bins/libbz2.so-x86: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ixty/wnm/af5bf95178714e7acdcb00c9d6bbbd2c614f64e8/bins/libbz2.so-x86 -------------------------------------------------------------------------------- /html/base.html: -------------------------------------------------------------------------------- 1 | {% from "tools.html" import flag_24 with context %} 2 | {% from "help.html" import help with context %} 3 | 4 | {% macro active_pagename(val) -%} 5 | {% if page.pagename == val %} 6 | active 7 | {% endif %} 8 | {%- endmacro %} 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | World Net Map 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 237 | 238 | 239 | {% block content %} 240 | {% endblock %} 241 | 242 | 243 |
244 | {{ help() }} 245 | 246 | 252 |
253 | 254 | 255 | 256 | 257 | 258 | -------------------------------------------------------------------------------- /html/data.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% from "tools.html" import credits with context %} 4 | 5 | {% macro data_explore(id, type, text) %} 6 |
7 |
8 |
9 | 12 | 13 | 17 |
18 | 19 |
20 | Limit to 21 | 29 | 30 | 34 |
35 |
36 |
37 | 43 |
44 | {% endmacro %} 45 | 46 | {% block content %} 47 | 48 | 209 | 210 |
211 |

212 | 213 | CyberSpace: World Internet Map 214 |

215 |
216 | 2017 - Martin Balch - 217 | 218 | contact 219 | 220 |
221 | Code, database & methodology available on github. 222 |
223 | {{ credits() }} 224 | 225 |
226 | 227 |
228 |
229 | 230 | {% call data_explore('id_data_cn', 'countries', 'Top Countries by') %} 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | {% endcall %} 248 | 249 | {% call data_explore('id_data_net', 'nets', 'Top Networks by') %} 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | {% endcall %} 261 | 262 | {% call data_explore('id_data_ix', 'ixs', 'Top Exchanges by') %} 263 | 264 | 265 | 266 | {% endcall %} 267 | 268 | {% call data_explore('id_data_fac', 'facilities', 'Top Facilities by') %} 269 | 270 | 271 | {% endcall %} 272 | 273 | {% call data_explore('id_data_cables', 'cables', 'Top Cables by') %} 274 | 275 | 276 | 277 | 278 | {% endcall %} 279 | 280 | {% call data_explore('id_data_landings', 'landings', 'Top Landings by') %} 281 | 282 | {% endcall %} 283 | 284 |
285 |
286 |
287 | {% endblock %} -------------------------------------------------------------------------------- /html/help.html: -------------------------------------------------------------------------------- 1 | {% macro help() %} 2 | 3 |
4 |
5 |
6 |

7 | Legend 8 |

9 | 12 |
13 |
14 |

15 | 16 |

17 |
18 |
19 |
20 |
21 | 22 | World 23 |
24 |

25 | This icon is used to represent country rankings based on a network connectivity indicator such as the number of networks, number of IP addresses, etc. 26 |

27 |
28 |
29 |
30 | 31 |
32 |
33 |
34 |
35 | 36 | Countries 37 |
38 |

39 | Countries are colored differently based on the current view.
40 | In world rankings, countries are colored based on the selected variable.
41 | In country view, other countries are colored if they are directly or indirectly linked to the focused country.
42 | In network view, countries are colored if the current network has direct or indirect links to this country.
43 |

44 |
45 |
46 |
47 | 48 |
49 |
50 |
51 |
52 | 53 | Network 54 |
55 |

56 | Networks or Autonomous Systems (AS) are collections of Internet Protocol (IP) routed prefixes under the control of a single administrative entity. To put it more plainly, an AS is network that is directly connected to the internet and that owns specific IP address ranges. Both Internet Service Providers (ex: AT&T, Vodafone, Orange) and content providers (Google, Facebook, Amazon) own at least one AS. 57 |

58 |
59 |
60 |
61 | 62 |
63 |
64 |
65 |
66 | 67 | Exchange 68 |
69 |

70 | An Internet exchange point (IX or IXP) is a physical infrastructure through which Internet service providers (ISPs) and content delivery networks (CDNs) exchange Internet traffic between their networks (autonomous systems).
71 | IX are displayed on the map only by the facilities at which they are present. 72 |

73 |
74 |
75 |
76 | 77 |
78 |
79 |
80 |
81 | 82 | Facilities 83 |
84 |

85 | Facilities are physical points of peering, where networks interconnect either via internet exchanges or private peering agreements.
86 | They are represented on the map as red dots. 87 |

88 |
89 |
90 |
91 | 92 |
93 |
94 |
95 |
96 | 97 | Cables 98 |
99 |

100 | Submarine cables which are used to connect countries or different parts of a country together. Usually owned by internet service providers, they form the backbone of the internet. 101 |

102 |
103 |
104 |
105 | 106 |
107 |
108 |
109 |
110 | 111 | Landings 112 |
113 |

114 | Landings are where submarine cables surface and give connectivity to a country. 115 |
116 | They are represented on the map as orange dots. 117 |

118 |
119 |
120 |
121 | 122 |
123 |

124 | 125 |
126 |
127 |
128 | 129 | {% endmacro %} 130 | -------------------------------------------------------------------------------- /html/map.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block content %} 4 |
5 |
6 | 7 | 8 |
9 | {% block sidebar %} 10 | {% endblock %} 11 | 12 |
13 |
14 |
15 | 16 | 17 |
18 |
19 |
20 | 21 | 284 |
285 | 286 |
287 |
288 | {% endblock %} 289 | -------------------------------------------------------------------------------- /html/search_res.html: -------------------------------------------------------------------------------- 1 | {% from "tools.html" import flag_24 with context %} 2 | 3 | {% if search_res|length > 0 %} 4 | {% filter stripblock %} 5 | {% for r in search_res %} 6 | {%- filter spaceless -%} 7 | 8 | {% if r['type'] == 'net' %} 9 | 10 | {{ flag_24(r['cc']) }} 11 | {% elif r['type'] == 'country' %} 12 | 13 | {{ flag_24(r['cc']) }} 14 | {% elif r['type'] == 'facility' %} 15 | 16 | {{ flag_24(r['cc']) }} 17 | {% elif r['type'] == 'ix' %} 18 | 19 | {{ flag_24(r['cc']) }} 20 | {% elif r['type'] == 'landing' %} 21 | 22 | {{ flag_24(r['cc']) }} 23 | {% elif r['type'] == 'cable' %} 24 | 25 | {% endif %} 26 | 27 | {% if r['type'] == 'net' and r['name']|length <= 0 %} 28 | AS{{ r['path'][5:] }} 29 | {% else %} 30 | {{ r['name'] }} 31 | {% endif %} 32 | 33 | {% endfilter %} 34 | {% endfor %} 35 | {% endfilter %} 36 | {% else %} 37 | No Result 38 | {% endif %} 39 | -------------------------------------------------------------------------------- /html/tools.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | {% macro flag_24(cc) %} 4 | {% if cc != '??' %} 5 | 6 | {% else %} 7 | 8 | {% endif %} 9 | {% endmacro %} 10 | 11 | 12 | {% macro flag_32(cc) %} 13 | {% if cc != '??' and cc != 'RE' and cc != 'XK' and cc != 'SX' and cc != 'GF' and cc != 'GP' and cc != 'PM' and cc != 'IO' %} 14 | 15 | {% else %} 16 | 17 | {% endif %} 18 | {% endmacro %} 19 | 20 | 21 | {% macro list_item_collapse(id, name, count, notheme=0, expanded=0) %} 22 |
  • 23 |
    24 | 25 | 26 | {{ name|safe }} 27 | {{ count }} 28 |
    29 | 30 |
    31 |
    32 | {{ caller() }} 33 |
    34 |
    35 |
    • 36 | {% endmacro %} 37 | 38 | 39 | {% macro item_net(net, speed=0) %} 40 |
    41 | 42 | {{ flag_24(net['cc']) }} {{ net['asn'] }} {{ net['name'] }} 43 | 44 |
    45 | {% endmacro %} 46 | 47 | 48 | {% macro item_facility(fac) %} 49 |
    50 | 51 | 52 | {{ flag_24(fac['country']) }} {{ fac['name'] }} 53 | 54 |
    55 | {% endmacro %} 56 | 57 | 58 | {% macro item_ix(ixi) %} 59 |
    60 | 61 | 62 | {{ flag_24(ixi['country']) }} {{ ixi['name'] }} 63 | 64 |
    65 | {% endmacro %} 66 | 67 | 68 | {% macro item_country(cc) %} 69 | 70 | {{ flag_24(cc) }} {{ cc }} 71 | 72 | {% endmacro %} 73 | 74 | {% macro item_country2(cc) %} 75 | 76 | {{ flag_24(cc) }} {{ db['countries'][cc]['name'] }} 77 | 78 | {% endmacro %} 79 | 80 | 81 | {% macro item_landing(ld) %} 82 |
    83 | 84 | 85 | {{ flag_24(ld['cc']) }} 86 | {{ '(%d) %s' % (ld['cables']|length, ld['name']) }} 87 | 88 |
    89 | {% endmacro %} 90 | 91 | 92 | {% macro item_cable(c) %} 93 |
    94 | 95 | 96 | {% if not c['ready'] %} 97 | 98 | {% endif %} 99 | {{ '(%d) %s' % (c['landings']|length, c['name']) }} 100 | {% if not c['ready'] %} 101 | 102 | {% endif %} 103 | 104 |
    105 | {% endmacro %} 106 | 107 | 108 | 109 | {% macro item_link(cc, type, id) %} 110 | {% if type == 'net' %} 111 | {{ item_net(db['nets'][id|string]) }} 112 | {% elif type == 'fac' %} 113 | {{ item_facility(db['facilities'][id|string]) }} 114 | {% elif type == 'ix' %} 115 | {{ item_ix(db['ixs'][id|string]) }} 116 | {% elif type == 'cable' %} 117 | {{ item_cable(db['cables'][id|string]) }} 118 | {% elif type == 'landing' %} 119 | {{ item_landing(db['landings'][id|string]) }} 120 | {% endif %} 121 | {% endmacro %} 122 | 123 | 124 | {% macro credits() %} 125 |
    126 | Data sources: 127 | 142 |
    143 | {% endmacro %} 144 | -------------------------------------------------------------------------------- /html/view_cable.html: -------------------------------------------------------------------------------- 1 | {% extends "map.html" %} 2 | 3 | {% from "tools.html" import list_item_collapse with context %} 4 | {% from "tools.html" import item_landing with context %} 5 | 6 | 7 | 8 | {% block sidebar %} 9 | {% filter stripblock %} 10 |
    11 |

    12 | 13 | {{ cable['name'] }} 14 |

    15 | 16 | 68 | 69 | 70 |
    71 | {% endfilter %} 72 | {% endblock %} 73 | -------------------------------------------------------------------------------- /html/view_country.html: -------------------------------------------------------------------------------- 1 | {% extends "map.html" %} 2 | 3 | {% from "tools.html" import flag_24 with context %} 4 | {% from "tools.html" import flag_32 with context %} 5 | {% from "tools.html" import list_item_collapse with context %} 6 | {% from "tools.html" import item_net with context %} 7 | {% from "tools.html" import item_facility with context %} 8 | {% from "tools.html" import item_ix with context %} 9 | {% from "tools.html" import item_link with context %} 10 | {% from "tools.html" import item_landing with context %} 11 | {% from "tools.html" import item_cable with context %} 12 | 13 | 14 | 15 | {% block sidebar %} 16 | {% filter stripblock %} 17 |
    18 |

    19 | 20 | {{ flag_32(country['cc']) }} 21 | {{ country['name'] }} 22 |

    23 | 24 | 201 |
    202 | {% endfilter %} 203 | {% endblock %} 204 | 205 | -------------------------------------------------------------------------------- /html/view_fac.html: -------------------------------------------------------------------------------- 1 | {% extends "map.html" %} 2 | 3 | {% from "tools.html" import flag_32 with context %} 4 | {% from "tools.html" import list_item_collapse with context %} 5 | {% from "tools.html" import item_net with context %} 6 | {% from "tools.html" import item_ix with context %} 7 | {% from "tools.html" import item_country with context %} 8 | 9 | 10 | {% block sidebar %} 11 | {% filter stripblock %} 12 |
    13 |

    14 | 15 | {{ flag_32(fac['country']) }} 16 | {{ fac['name'] }} 17 |

    18 | 19 | 69 |
    70 | {% endfilter %} 71 | {% endblock %} 72 | 73 | -------------------------------------------------------------------------------- /html/view_ix.html: -------------------------------------------------------------------------------- 1 | {% extends "map.html" %} 2 | 3 | {% from "tools.html" import flag_24 with context %} 4 | {% from "tools.html" import flag_32 with context %} 5 | {% from "tools.html" import list_item_collapse with context %} 6 | {% from "tools.html" import item_facility with context %} 7 | 8 | 9 | 10 | {% block sidebar %} 11 | {% filter stripblock %} 12 |
    13 |

    14 | 15 | {{ flag_32(ix['country']) }} 16 | {{ ix['name'] }} 17 |

    18 | 19 | 61 |
    62 | {% endfilter %} 63 | {% endblock %} 64 | -------------------------------------------------------------------------------- /html/view_landing.html: -------------------------------------------------------------------------------- 1 | {% extends "map.html" %} 2 | 3 | {% from "tools.html" import flag_32 with context %} 4 | {% from "tools.html" import list_item_collapse with context %} 5 | {% from "tools.html" import item_country2 with context %} 6 | {% from "tools.html" import item_cable with context %} 7 | 8 | 9 | 10 | {% block sidebar %} 11 | {% filter stripblock %} 12 |
    13 |

    14 | 15 | {{ flag_32(landing['cc']) }} 16 | {{ landing['name'] }} 17 |

    18 | 19 | 33 | 34 |
    35 | {% endfilter %} 36 | {% endblock %} 37 | -------------------------------------------------------------------------------- /html/view_net.html: -------------------------------------------------------------------------------- 1 | {% extends "map.html" %} 2 | 3 | {% from "tools.html" import flag_24 with context %} 4 | {% from "tools.html" import flag_32 with context %} 5 | {% from "tools.html" import list_item_collapse with context %} 6 | {% from "tools.html" import item_net with context %} 7 | {% from "tools.html" import item_facility with context %} 8 | {% from "tools.html" import item_link with context %} 9 | 10 | 11 | {% block sidebar %} 12 | {% filter stripblock %} 13 |
    14 |

    15 | 16 | {{ flag_32(net['cc']) }} AS{{ net['asn'] }} 17 |

    18 |

    19 | {% if net['website']|length > 0 %} 20 | 21 | {% endif %} 22 | {{ net['name'] }} 23 |

    24 | 25 | 160 |
    161 | {% endfilter %} 162 | {% endblock %} 163 | 164 | -------------------------------------------------------------------------------- /html/view_world.html: -------------------------------------------------------------------------------- 1 | {% extends "map.html" %} 2 | 3 | {% from "tools.html" import flag_24 with context %} 4 | {% from "tools.html" import credits with context %} 5 | 6 | {% block sidebar %} 7 |
    8 |
    9 |

    {{ page.title }}

    10 |

    11 | {{ page.text }} 12 | 13 | {% if page.pagename == 'ranks' and page.opt|length %} 14 | 15 | {{ page.detail }} 16 | 17 | {% endif %} 18 |

    19 | 20 | {% if page.pagename == 'ranks' %} 21 |
    22 | 23 | Abs 24 | 25 | {% if not page.opt|length %} 26 | 29 | 32 | {% endif %} 33 | 34 | Pop 35 | 36 | 37 | Km^2 38 | 39 |
    40 | {% endif %} 41 | 42 |
    43 | 44 | {% if page.pagename == 'ranks' %} 45 |
    46 | {% for cn in page.data %} 47 | 48 | {{ flag_24(cn['cc']) }} 49 | {{ cn['name'] }} 50 | 51 | 52 | {% if not page.opt|length %} 53 | 54 | {{ '%.2f %%' % (100.0 * cn['val'] / page.total) }} 55 | 56 | 59 | {% else %} 60 | 61 | {{ cn['val']|bignumb }} 62 | 63 | {% endif %} 64 | 65 |
    66 | {% endfor %} 67 |
    68 | 69 | 72 | {% else %} 73 |
    74 |

    75 | This is an experiment at mapping the world's internet connectivity.
    76 |

    77 |

    78 | In this view, countries are colored according to their number of networks, red bubbles represent points where different networks interconnect and orange bubbles are where submarine cables land. 79 |

    80 |

    81 | Move around, zoom and click on the map or use the search bar to explore countries, networks, exchanges, etc. 82 |

    83 |
    84 |

    85 | Code, database & methodology available on github. 86 |
    87 |

    88 |
    89 | {{ credits() }} 90 |
    91 | 92 |
    93 | 2017 - Martin Balch - 94 | 95 | contact 96 | 97 |
    98 | {% endif %} 99 | 100 |
    101 | {% endblock %} 102 | 103 | -------------------------------------------------------------------------------- /pwui/pwui.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | import os, sys, json, gzip 4 | import flask, flask_compress 5 | import pwui_jinja, pwui_pages, pwui_tools 6 | 7 | # debug mode? 8 | debug = 1 9 | if len(sys.argv) > 1 and sys.argv[1].lower() == 'release': 10 | debug = 0 11 | addr = '127.0.0.1' if debug else '0.0.0.0' 12 | port = 5000 if debug else 80 13 | print('> debug: %d' % debug) 14 | 15 | # load our database 16 | print('> loading database') 17 | with gzip.open('./data/final.json.gz', 'rb') as f: 18 | db = json.load(f) 19 | 20 | # options 21 | # cache_enabled = not debug 22 | cache_enabled = False 23 | 24 | # init flask app 25 | print('> starting web server on %s:%d' % (addr, port)) 26 | app = flask.Flask('wnm', static_folder='res', template_folder='html') 27 | 28 | # auto reload templates + static files 29 | app.config['TEMPLATES_AUTO_RELOAD'] = 1 30 | 31 | import pwui_routes 32 | 33 | # add content compression 34 | flask_compress.Compress(app) 35 | 36 | # run interface 37 | if debug: 38 | app.run(host=addr, port=port, threaded=True) 39 | else: 40 | from gevent.wsgi import WSGIServer 41 | srv = WSGIServer((addr, port), app) 42 | try: 43 | srv.serve_forever() 44 | except KeyboardInterrupt: 45 | print '> stop.' 46 | -------------------------------------------------------------------------------- /pwui/pwui_colors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # theme & colors 5 | pwui_colors = { 6 | 'bg_body': '#FFFFFF', 7 | 'bg_map': '#e6f3fb', 8 | 'bg_header': '#80A7C3', 9 | 'link1': 'rgba(57, 224, 64, 0.4)', # #39E040 10 | 'link2': 'rgba(230, 230, 44, 0.4)', # #E5EC2B 11 | 'link_selected': 'rgba(90, 90, 90, 0.75)', # #5D5D5D 12 | 'fac_base': '#FD3A77', 13 | 'fac_border': '#ffffff', 14 | 'fac_bad': '#0B650B', 15 | 'lnd_base': '#FF9800', 16 | 'lnd_border': '#ffffff', 17 | 'cn_border': '#000000', 18 | 'cn_empty': '#FFFFFF', 19 | 'cn_base': '#C0E6FC', 20 | 'cn_high': '#002BFF', 21 | 'cn_selected': 'rgba(90, 90, 90, 0.75)', 22 | } 23 | 24 | -------------------------------------------------------------------------------- /pwui/pwui_jinja.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | import jinja2 4 | 5 | # =========================================================================== # 6 | # jinja filters 7 | # =========================================================================== # 8 | 9 | # custom filter 10 | def bandwith(mbits): 11 | if mbits >= 1000 * 1000: 12 | return '%.1f Tbps' % (float(mbits) / 1000000.0) 13 | elif mbits >= 1000: 14 | return '%.1f Gbps' % (float(mbits) / 1000.0) 15 | return '%d Mbps' % mbits 16 | 17 | def bandwith2(mbits): 18 | if mbits >= 1000 * 1000: 19 | return '%.0f Tbps' % (float(mbits) / 1000000.0) 20 | elif mbits >= 1000: 21 | return '%.0f Gbps' % (float(mbits) / 1000.0) 22 | return '%d Mbps' % mbits 23 | 24 | def bignumb(val): 25 | val = long(float(val)) 26 | if val >= 1000*1000*1000*1000*1000: 27 | return '%.1e' % val 28 | elif val >= 1000*1000*1000*1000: 29 | return '%.1f T' % (float(val) / (1000*1000*1000*1000)) 30 | elif val >= 1000*1000*1000: 31 | return '%.1f B' % (float(val) / (1000*1000*1000)) 32 | elif val >= 1000*1000: 33 | return '%.1f M' % (float(val) / (1000*1000)) 34 | elif val >= 1000: 35 | return '%.1f k' % (float(val) / 1000) 36 | elif val != 0: 37 | return '%d' % val 38 | else: 39 | return '0' 40 | 41 | def distance(val): 42 | if val < 0: 43 | return '?' 44 | return "{:,} km".format(val) 45 | 46 | def spaceless(s): 47 | out = [] 48 | for l in s.splitlines(): 49 | l = l.strip() 50 | if not len(l): 51 | continue 52 | out += [l] 53 | return ''.join(out) 54 | 55 | def stripblock(s): 56 | out = [] 57 | for l in s.splitlines(): 58 | l = l.strip() 59 | if not len(l): 60 | continue 61 | out += [l] 62 | return '\n'.join(out) 63 | 64 | jinja2.filters.FILTERS['bandwith'] = bandwith 65 | jinja2.filters.FILTERS['bandwith2'] = bandwith2 66 | jinja2.filters.FILTERS['spaceless'] = spaceless 67 | jinja2.filters.FILTERS['stripblock'] = stripblock 68 | jinja2.filters.FILTERS['bignumb'] = bignumb 69 | jinja2.filters.FILTERS['distance'] = distance 70 | -------------------------------------------------------------------------------- /pwui/pwui_pages.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | import math 4 | import pwui_jinja 5 | from pwui_tools import * 6 | 7 | 8 | # =========================================================================== # 9 | # build data for country world rankings 10 | # =========================================================================== # 11 | cn_ranks = { 12 | 'world': [ 'World Internet Map', 'AS', ''], 13 | 'nets_loc': [ 'Networks (AS)', 'AS', 'Autonomous Systems repartition around the world' ], 14 | 'nets_ext': [ 'Incoming Networks', 'ext AS', 'External AS connected to countries, aka country "open-ness" to the rest of the world' ], 15 | 'num_fac': [ 'Facilities', 'facilities', 'Private & public internet points of peering' ], 16 | 'num_ix': [ 'Exchanges (IX)', 'IX', 'Public internet points of peering' ], 17 | 'landings': [ 'Landings', 'landings', 'Submarine cables landings' ], 18 | 'addr4': [ 'IPv4 Addresses', 'IPv4 addresses', 'IPv4 addresses repartition around the world' ], 19 | 'addr6': [ 'IPv6 Addresses', 'IPv6 addresses', 'IPv6 addresses repartition around the world' ], 20 | } 21 | 22 | def pagedata_rank(db, key, opt): 23 | # get options for this view 24 | r = cn_ranks[key] 25 | 26 | # create default empty page struct 27 | p = page_default('ranks' if key != 'world' else 'world') 28 | p['title'] = r[0] 29 | p['text' ] = r[2] 30 | p['key' ] = key 31 | p['opt'] = opt 32 | p['unit'] = r[1] 33 | unit = r[1] 34 | 35 | # build data list for each countrie's colorization 36 | for cc in db['countries']: 37 | if key == 'world': 38 | n = len(db['countries'][cc]['nets_loc']) 39 | elif isinstance(db['countries'][cc][key], list): 40 | n = len(db['countries'][cc][key]) 41 | else: 42 | n = db['countries'][cc][key] 43 | 44 | if len(opt): 45 | if db['countries'][cc][opt]: 46 | if opt == 'pop': 47 | div = float(db['countries'][cc][opt]) / 1000000.0 48 | n /= div if div >= 1 else 1 49 | unit = r[1] + ' per M inhabitant' 50 | p['detail'] = ' per M inhabitant' 51 | elif opt == 'lnd': 52 | div = float(db['countries'][cc][opt]) / 1000.0 53 | n /= div if div >= 1 else 1 54 | unit = r[1] + ' per 1,000 km^2' 55 | p['detail'] = ' per 1,000 km^2' 56 | else: 57 | n = 0 58 | 59 | itm = {} 60 | itm['cc'] = db['countries'][cc]['cc'] 61 | itm['iso3'] = db['countries'][cc]['iso3'] 62 | itm['name'] = db['countries'][cc]['name'] 63 | itm['val'] = n 64 | itm['col'] = max(1, math.log(n + 1)) 65 | # itm['col'] = n 66 | itm['hover']= '%s %s' % (pwui_jinja.bignumb(n), unit) 67 | 68 | if n == 0: 69 | itm['col'] = 0.1 70 | p['data'].append(itm) 71 | 72 | # get max color val 73 | p['col_max'] = max([itm['col'] for itm in p['data']] + [1]) 74 | p['total'] = sum([itm['val'] for itm in p['data']]) 75 | 76 | # sort data 77 | p['data'] = sorted(p['data'], key=lambda n: n['val'], reverse=1) 78 | for i in range(len(p['data'])): 79 | p['data'][i]['hover'] = '# %d
    ' % (i+1) + p['data'][i]['hover'] 80 | 81 | 82 | # add facility bubbles 83 | if key == 'num_fac' or key == 'world': 84 | for i in db['facilities']: 85 | f = db['facilities'][i] 86 | if len(f['nets']) > 0: 87 | page_add_facility(p, f) 88 | 89 | # add facility bubbles - only those with ixs 90 | if key == 'num_ix': 91 | for i in db['facilities']: 92 | f = db['facilities'][i] 93 | if len(f['ix']) > 0: 94 | page_add_ix(p, f) 95 | 96 | # add landings bubbles 97 | if key == 'landings' or key == 'world': 98 | for i in db['landings']: 99 | page_add_landing(p, db['landings'][i]) 100 | 101 | return p 102 | 103 | 104 | # =========================================================================== # 105 | # build data for country view 106 | # =========================================================================== # 107 | def pagedata_country(db, cc): 108 | p = page_default('country') 109 | country = db['countries'][cc] 110 | 111 | # get country incoming/outgoing connectivity score for each country 112 | nl_dir_inc = { cc: len(country['links_dir_inc'][cc][subcat]) for cc in country['links_dir_inc'] for subcat in country['links_dir_inc'][cc] } 113 | nl_dir_out = { cc: len(country['links_dir_out'][cc][subcat]) for cc in country['links_dir_out'] for subcat in country['links_dir_out'][cc] } 114 | nl_ind_inc = { cc: len(country['links_ind_inc'][cc][subcat]) for cc in country['links_ind_inc'] for subcat in country['links_ind_inc'][cc] } 115 | nl_ind_out = { cc: len(country['links_ind_out'][cc][subcat]) for cc in country['links_ind_out'] for subcat in country['links_ind_out'][cc] } 116 | 117 | # add all countries with links to this one 118 | for cc in country['links_dir_inc'].keys() + country['links_ind_inc'].keys(): 119 | page_add_cnl(p, db['countries'][cc], nl_dir_inc.get(cc, 0), nl_ind_inc.get(cc, 0)) 120 | 121 | # get max val 122 | p['col_max'] = max([itm['col'] for itm in p['data']] + [1]) 123 | 124 | # add current currently selected country with max_val 125 | page_add_cn(p, country, p['col_max'], p['col_max'], 'Selected Country') 126 | 127 | # sort data 128 | p['data'] = sorted(p['data'], key=lambda n: len(n['val']) if isinstance(n['val'], list) else n['val'], reverse=1) 129 | 130 | # add arcs for outgoing links 131 | for cc in country['links_dir_out'].keys() + country['links_ind_out'].keys(): 132 | if cc != '??': 133 | page_add_link_c2c(p, country, db['countries'][cc], nl_dir_out.get(cc, 0), nl_ind_out.get(cc, 0)) 134 | 135 | # add facilities 136 | for fid in country['fac_ids']: 137 | f = db['facilities'][str(fid)] 138 | if len(f['nets']) > 0: 139 | page_add_facility(p, f) 140 | 141 | # add landings 142 | for lid in country['landings']: 143 | page_add_landing(p, db['landings'][lid]) 144 | 145 | return p 146 | 147 | 148 | # =========================================================================== # 149 | # build data for network view 150 | # =========================================================================== # 151 | def pagedata_net(db, asn): 152 | p = page_default('net') 153 | net = db['nets'][asn] 154 | 155 | # get outgoing country links for this network 156 | nl_dir = { cc: len(net['links_dir'][cc][subcat]) for cc in net['links_dir'] for subcat in net['links_dir'][cc] } 157 | nl_ind = { cc: len(net['links_ind'][cc][subcat]) for cc in net['links_ind'] for subcat in net['links_ind'][cc] } 158 | 159 | for cc in nl_dir.keys() + nl_ind.keys(): 160 | page_add_cnl(p, db['countries'][cc], nl_dir.get(cc, 0), nl_ind.get(cc, 0)) 161 | 162 | p['col_max'] = max([itm['col'] for itm in p['data']] + [1]) 163 | page_add_cn(p, db['countries'][net['cc']], p['col_max'], p['col_max'], 'Home Country') 164 | 165 | # sort data 166 | p['data'] = sorted(p['data'], key=lambda n: len(n['val']) if isinstance(n['val'], list) else n['val'], reverse=1) 167 | 168 | # add facilities 169 | for i in net['fac_ids']: 170 | page_add_facility(p, db['facilities'][str(i)]) 171 | 172 | # add country links 173 | for cc in net['links_dir'].keys() + net['links_ind'].keys(): 174 | if cc != '??': 175 | page_add_link_n2c(db, p, net, db['countries'][cc], nl_dir.get(cc, 0), nl_ind.get(cc, 0)) 176 | 177 | return p 178 | 179 | 180 | # =========================================================================== # 181 | # build page data for facility 182 | # =========================================================================== # 183 | def pagedata_fac(db, fid): 184 | p = page_default('facility') 185 | f = db['facilities'][fid] 186 | 187 | scores = {} 188 | scores[f['country']] = 1 189 | for xid in f['ix']: 190 | ix = db['ixs'][str(xid)] 191 | if not ix['country'] in scores: 192 | scores[ix['country']] = 0 193 | scores[ix['country']] += 1 194 | 195 | for cc in scores: 196 | page_add_cn(p, db['countries'][cc], scores[cc], scores[cc], 'linked to %d exchanges in %s' % (scores[cc], cc)) 197 | 198 | intpres = [] 199 | for asn in f['nets']: 200 | net = db['nets'][str(asn)] 201 | if net['cc'] != f['country'] and net['cc'] != '??' and not net['cc'] in intpres: 202 | intpres += [ net['cc'] ] 203 | f['net_countries'] = intpres 204 | 205 | page_add_facility(p, f) 206 | 207 | p['col_max'] = max([itm['col'] for itm in p['data']] + [1]) 208 | return p 209 | 210 | 211 | # =========================================================================== # 212 | # build page data for exchange 213 | # =========================================================================== # 214 | def pagedata_ix(db, xid): 215 | p = page_default('exchange') 216 | ix = db['ixs'][xid] 217 | 218 | scores = {} 219 | for fid in ix['facilities']: 220 | fac = db['facilities'][str(fid)] 221 | page_add_facility(p, fac) 222 | if not fac['country'] in scores: 223 | scores[fac['country']] = 0 224 | scores[fac['country']] += 1 225 | 226 | for cc in scores: 227 | page_add_cn(p, db['countries'][cc], scores[cc], scores[cc], '%d facilities in %s' % (scores[cc], cc)) 228 | 229 | p['col_max'] = max([itm['col'] for itm in p['data']] + [1]) 230 | return p 231 | 232 | 233 | # =========================================================================== # 234 | # build page data for cable 235 | # =========================================================================== # 236 | def pagedata_cable(db, cid): 237 | p = page_default('cable') 238 | cable = db['cables'][cid] 239 | 240 | scores = {} 241 | for lid in cable['landings']: 242 | ld = db['landings'][lid] 243 | page_add_landing(p, ld) 244 | if not ld['cc'] in scores: 245 | scores[ld['cc']] = 0 246 | 247 | scores[ld['cc']] += 1 248 | 249 | for cc in scores: 250 | page_add_cn(p, db['countries'][cc], scores[cc], scores[cc], '%d landings in %s' % (scores[cc], cc)) 251 | 252 | 253 | p['col_max'] = max([itm['col'] for itm in p['data']] + [1]) 254 | return p 255 | 256 | 257 | # =========================================================================== # 258 | # build page data for landing 259 | # =========================================================================== # 260 | def pagedata_landing(db, lid): 261 | p = page_default('landing') 262 | landing = db['landings'][lid] 263 | 264 | page_add_landing(p, landing) 265 | 266 | scores = {} 267 | for cid in landing['cables']: 268 | for lid in db['cables'][cid]['landings']: 269 | c = db['landings'][lid]['cc'] 270 | if not c in scores: 271 | scores[c] = [] 272 | scores[c] += [ db['cables'][cid]['name'] ] 273 | 274 | for cc in scores: 275 | if cc != landing['cc']: 276 | n = len(scores[cc]) 277 | hov = 'Reachable by' 278 | for s in scores[cc]: 279 | hov += '
    %s' % s 280 | page_add_cn(p, db['countries'][cc], n, n, hov) 281 | 282 | p['col_max'] = max([itm['col'] for itm in p['data']] + [1]) 283 | page_add_cn(p, db['countries'][landing['cc']], p['col_max'] + 1, p['col_max'] + 1, 'Home country') 284 | p['col_max'] += 1 285 | 286 | return p 287 | 288 | -------------------------------------------------------------------------------- /pwui/pwui_routes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | import sys, re, flask 4 | from pwui_tools import * 5 | from pwui_pages import * 6 | from pwui_colors import * 7 | 8 | main = sys.modules['.'.join(__name__.split('.')[:-1]) or '__main__'] 9 | db = main.db 10 | colors = pwui_colors 11 | app = main.app 12 | 13 | # =========================================================================== # 14 | # simple cache, no timeout 15 | # =========================================================================== # 16 | class cached(object): 17 | 18 | def __init__(self, timeout=None): 19 | self.cache = {} 20 | 21 | def __call__(self, func): 22 | def decorator(*args, **kwargs): 23 | if main.cache_enabled: 24 | # clear cache if memory usage is too high 25 | if get_mem_usage() >= 75.0: 26 | self.cache = {} 27 | 28 | # lookup cache 29 | if flask.request.path in self.cache: 30 | return self.cache[flask.request.path] 31 | 32 | # not in cache, add it 33 | self.cache[flask.request.path] = func(*args, **kwargs) 34 | return self.cache[flask.request.path] 35 | else: 36 | return func(*args, **kwargs) 37 | 38 | decorator.func_name = func.func_name 39 | return decorator 40 | 41 | 42 | # =========================================================================== # 43 | # flask routes 44 | # =========================================================================== # 45 | @app.route("/") 46 | def page_index(): 47 | return flask.redirect('/ranks/world', code=302) 48 | 49 | @app.route("/home") 50 | def page_home(): 51 | p = { 52 | 'pagename': 'home', 53 | 'reqip': flask.request.remote_addr 54 | } 55 | return flask.render_template('home.html', page=p, colors=colors) 56 | 57 | @app.route("/data") 58 | @cached() 59 | def page_data(): 60 | p = { 61 | 'pagename': 'data', 62 | 'countries': [ cc for cc in db['countries'] ] 63 | } 64 | return flask.render_template('data.html', page=p, db=db, colors=colors) 65 | 66 | @app.route('/ranks/') 67 | @app.route('/ranks//') 68 | @cached() 69 | def map_ranks(key='nets_loc', opt=''): 70 | if len(opt) > 0 and opt != 'pop' and opt != 'lnd': 71 | return flask.abort(404) 72 | 73 | page = pagedata_rank(db, key, opt) 74 | return flask.render_template('view_world.html', page=page, colors=colors) 75 | 76 | @app.route('/net/') 77 | @cached() 78 | def map_net(asn): 79 | if not asn in db['nets']: 80 | return flask.abort(404) 81 | 82 | p = pagedata_net(db, asn) 83 | return flask.render_template('view_net.html', page=p, net=db['nets'][asn], db=db, colors=colors) 84 | 85 | @app.route('/country/') 86 | @cached() 87 | def map_country(cc): 88 | if cc.upper() not in db['countries']: 89 | return flask.abort(404) 90 | 91 | p = pagedata_country(db, cc.upper()) 92 | return flask.render_template('view_country.html', page=p, country=db['countries'][cc.upper()], db=db, colors=colors) 93 | 94 | @app.route('/fac/') 95 | @cached() 96 | def map_fac(fid): 97 | if fid not in db['facilities']: 98 | return flask.abort(404) 99 | 100 | p = pagedata_fac(db, fid) 101 | return flask.render_template('view_fac.html', page=p, fac=db['facilities'][fid], db=db, colors=colors) 102 | 103 | @app.route('/ix/') 104 | @cached() 105 | def map_ix(xid): 106 | if xid not in db['ixs']: 107 | return flask.abort(404) 108 | 109 | p = pagedata_ix(db, xid) 110 | return flask.render_template('view_ix.html', page=p, ix=db['ixs'][xid], db=db, colors=colors) 111 | 112 | @app.route('/cable/') 113 | @cached() 114 | def map_cable(cid): 115 | if cid not in db['cables']: 116 | return flask.abort(404) 117 | 118 | p = pagedata_cable(db, cid) 119 | return flask.render_template('view_cable.html', page=p, cable=db['cables'][cid], db=db, colors=colors) 120 | 121 | @app.route('/landing/') 122 | @cached() 123 | def map_landing(lid): 124 | if lid not in db['landings']: 125 | return flask.abort(404) 126 | 127 | p = pagedata_landing(db, lid) 128 | return flask.render_template('view_landing.html', page=p, landing=db['landings'][lid], db=db, colors=colors) 129 | 130 | @app.route('/redir/') 131 | @cached() 132 | def iso3_redir(iso3): 133 | for cc in db['countries']: 134 | if db['countries'][cc]['iso3'] == iso3: 135 | return flask.redirect('/country/' + cc, code=302) 136 | flask.abort(404) 137 | 138 | @app.route('/ip/') 139 | @cached() 140 | def redir_ip(cidr): 141 | ripv4 = r'^((2[0-5]{2}|2[0-4]\d|1\d{2}|[1-9]\d|\d)\.){3}(2[0-5]{2}|2[0-4]\d|1\d{2}|[1-9]\d|\d)(/(3[012]|[12]\d|\d))?$' 142 | ripv6 = r'^s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:)))(%.+)?s*(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$' 143 | 144 | # lookup ipv4 or cidr 145 | is_ipv4 = re.match(ripv4, cidr) 146 | is_ipv6 = re.match(ripv6, cidr) 147 | if is_ipv4 or is_ipv6: 148 | for asn in lookup_cidr(db, cidr, is_ipv6): 149 | return flask.redirect('/net/%s' % asn, code=302) 150 | flask.abort(404) 151 | 152 | @app.route('/top///') 153 | @cached() 154 | def top_data(table, key, limit): 155 | lst = db[table].values() 156 | lst = sorted(lst, key=lambda n: len(n[key]) if isinstance(n[key], list) else n[key], reverse=1) 157 | if limit != 'all': 158 | lst = lst[:int(limit)] 159 | return flask.jsonify(lst) 160 | 161 | @app.route('/search/') 162 | @cached() 163 | def search(text): 164 | text = text.strip() 165 | print '> searching for "%s"' % text 166 | 167 | ripv4 = r'^((2[0-5]{2}|2[0-4]\d|1\d{2}|[1-9]\d|\d)\.){3}(2[0-5]{2}|2[0-4]\d|1\d{2}|[1-9]\d|\d)(/(3[012]|[12]\d|\d))?$' 168 | ripv6 = r'^s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:)))(%.+)?s*(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$' 169 | rasn = r'^(?:as|AS|As|aS)?\s?_?(\d+)$' 170 | 171 | res = []; 172 | def add_res(type, cc, name, path): 173 | res.append({ 174 | 'type': type, 175 | 'cc': cc, 176 | 'name': name, 177 | 'path': path 178 | }) 179 | 180 | # lookup ipv4 or cidr 181 | is_ipv4 = re.match(ripv4, text) 182 | is_ipv6 = re.match(ripv6, text) 183 | if is_ipv4 or is_ipv6: 184 | print '> cidr match %s' % text 185 | for asn in lookup_cidr(db, text, is_ipv6): 186 | net = db['nets'][asn] 187 | add_res('net', net['cc'], net['name'], '/net/' + asn) 188 | 189 | # lookup country / cc 190 | if len(text) == 2: 191 | cc = text.upper() 192 | if cc in db['countries']: 193 | add_res('country', db['countries'][cc]['cc'], db['countries'][cc]['name'], '/country/' + cc) 194 | else: 195 | for cc in db['countries']: 196 | if text.lower() in db['countries'][cc]['name'].lower(): 197 | add_res('country', db['countries'][cc]['cc'], db['countries'][cc]['name'], '/country/' + cc) 198 | 199 | # lookup cable name 200 | for cid in db['cables']: 201 | if text.lower() in db['cables'][cid]['name'].lower(): 202 | add_res('cable', '??', db['cables'][cid]['name'], '/cable/%s' % cid) 203 | continue 204 | # and cable owners 205 | for o in db['cables'][cid]['owners']: 206 | if text.lower() in o.lower(): 207 | add_res('cable', '??', db['cables'][cid]['name'], '/cable/%s' % cid) 208 | continue 209 | 210 | # lookup ix name 211 | for xid in db['ixs']: 212 | if text.lower() in db['ixs'][xid]['name'].lower(): 213 | add_res('ix', db['ixs'][xid]['country'], db['ixs'][xid]['name'], '/ix/%s' % xid) 214 | 215 | # lookup fac name 216 | for fid in db['facilities']: 217 | if text.lower() in db['facilities'][fid]['name'].lower(): 218 | add_res('facility', db['facilities'][fid]['country'], db['facilities'][fid]['name'], '/fac/%s' % fid) 219 | 220 | # lookup landing name 221 | for lid in db['landings']: 222 | if text.lower() in db['landings'][lid]['name'].lower(): 223 | add_res('landing', db['landings'][lid]['cc'], db['landings'][lid]['name'], '/landing/%s' % lid) 224 | 225 | # lookup AS 226 | asm = re.match(rasn, text) 227 | if asm: 228 | asn = asm.group(1) 229 | print '> AS match %s' % asn 230 | if asn in db['nets']: 231 | add_res('net', db['nets'][asn]['cc'], db['nets'][asn]['name'], '/net/' + asn) 232 | 233 | # as name 234 | for asn in db['nets']: 235 | if text.lower() in db['nets'][asn]['name'].lower(): 236 | add_res('net', db['nets'][asn]['cc'], db['nets'][asn]['name'], '/net/' + asn) 237 | 238 | 239 | return flask.render_template('search_res.html', search_res=res, colors=colors) 240 | -------------------------------------------------------------------------------- /pwui/pwui_tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | import os, math, ipaddress 4 | from pwui_colors import pwui_colors 5 | 6 | # returns % of non-available memory 7 | def get_mem_usage(): 8 | with os.popen('free -b') as p: 9 | for l in p.readlines(): 10 | if l.lower()[0:3] != 'mem': 11 | continue 12 | tmp = l.split() 13 | tot = float(tmp[1]) 14 | if len(tmp) >= 7: 15 | return (tot - float(tmp[6])) / tot * 100.0 16 | return float(tmp[2]) / tot * 100.0 17 | return -1 18 | 19 | # =========================================================================== # 20 | # ip lookup tools (ip to ASn) 21 | # =========================================================================== # 22 | cidr_lookup = {} # key = cidr, val = asn 23 | def _lookup_cidr_build(db): 24 | for asn in db['nets']: 25 | for pfx in db['nets'][asn]['prefix']: 26 | if not pfx in cidr_lookup: 27 | cidr_lookup[pfx] = [] 28 | cidr_lookup[pfx].append(asn) 29 | 30 | def lookup_cidr(db, ip_or_cidr, ipv6=0): 31 | if not len(cidr_lookup): 32 | _lookup_cidr_build(db) 33 | 34 | if '/' in ip_or_cidr: 35 | cidr_base = ip_or_cidr.split('/')[0] 36 | cidr_mask = int(ip_or_cidr.split('/')[1]) 37 | else: 38 | cidr_base = ip_or_cidr 39 | cidr_mask = 128 if ipv6 else 32 40 | 41 | for i in range(cidr_mask, 0, -1): 42 | cidr = str(ipaddress.ip_network(u"%s/%d" % (cidr_base, i), strict=False)) 43 | if cidr in cidr_lookup: 44 | return cidr_lookup[cidr] 45 | 46 | return [] 47 | 48 | 49 | # =========================================================================== # 50 | # page data building tools 51 | # =========================================================================== # 52 | 53 | # ========================= # 54 | # sizing for bubbles & arcs 55 | # ========================= # 56 | def size_facility(fac): 57 | n = len(fac['nets']) 58 | if not n: 59 | v = 0 60 | else: 61 | v = max(0, math.log(n, 2)) 62 | v += 1 63 | 64 | return 2.5 * v + 1 65 | 66 | def size_facility_ix(fac): 67 | return 5 * len(fac['ix']) 68 | 69 | def size_arc(ldir, lind): 70 | v = math.log(100 * ldir + 5*lind + 1) 71 | if v < 1: 72 | v = 1 73 | return 1.5 * v 74 | 75 | def size_landing(ld): 76 | return 2 * len(ld['cables']) + 4, 77 | 78 | # create an empty page data 79 | def page_default(pn=''): 80 | return { 'pagename': pn, 'total': 0, 'col_max': 0, 'data': [], 'bubbles': [], 'arcs': [] } 81 | 82 | # ========================= # 83 | # add bubbles 84 | # ========================= # 85 | def page_add_bubble(page, name, lat, lng, radius, fill, hover, href): 86 | page['bubbles'].append({ 87 | 'name': name, 88 | 'latitude': lat, 89 | 'longitude': lng, 90 | 'radius': radius, 91 | 'border': 1, 92 | 'fillKey': fill, 93 | 'hover': hover, 94 | 'url': href, 95 | }) 96 | 97 | def page_add_facility(page, f): 98 | lat = f['lat'] if f['citydist'] <= 10 else f['citycoords'][0] 99 | lng = f['lng'] if f['citydist'] <= 10 else f['citycoords'][1] 100 | hov = 'city: %s
    ix: %s
    nets: %s
    ' % (f['city'], len(f['ix']), len(f['nets'])) 101 | url = '/fac/%d' % f['id'] 102 | page_add_bubble(page, f['name'], lat, lng, size_facility(f), 'facility', hov, url) 103 | 104 | def page_add_ix(page, f): 105 | hov = 'city: %s
    ix: %s
    nets: %s
    ' % (f['city'], len(f['ix']), len(f['nets'])) 106 | url = '/fac/%d' % f['id'] 107 | page_add_bubble(page, f['name'], f['lat'], f['lng'], size_facility_ix(f), 'facility', hov, url) 108 | 109 | def page_add_landing(page, l): 110 | hov = 'cables: %d
    ' % len(l['cables']) 111 | url = '/landing/%s' % l['id'] 112 | page_add_bubble(page, l['name'], l['lat'], l['lng'], size_landing(l), 'landing', hov, url) 113 | 114 | 115 | # ========================= # 116 | # add arcs 117 | # ========================= # 118 | def page_add_arc(page, org, dst, name, hov, size, color, cclass): 119 | page['arcs'].append({ 120 | 'origin': { 121 | 'latitude': org[0], 122 | 'longitude': org[1], 123 | }, 124 | 'destination': { 125 | 'latitude': dst[0], 126 | 'longitude': dst[1], 127 | }, 128 | 'name': name, 129 | 'hover': hov, 130 | 'orig_width': size, 131 | 'orig_color': color, 132 | 'cclass': cclass, 133 | 'options': { 134 | 'strokeWidth': size, 135 | 'strokeColor': color, 136 | } 137 | }) 138 | 139 | def page_add_link_c2c(page, c1, c2, ldir, lind): 140 | name = '%s to %s' % (c1['cc'], c2['cc']) 141 | hover = 'Direct Outgoing Links: %d
    Indirect Outgoing Links: %d' % (ldir, lind) 142 | size = size_arc(ldir, lind) 143 | color = pwui_colors['link1'] if ldir > 0 else pwui_colors['link2'] 144 | cclass = 'dirlink' if ldir > 0 else 'indlink' 145 | page_add_arc(page, c1['center'], c2['center'], name, hover, size, color, cclass) 146 | 147 | def page_add_link_n2c(db, page, net, c, ldir, lind): 148 | name = '%s to %s' % (net['name'], c['name']) 149 | hover = 'Direct Links: %d
    Indirect Links: %d' % (ldir, lind) 150 | size = size_arc(ldir, lind) 151 | color = pwui_colors['link1'] if ldir > 0 else pwui_colors['link2'] 152 | cclass = 'dirlink' if ldir > 0 else 'indlink' 153 | page_add_arc(page, db['countries'][net['cc']]['center'], c['center'], name, hover, size, color, cclass) 154 | 155 | 156 | # ========================= # 157 | # add country colors 158 | # ========================= # 159 | def page_add_cn(page, cn, val, col, hover): 160 | page['data'].append({ 161 | 'cc': cn['cc'], 162 | 'iso3': cn['iso3'], 163 | 'name': cn['name'], 164 | 'val': val, 165 | 'col': col, 166 | 'hover': hover, 167 | }) 168 | 169 | def page_add_cnl(page, cn, ld, li): 170 | hov = 'Direct Links: %d
    Indirect Links: %d' % (ld, li) 171 | val = 10 * ld + li 172 | page_add_cn(page, cn, val, val, hov) 173 | 174 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2017.7.27.1 2 | chardet==3.0.4 3 | click==6.7 4 | Django==1.8.18 5 | django-countries==5.0 6 | django-handleref==0.1.5 7 | django-inet==0.3.2 8 | django-peeringdb==0.4.0 9 | Flask==0.12.2 10 | Flask-Compress==1.4.0 11 | future==0.16.0 12 | gevent==1.2.2 13 | greenlet==0.4.12 14 | idna==2.6 15 | ipaddress==1.0.18 16 | itsdangerous==0.24 17 | Jinja2==2.9.6 18 | MarkupSafe==1.0 19 | munge==0.4.0 20 | netaddr==0.7.19 21 | peeringdb==0.6.0 22 | pkg-resources==0.0.0 23 | pytz==2017.2 24 | PyYAML==3.12 25 | requests==2.18.4 26 | six==1.11.0 27 | tqdm==4.19.4 28 | twentyc.rpc==0.3.5 29 | Unidecode==0.4.21 30 | urllib3==1.22 31 | Werkzeug==0.12.2 32 | -------------------------------------------------------------------------------- /res/Flaticon.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ixty/wnm/af5bf95178714e7acdcb00c9d6bbbd2c614f64e8/res/Flaticon.eot -------------------------------------------------------------------------------- /res/Flaticon.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ixty/wnm/af5bf95178714e7acdcb00c9d6bbbd2c614f64e8/res/Flaticon.ttf -------------------------------------------------------------------------------- /res/Flaticon.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ixty/wnm/af5bf95178714e7acdcb00c9d6bbbd2c614f64e8/res/Flaticon.woff -------------------------------------------------------------------------------- /res/blank.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ixty/wnm/af5bf95178714e7acdcb00c9d6bbbd2c614f64e8/res/blank.gif -------------------------------------------------------------------------------- /res/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ixty/wnm/af5bf95178714e7acdcb00c9d6bbbd2c614f64e8/res/favicon.png -------------------------------------------------------------------------------- /res/flags-24.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ixty/wnm/af5bf95178714e7acdcb00c9d6bbbd2c614f64e8/res/flags-24.png -------------------------------------------------------------------------------- /res/flags-32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ixty/wnm/af5bf95178714e7acdcb00c9d6bbbd2c614f64e8/res/flags-32.png -------------------------------------------------------------------------------- /res/flaticon.css: -------------------------------------------------------------------------------- 1 | /* 2 | Flaticon icon font: Flaticon 3 | Creation date: 20/12/2017 11:54 4 | */ 5 | 6 | @font-face { 7 | font-family: "Flaticon"; 8 | src: url("./Flaticon.eot"); 9 | src: url("./Flaticon.eot?#iefix") format("embedded-opentype"), 10 | url("./Flaticon.woff") format("woff"), 11 | url("./Flaticon.ttf") format("truetype"), 12 | url("./Flaticon.svg#Flaticon") format("svg"); 13 | font-weight: normal; 14 | font-style: normal; 15 | } 16 | 17 | @media screen and (-webkit-min-device-pixel-ratio:0) { 18 | @font-face { 19 | font-family: "Flaticon"; 20 | src: url("./Flaticon.svg#Flaticon") format("svg"); 21 | } 22 | } 23 | 24 | [class^="flaticon-"]:before, [class*=" flaticon-"]:before, 25 | [class^="flaticon-"]:after, [class*=" flaticon-"]:after { 26 | font-family: Flaticon; 27 | font-size: 20px; 28 | font-style: normal; 29 | padding-left: 2px; 30 | padding-right: 2px; 31 | vertical-align: middle; 32 | } 33 | 34 | .flaticon-caret-down:before { content: "\f100"; font-size: 12px; } 35 | .flaticon-caret-right:before { content: "\f101"; font-size: 12px; } 36 | .flaticon-anchor:before { content: "\f102"; } 37 | .flaticon-flag2:before { content: "\f103"; } 38 | .flaticon-cloud-up:before { content: "\f104"; } 39 | .flaticon-cloud-down:before { content: "\f105"; } 40 | .flaticon-building:before { content: "\f106"; } 41 | .flaticon-cloud-exchange:before { content: "\f107"; } 42 | .flaticon-city:before { content: "\f108"; } 43 | .flaticon-network3:before { content: "\f109"; } 44 | .flaticon-interconnect:before { content: "\f10a"; } 45 | .flaticon-submarine:before { content: "\f10b"; } 46 | .flaticon-flag1:before { content: "\f10c"; } 47 | .flaticon-factory:before { content: "\f10d"; } 48 | .flaticon-mesh:before { content: "\f10e"; } 49 | .flaticon-network2:before { content: "\f10f"; } 50 | .flaticon-address:before { content: "\f110"; } 51 | .flaticon-internet:before { content: "\f111"; } 52 | .flaticon-pipe1:before { content: "\f112"; } 53 | .flaticon-deal2:before { content: "\f113"; } 54 | .flaticon-deal1:before { content: "\f114"; } 55 | .flaticon-office:before { content: "\f115"; } 56 | .flaticon-cloud-net:before { content: "\f116"; } 57 | .flaticon-cable:before { content: "\f117"; } 58 | .flaticon-analytics:before { content: "\f118"; } 59 | .flaticon-network1:before { content: "\f119"; } 60 | .flaticon-earth:before { content: "\f11a"; } -------------------------------------------------------------------------------- /res/popper.min.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) Federico Zivolo 2017 3 | Distributed under the MIT License (license terms are at http://opensource.org/licenses/MIT). 4 | */(function(e,t){'object'==typeof exports&&'undefined'!=typeof module?module.exports=t():'function'==typeof define&&define.amd?define(t):e.Popper=t()})(this,function(){'use strict';function e(e){return e&&'[object Function]'==={}.toString.call(e)}function t(e,t){if(1!==e.nodeType)return[];var o=window.getComputedStyle(e,null);return t?o[t]:o}function o(e){return'HTML'===e.nodeName?e:e.parentNode||e.host}function n(e){if(!e||-1!==['HTML','BODY','#document'].indexOf(e.nodeName))return window.document.body;var i=t(e),r=i.overflow,p=i.overflowX,s=i.overflowY;return /(auto|scroll)/.test(r+s+p)?e:n(o(e))}function r(e){var o=e&&e.offsetParent,i=o&&o.nodeName;return i&&'BODY'!==i&&'HTML'!==i?-1!==['TD','TABLE'].indexOf(o.nodeName)&&'static'===t(o,'position')?r(o):o:window.document.documentElement}function p(e){var t=e.nodeName;return'BODY'!==t&&('HTML'===t||r(e.firstElementChild)===e)}function s(e){return null===e.parentNode?e:s(e.parentNode)}function d(e,t){if(!e||!e.nodeType||!t||!t.nodeType)return window.document.documentElement;var o=e.compareDocumentPosition(t)&Node.DOCUMENT_POSITION_FOLLOWING,i=o?e:t,n=o?t:e,a=document.createRange();a.setStart(i,0),a.setEnd(n,0);var l=a.commonAncestorContainer;if(e!==l&&t!==l||i.contains(n))return p(l)?l:r(l);var f=s(e);return f.host?d(f.host,t):d(e,s(t).host)}function a(e){var t=1=o.clientWidth&&i>=o.clientHeight}),l=0i[e]&&!t.escapeWithReference&&(n=V(p[o],i[e]-('right'===e?p.width:p.height))),se({},o,n)}};return n.forEach(function(e){var t=-1===['left','top'].indexOf(e)?'secondary':'primary';p=de({},p,s[t](e))}),e.offsets.popper=p,e},priority:['left','right','top','bottom'],padding:5,boundariesElement:'scrollParent'},keepTogether:{order:400,enabled:!0,fn:function(e){var t=e.offsets,o=t.popper,i=t.reference,n=e.placement.split('-')[0],r=_,p=-1!==['top','bottom'].indexOf(n),s=p?'right':'bottom',d=p?'left':'top',a=p?'width':'height';return o[s]r(i[s])&&(e.offsets.popper[d]=r(i[s])),e}},arrow:{order:500,enabled:!0,fn:function(e,o){if(!F(e.instance.modifiers,'arrow','keepTogether'))return e;var i=o.element;if('string'==typeof i){if(i=e.instance.popper.querySelector(i),!i)return e;}else if(!e.instance.popper.contains(i))return console.warn('WARNING: `arrow.element` must be child of its popper element!'),e;var n=e.placement.split('-')[0],r=e.offsets,p=r.popper,s=r.reference,d=-1!==['left','right'].indexOf(n),a=d?'height':'width',l=d?'Top':'Left',f=l.toLowerCase(),m=d?'left':'top',c=d?'bottom':'right',g=O(i)[a];s[c]-gp[c]&&(e.offsets.popper[f]+=s[f]+g-p[c]);var u=s[f]+s[a]/2-g/2,b=t(e.instance.popper,'margin'+l).replace('px',''),y=u-h(e.offsets.popper)[f]-b;return y=X(V(p[a]-g,y),0),e.arrowElement=i,e.offsets.arrow={},e.offsets.arrow[f]=Math.round(y),e.offsets.arrow[m]='',e},element:'[x-arrow]'},flip:{order:600,enabled:!0,fn:function(e,t){if(W(e.instance.modifiers,'inner'))return e;if(e.flipped&&e.placement===e.originalPlacement)return e;var o=w(e.instance.popper,e.instance.reference,t.padding,t.boundariesElement),i=e.placement.split('-')[0],n=L(i),r=e.placement.split('-')[1]||'',p=[];switch(t.behavior){case fe.FLIP:p=[i,n];break;case fe.CLOCKWISE:p=K(i);break;case fe.COUNTERCLOCKWISE:p=K(i,!0);break;default:p=t.behavior;}return p.forEach(function(s,d){if(i!==s||p.length===d+1)return e;i=e.placement.split('-')[0],n=L(i);var a=e.offsets.popper,l=e.offsets.reference,f=_,m='left'===i&&f(a.right)>f(l.left)||'right'===i&&f(a.left)f(l.top)||'bottom'===i&&f(a.top)f(o.right),g=f(a.top)f(o.bottom),b='left'===i&&c||'right'===i&&h||'top'===i&&g||'bottom'===i&&u,y=-1!==['top','bottom'].indexOf(i),w=!!t.flipVariations&&(y&&'start'===r&&c||y&&'end'===r&&h||!y&&'start'===r&&g||!y&&'end'===r&&u);(m||b||w)&&(e.flipped=!0,(m||b)&&(i=p[d+1]),w&&(r=j(r)),e.placement=i+(r?'-'+r:''),e.offsets.popper=de({},e.offsets.popper,S(e.instance.popper,e.offsets.reference,e.placement)),e=N(e.instance.modifiers,e,'flip'))}),e},behavior:'flip',padding:5,boundariesElement:'viewport'},inner:{order:700,enabled:!1,fn:function(e){var t=e.placement,o=t.split('-')[0],i=e.offsets,n=i.popper,r=i.reference,p=-1!==['left','right'].indexOf(o),s=-1===['top','left'].indexOf(o);return n[p?'left':'top']=r[o]-(s?n[p?'width':'height']:0),e.placement=L(t),e.offsets.popper=h(n),e}},hide:{order:800,enabled:!0,fn:function(e){if(!F(e.instance.modifiers,'hide','preventOverflow'))return e;var t=e.offsets.reference,o=T(e.instance.modifiers,function(e){return'preventOverflow'===e.name}).boundaries;if(t.bottomo.right||t.top>o.bottom||t.right | BSD-3-Clause */ 2 | !function(){"use strict";function e(e){return r(n(e),arguments)}function t(t,r){return e.apply(null,[t].concat(r||[]))}function r(t,r){var n,i,a,o,p,c,u,f,l,d=1,g=t.length,b="";for(i=0;i=0),o[8]){case"b":n=parseInt(n,10).toString(2);break;case"c":n=String.fromCharCode(parseInt(n,10));break;case"d":case"i":n=parseInt(n,10);break;case"j":n=JSON.stringify(n,null,o[6]?parseInt(o[6]):0);break;case"e":n=o[7]?parseFloat(n).toExponential(o[7]):parseFloat(n).toExponential();break;case"f":n=o[7]?parseFloat(n).toFixed(o[7]):parseFloat(n);break;case"g":n=o[7]?String(Number(n.toPrecision(o[7]))):parseFloat(n);break;case"o":n=(parseInt(n,10)>>>0).toString(8);break;case"s":n=String(n),n=o[7]?n.substring(0,o[7]):n;break;case"t":n=String(!!n),n=o[7]?n.substring(0,o[7]):n;break;case"T":n=Object.prototype.toString.call(n).slice(8,-1).toLowerCase(),n=o[7]?n.substring(0,o[7]):n;break;case"u":n=parseInt(n,10)>>>0;break;case"v":n=n.valueOf(),n=o[7]?n.substring(0,o[7]):n;break;case"x":n=(parseInt(n,10)>>>0).toString(16);break;case"X":n=(parseInt(n,10)>>>0).toString(16).toUpperCase()}s.json.test(o[8])?b+=n:(!s.number.test(o[8])||f&&!o[3]?l="":(l=f?"+":"-",n=n.toString().replace(s.sign,"")),c=o[4]?"0"===o[4]?"0":o[4].charAt(1):" ",u=o[6]-(l+n).length,p=o[6]&&u>0?c.repeat(u):"",b+=o[5]?l+n+p:"0"===c?l+p+n:p+l+n)}return b}function n(e){if(i[e])return i[e];for(var t,r=e,n=[],a=0;r;){if(null!==(t=s.text.exec(r)))n.push(t[0]);else if(null!==(t=s.modulo.exec(r)))n.push("%");else{if(null===(t=s.placeholder.exec(r)))throw new SyntaxError("[sprintf] unexpected placeholder");if(t[2]){a|=1;var o=[],p=t[2],c=[];if(null===(c=s.key.exec(p)))throw new SyntaxError("[sprintf] failed to parse named argument key");for(o.push(c[1]);""!==(p=p.substring(c[0].length));)if(null!==(c=s.key_access.exec(p)))o.push(c[1]);else{if(null===(c=s.index_access.exec(p)))throw new SyntaxError("[sprintf] failed to parse named argument key");o.push(c[1])}t[2]=o}else a|=2;if(3===a)throw new Error("[sprintf] mixing positional and named placeholders is not (yet) supported");n.push(t)}r=r.substring(t[0].length)}return i[e]=n}var s={not_string:/[^s]/,not_bool:/[^t]/,not_type:/[^T]/,not_primitive:/[^v]/,number:/[diefg]/,numeric_arg:/[bcdiefguxX]/,json:/[j]/,not_json:/[^j]/,text:/^[^\x25]+/,modulo:/^\x25{2}/,placeholder:/^\x25(?:([1-9]\d*)\$|\(([^\)]+)\))?(\+)?(0|'[^$])?(-)?(\d+)?(?:\.(\d+))?([b-gijostTuvxX])/,key:/^([a-z_][a-z_\d]*)/i,key_access:/^\.([a-z_][a-z_\d]*)/i,index_access:/^\[(\d+)\]/,sign:/^[\+\-]/},i=Object.create(null);"undefined"!=typeof exports&&(exports.sprintf=e,exports.vsprintf=t),"undefined"!=typeof window&&(window.sprintf=e,window.vsprintf=t,"function"==typeof define&&define.amd&&define(function(){return{sprintf:e,vsprintf:t}}))}(); 3 | //# sourceMappingURL=sprintf.min.js.map 4 | -------------------------------------------------------------------------------- /res/topojson.v1.min.js: -------------------------------------------------------------------------------- 1 | // https://github.com/topojson/topojson-client Version 1.8.0. Copyright 2016 Mike Bostock. 2 | !function(n,t){"object"==typeof exports&&"undefined"!=typeof module?t(exports):"function"==typeof define&&define.amd?define(["exports"],t):t(n.topojson=n.topojson||{})}(this,function(n){"use strict";function t(n){if(!n)return h;var t,r,e=n.scale[0],o=n.scale[1],i=n.translate[0],u=n.translate[1];return function(n,f){f||(t=r=0),n[0]=(t+=n[0])*e+i,n[1]=(r+=n[1])*o+u}}function r(n){if(!n)return h;var t,r,e=n.scale[0],o=n.scale[1],i=n.translate[0],u=n.translate[1];return function(n,f){f||(t=r=0);var c=Math.round((n[0]-i)/e),a=Math.round((n[1]-u)/o);n[0]=c-t,n[1]=a-r,t=c,r=a}}function e(n,t){for(var r,e=n.length,o=e-t;o<--e;)r=n[o],n[o++]=n[e],n[e]=r}function o(n,t){for(var r=0,e=n.length;r>>1;n[o]1){var c,a=[],s={LineString:o,MultiLineString:i,Polygon:i,MultiPolygon:function(n){n.forEach(i)}};u(t),a.forEach(arguments.length<3?function(n){f.push(n[0].i)}:function(n){r(n[0].g,n[n.length-1].g)&&f.push(n[0].i)})}else for(var l=0,h=n.arcs.length;l1)for(var u,f,c=1,a=e(i[0]);ca&&(f=i[0],i[0]=i[c],i[c]=f,a=u);return i})}}function l(n,t){return n[1][2]-t[1][2]}var h=function(){},p=function(n,t){return"GeometryCollection"===t.type?{type:"FeatureCollection",features:t.geometries.map(function(t){return i(n,t)})}:i(n,t)},v=function(n,t){function r(t){var r,e=n.arcs[t<0?~t:t],o=e[0];return n.transform?(r=[0,0],e.forEach(function(n){r[0]+=n[0],r[1]+=n[1]})):r=e[e.length-1],t<0?[r,o]:[o,r]}function e(n,t){for(var r in n){var e=n[r];delete t[e.start],delete e.start,delete e.end,e.forEach(function(n){o[n<0?~n:n]=1}),f.push(e)}}var o={},i={},u={},f=[],c=-1;return t.forEach(function(r,e){var o,i=n.arcs[r<0?~r:r];i.length<3&&!i[1][0]&&!i[1][1]&&(o=t[++c],t[c]=r,t[e]=o)}),t.forEach(function(n){var t,e,o=r(n),f=o[0],c=o[1];if(t=u[f])if(delete u[t.end],t.push(n),t.end=c,e=i[c]){delete i[e.start];var a=e===t?t:t.concat(e);i[a.start=t.start]=u[a.end=e.end]=a}else i[t.start]=u[t.end]=t;else if(t=i[c])if(delete i[t.start],t.unshift(n),t.start=f,e=u[f]){delete u[e.end];var s=e===t?t:e.concat(t);i[s.start=e.start]=u[s.end=t.end]=s}else i[t.start]=u[t.end]=t;else t=[n],i[t.start=f]=u[t.end=c]=t}),e(u,i),e(i,u),t.forEach(function(n){o[n<0?~n:n]||f.push([n])}),f},g=function(n){return u(n,f.apply(this,arguments))},d=function(n){return u(n,s.apply(this,arguments))},y=function(n){function t(n,t){n.forEach(function(n){n<0&&(n=~n);var r=i[n];r?r.push(t):i[n]=[t]})}function r(n,r){n.forEach(function(n){t(n,r)})}function e(n,t){"GeometryCollection"===n.type?n.geometries.forEach(function(n){e(n,t)}):n.type in f&&f[n.type](n.arcs,t)}var i={},u=n.map(function(){return[]}),f={LineString:t,MultiLineString:r,Polygon:r,MultiPolygon:function(n,t){n.forEach(function(n){r(n,t)})}};n.forEach(e);for(var c in i)for(var a=i[c],s=a.length,l=0;l0;){var r=(t+1>>1)-1,o=e[r];if(l(n,o)>=0)break;e[o._=t]=o,e[n._=t=r]=n}}function t(n,t){for(;;){var r=t+1<<1,i=r-1,u=t,f=e[u];if(i0&&(n=e[o],t(e[n._=0]=n,0)),r}},r.remove=function(r){var i,u=r._;if(e[u]===r)return u!==--o&&(i=e[o],(l(i,r)<0?n:t)(e[i._=u]=i,u)),u},r},E=function(n,e){function o(n){f.remove(n),n[1][2]=e(n),f.push(n)}var i=t(n.transform),u=r(n.transform),f=m();return null==e&&(e=c),n.arcs.forEach(function(n){var t,r,c,a,s=[],l=0;for(r=0,c=n.length;r updating PeeringDB ' 21 | echo '========================================' 22 | peeringdb sync || exit 1 23 | cp ~/.peeringdb/peeringdb.sqlite3 ./data-raw/ || cp ./peeringdb.sqlite3 ./data-raw/ || exit 1 24 | echo 25 | 26 | # =========================================================================== # 27 | # merge all databases 28 | # =========================================================================== # 29 | ./wnm/wnm_merge.py || exit 1 30 | 31 | d=$SECONDS 32 | echo "> all done :) ($(($d / 60))m $(($d % 60))s)" 33 | -------------------------------------------------------------------------------- /wnm/wnm_asnames.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import sys, os, json, gzip 5 | from wnm_utils import * 6 | 7 | # ================================= 8 | # vars 9 | # ================================= 10 | urlz = { 11 | 'as-radb.db.gz': 'ftp://ftp.radb.net/radb/dbase/radb.db.gz', 12 | 'as-afrinic.db.gz': 'ftp://ftp.afrinic.net/pub/dbase/afrinic.db.gz', 13 | 'as-arin.db.gz': 'ftp://ftp.arin.net/pub/rr/arin.db', 14 | 'as-ripe.db.gz': 'ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz', 15 | 'as-level3.db.gz': 'ftp://rr.level3.net/pub/rr/level3.db.gz', 16 | 'as-apnic.db.gz': 'https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz', 17 | } 18 | 19 | path_data = './data-raw/' 20 | path_db = './data/asnames.json' 21 | 22 | def db_download(): 23 | count = 0 24 | print '=' * 40 25 | print '> updating AS names database' 26 | print '=' * 40 27 | 28 | for u in urlz: 29 | 30 | path = path_data + u 31 | if urlz[u][-3:] != '.gz': 32 | path = path_data + u[:-3] 33 | 34 | count += get_url(urlz[u], path) 35 | if urlz[u][-3:] != '.gz': 36 | os.system('gzip -f ' + path) 37 | 38 | if count != len(urlz.values()): 39 | print '> errors occured during update' 40 | return -1 41 | 42 | print '=' * 40 43 | print '> done.' 44 | 45 | def db_update(): 46 | out = {} 47 | 48 | for db in urlz: 49 | print '> parsing %s' % db 50 | with gzip.open(path_data + db, 'rb') as f: 51 | lines = f.read().splitlines() 52 | 53 | for i in range(len(lines)-1): 54 | l0 = lines[i] 55 | l1 = lines[i+1] 56 | if l0[0:8] != 'aut-num:' or l1[0:8] != 'as-name:': 57 | continue 58 | l0 = l0.replace('aut-num:', '').strip() 59 | l1 = l1.replace('as-name:', '').strip() 60 | 61 | if '#' in l0: 62 | l0 = l0[0:l0.find('#')] 63 | if '#' in l1: 64 | l1 = l1[0:l1.find('#')] 65 | 66 | l1 = l1.replace('DBP', '').replace('UNSPECIFIED', '') 67 | if not len(l1): 68 | continue 69 | 70 | asn = int(l0[2:]) 71 | name = l1 72 | 73 | if asn not in out: 74 | out[asn] = name 75 | elif len(name) > len(out[asn]): 76 | out[asn] = name 77 | 78 | wnm_save(path_db, out) 79 | 80 | 81 | def usage(): 82 | print '%s usage ' % sys.argv[0] 83 | print '' 84 | print 'commands:' 85 | print ' download -- download rr databases' 86 | print ' rebuild -- rebuild local json database' 87 | print ' update -- download & rebuild' 88 | print 89 | sys.exit(1) 90 | 91 | 92 | if __name__ == '__main__': 93 | if len(sys.argv) < 2: 94 | usage() 95 | sys.exit(-1) 96 | 97 | action = sys.argv[1] 98 | if action == 'download': 99 | if db_download(): 100 | sys.exit(-1) 101 | elif action == 'rebuild': 102 | if db_update(): 103 | sys.exit(-1) 104 | elif action == 'update': 105 | if db_download(): 106 | sys.exit(-1) 107 | if db_update(): 108 | sys.exit(-1) 109 | 110 | -------------------------------------------------------------------------------- /wnm/wnm_bgp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | import os, sys, datetime, time, json, gc, ipaddress 4 | from subprocess import Popen, PIPE 5 | from wnm_utils import * 6 | from tqdm import tqdm 7 | 8 | # ================================= 9 | # vars 10 | # ================================= 11 | 12 | # url for bgp data 13 | 14 | # EU: http://routeviews.org/route-views.linx/bgpdata/2015.01/RIBS/rib.20150105.0800.bz2 15 | # NA: http://routeviews.org/route-views.eqix/bgpdata/2017.11/RIBS/rib.20171113.1600.bz2 16 | # PAC: http://routeviews.org/route-views.sg/bgpdata/2017.11/RIBS/rib.20171112.1000.bz2 17 | # AF: http://routeviews.org/route-views.jinx/bgpdata/2017.11/RIBS/rib.20171112.2000.bz2 18 | # SA: http://routeviews.org/route-views.saopaulo/bgpdata/2017.11/RIBS/rib.20171113.0000.bz2 19 | 20 | # we try to get 1 bgp view from each continent 21 | bgp_provs = [ 'linx', 'eqix', 'jinx', 'saopaulo', 'sg' ] 22 | 23 | url_bgp = 'http://routeviews.org/route-views.%s/bgpdata/%s/RIBS/rib.%s.0000.bz2' 24 | path_bin_bgpdump = './bins/bgpdump' 25 | path_bgp_dir = './data-raw/' 26 | path_bgp_bz2 = path_bgp_dir + 'bgp-%s.bz2' 27 | path_bgp_bz2_bak = path_bgp_dir + 'bgp-%s.bak.bz2' 28 | path_bgp_db = './data/bgp.json' 29 | 30 | # ================================= 31 | # funcs 32 | # ================================= 33 | def bgp_download(ix): 34 | path = path_bgp_bz2 % ix 35 | pbak = path_bgp_bz2_bak % ix 36 | 37 | # backup current bgp db if any 38 | if os.path.exists(path): 39 | if os.path.exists(pbak): 40 | os.unlink(pbak) 41 | os.rename(path, pbak) 42 | 43 | # try to download today's db 44 | d = datetime.date.today() 45 | url = url_bgp % (ix, d.strftime('%Y.%m'), d.strftime('%Y%m%d')) 46 | if not get_url(url, path): 47 | # try yesterday's db 48 | d = datetime.date.today() - datetime.timedelta(days=1) 49 | url = url_bgp % (ix, d.strftime('%Y.%m'), d.strftime('%Y%m%d')) 50 | if not get_url(url, path): 51 | return -1 52 | 53 | print '> downloaded [%s] bgp database.' % ix 54 | return 0 55 | 56 | def bgp_download_all(): 57 | print '=' * 40 58 | print '> updating BGP database' 59 | print '=' * 40 60 | ret = 0 61 | for ix in bgp_provs: 62 | if bgp_download(ix): 63 | print '> failed to download [%s] bgp database.' % ix 64 | ret -= 1 65 | print '> done %d/%d' % (len(bgp_provs) + ret, len(bgp_provs)) 66 | return ret 67 | 68 | def bgp_load(path): 69 | pop = Popen([path_bin_bgpdump, '-m', path], stdout=PIPE, stderr=PIPE) 70 | data = pop.stdout.read() 71 | try: 72 | pop.kill() 73 | except: 74 | pass 75 | return data 76 | 77 | def bgp_build(asl, ix): 78 | path = path_bgp_bz2 % ix 79 | 80 | print '> loading %s' % path 81 | data = bgp_load(path) 82 | lines = data.splitlines() 83 | 84 | # free memory 85 | data = '' 86 | gc.collect() 87 | 88 | print '> processing %s' % path 89 | for l in tqdm(lines): 90 | if not len(l): 91 | break 92 | tmp = ascii_clean(l).split('|') 93 | if len(tmp) < 7: 94 | print '! %s ' % l 95 | continue 96 | pfx = tmp[5] 97 | cstr = tmp[6].replace('{', '').replace('}', '').replace(',', ' ') 98 | chain = [ int(_) for _ in cstr.split(' ') ] 99 | asn = chain[-1] 100 | # print 'cidr[%s] chain[%r] as[%d]' % (pfx, chain, asn) 101 | 102 | for i in range(len(chain)-1): 103 | asn_link(asl, chain[i], chain[i+1]) 104 | 105 | # skip private prefixes 106 | p = ipaddress.ip_network(unicode(pfx), strict=0) 107 | if p.is_multicast or p.is_private or p.is_unspecified or p.is_reserved or p.is_loopback or p.is_link_local: 108 | continue 109 | 110 | # add prefix to AS 111 | aso = asn_check(asl, asn) 112 | if pfx not in aso['prefix']: 113 | aso['prefix'].append(pfx) 114 | if ':' in pfx: 115 | aso['prefix6'] += 1 116 | else: 117 | aso['prefix4'] += 1 118 | 119 | # free memory 120 | lines = [] 121 | gc.collect() 122 | return 0 123 | 124 | def bgp_build_all(): 125 | asl = {} 126 | 127 | # build all bgp route views 128 | for ix in bgp_provs: 129 | # for ix in ['jinx']: 130 | bgp_build(asl, ix) 131 | 132 | # dump output db 133 | wnm_save(path_bgp_db, asl) 134 | 135 | return 0 136 | 137 | # =========================================================================== # 138 | # utils 139 | # =========================================================================== # 140 | def ascii_clean(s): 141 | return filter(lambda x: x in string.printable, s) 142 | 143 | def asn_check(asl, asn): 144 | if not asn in asl: 145 | asl[asn] = {} 146 | asl[asn]['asn'] = asn 147 | asl[asn]['prefix'] = [] 148 | asl[asn]['prefix4'] = 0 149 | asl[asn]['prefix6'] = 0 150 | asl[asn]['links'] = [] 151 | return asl[asn] 152 | 153 | def asn_link(asl, asn1, asn2): 154 | as1 = asn_check(asl, asn1) 155 | as2 = asn_check(asl, asn2) 156 | 157 | if asn1 not in as2['links']: 158 | as2['links'].append(asn1) 159 | if asn2 not in as1['links']: 160 | as1['links'].append(asn2) 161 | 162 | 163 | # ================================= 164 | # main 165 | # ================================= 166 | if __name__ == '__main__': 167 | if not os.path.exists(path_bgp_dir): 168 | os.mkdir(path_bgp_dir) 169 | 170 | if len(sys.argv) < 2: 171 | print 'usage: %s [update|download|rebuild|test]' % sys.argv[0] 172 | sys.exit(-1) 173 | 174 | action = sys.argv[1] 175 | if action == 'update': 176 | if bgp_download_all(): 177 | print '[-] error downloading bgp view' 178 | sys.exit(-1) 179 | if bgp_build_all(): 180 | print '[-] error building bgp db' 181 | sys.exit(-1) 182 | 183 | elif action == 'download': 184 | if bgp_download_all(): 185 | print '[-] error downloading bgp view' 186 | sys.exit(-1) 187 | 188 | elif action == 'rebuild': 189 | if bgp_build_all(): 190 | print '[-] error building bgp db' 191 | sys.exit(-1) 192 | 193 | elif action == 'test': 194 | 195 | print '> loading db..' 196 | with open(path_bgp_db, 'rb') as f: 197 | db = json.load(f) 198 | print json.dumps(db['21502'], indent=4, sort_keys=1) 199 | 200 | else: 201 | print '[-] unknown command "%s"' % action 202 | print 'usage: %s [update|download|rebuild|test]' % sys.argv[0] 203 | sys.exit(-1) 204 | 205 | sys.exit(0) 206 | -------------------------------------------------------------------------------- /wnm/wnm_geo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | import sys, os, json, csv, gzip 4 | from wnm_utils import * 5 | 6 | # submarine cable database 7 | url = 'http://download.maxmind.com/download/worldcities/worldcitiespop.txt.gz' 8 | path_raw = './data-raw/worldcitiespop.txt.gz' 9 | path_out = './data/cities.json' 10 | 11 | def geo_download(): 12 | print '=' * 40 13 | print '> updating world cities database' 14 | print '=' * 40 15 | if not get_url(url, path_raw, chunk_size=2048): 16 | return -1 17 | 18 | print '> done' 19 | return 0 20 | 21 | def geo_rebuild(): 22 | 23 | geo = {} 24 | n = 0 25 | 26 | print '> processing %s' % path_raw 27 | with gzip.open(path_raw, 'rb') as f: 28 | rd = csv.reader(f, delimiter=',') 29 | hdr = rd.next() 30 | for row in rd: 31 | cc = row[0].upper() 32 | ct = row[1].decode('latin-1') 33 | ct = ct.lower().replace(' ', '-').encode('utf-8') 34 | city = row[2].decode('latin-1') 35 | city = city.lower().replace(' ', '-').encode('utf-8') 36 | state = row[3] 37 | pop = int(row[4]) if len(row[4]) else 0 38 | lat = float(row[5]) 39 | lng = float(row[6]) 40 | 41 | if not cc in geo: 42 | geo[cc] = {} 43 | 44 | # include state in cityname for the us 45 | if cc == 'US': 46 | city = '%s-%s' % (state, city) 47 | ct = '%s-%s' % (state, ct) 48 | 49 | # update if city is not known yet, or if its a city with same name but more population 50 | if not city in geo[cc] or pop > geo[cc][city]['pop']: 51 | geo[cc][city] = {'lat': lat, 'lng': lng, 'pop': pop } 52 | if ct != city: 53 | geo[cc][ct] = {'lat': lat, 'lng': lng, 'pop': pop } 54 | # else: pass 55 | 56 | n += 1 57 | 58 | geo2 = {} 59 | for cc in geo: 60 | geo2[cc] = {} 61 | for city in geo[cc]: 62 | geo2[cc][city] = [ geo[cc][city]['lat'], geo[cc][city]['lng'] ] 63 | 64 | wnm_save(path_out, geo2) 65 | 66 | return 0 67 | 68 | 69 | # ================================= 70 | # main 71 | # ================================= 72 | def usage(): 73 | print '%s usage ' % sys.argv[0] 74 | print '' 75 | print 'commands:' 76 | print ' download -- download geo cities database' 77 | print ' rebuild -- rebuild local json database' 78 | print ' update -- download & rebuild' 79 | print 80 | sys.exit(1) 81 | 82 | if __name__ == '__main__': 83 | 84 | if len(sys.argv) < 2: 85 | usage() 86 | sys.exit(-1) 87 | 88 | action = sys.argv[1] 89 | 90 | if action == 'download': 91 | geo_download() 92 | 93 | elif action == 'update': 94 | if geo_download(): 95 | sys.exit(-1) 96 | if geo_rebuild(): 97 | sys.exit(-1) 98 | 99 | elif action == 'rebuild': 100 | if geo_rebuild(): 101 | sys.exit(-1) 102 | 103 | else: 104 | print '[-] unknown command "%s"' % action 105 | sys.exit(-1) 106 | 107 | sys.exit(0) 108 | 109 | -------------------------------------------------------------------------------- /wnm/wnm_nics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | import sys, os, socket, struct, urllib2, json, bisect 4 | from wnm_utils import * 5 | 6 | # ================================= 7 | # vars 8 | # ================================= 9 | nics = { 10 | 'afrinic': 'ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest', 11 | 'apnic': 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest', 12 | 'arin': 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest', 13 | 'lacnic': 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest', 14 | 'ripe': 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest', 15 | } 16 | 17 | path_nic = './data-raw/' 18 | path_data = './data/' 19 | path_db = path_data + 'nics.json' 20 | default_db = { 'asn': {}, 'ipv4': {}, 'ipv6': {}, 'dbg': {} } 21 | db = default_db 22 | keys_asn = [] 23 | keys_ipv4 = [] 24 | keys_ipv6 = [] 25 | 26 | 27 | # ================================= 28 | # funcs 29 | # ================================= 30 | 31 | def parse_rir(db, data): 32 | header = 0 33 | 34 | for l in data.splitlines(False): 35 | l = l.strip() 36 | # skip coments & empty lines 37 | if l[0] == '#' or not len(l): 38 | continue 39 | tmp = l.split('|') 40 | 41 | # parse header 42 | if not header: 43 | header = 1 44 | version, nic, records, enddate = tmp[0], tmp[1], int(tmp[3]), tmp[5] 45 | if version != '2' and version != '2.3': 46 | print '[-1] unknown version' 47 | return 0 48 | print '[%7s] date: %s records %d ' % (nic, enddate, records) 49 | continue 50 | 51 | # parse summary lines 52 | if tmp[-1] == 'summary': 53 | continue 54 | 55 | # normal entry 56 | fields = ['registry', 'cc', 'type', 'start', 'value', 'date', 'status', 'extensions'] 57 | entry = dict(zip(fields, tmp)) 58 | 59 | entry['intstart'] = val_to_int(entry['type'], entry['start']) 60 | entry['count'] = int(entry['value']) 61 | entry['reg'] = nic 62 | 63 | # add it based on type 64 | if entry['type'] == 'ipv4': 65 | startval = ipv4_to_int(entry['start']) 66 | entry['cidr'] = entry['start'] 67 | for i in range(32): 68 | if (1 << i) == entry['count']: 69 | entry['cidr'] += '/%d' % (32 - i) 70 | break 71 | if not '/' in entry['cidr']: 72 | entry['cidr'] += '/?' 73 | elif entry['type'] == 'ipv6': 74 | startval = ipv6_to_int(entry['start']) 75 | entry['count'] = 1 << (128 - int(entry['value'])) 76 | entry['cidr'] = entry['start'] + '/' + entry['value'] 77 | else: 78 | startval = int(entry['start']) 79 | for i in range(int(startval) + 1, int(startval) + int(entry['count'])): 80 | db[entry['type']][str(i)] = entry 81 | 82 | 83 | db[entry['type']][startval] = entry 84 | 85 | return db 86 | 87 | def db_download(): 88 | count = 0 89 | print '=' * 40 90 | print '> updating nics database' 91 | print '=' * 40 92 | 93 | for u in nics: 94 | count += get_url(nics[u], path_nic + 'nic-' + u + '.txt') 95 | 96 | if count != len(nics): 97 | print '> errors occured during update' 98 | return -1 99 | 100 | return 0 101 | 102 | def db_consolidate(db): 103 | 104 | print '=' * 40 105 | print '> consolidating database..' 106 | 107 | for f in nics: 108 | with open(path_nic + 'nic-' + f + '.txt', 'rb') as f: 109 | parse_rir(db, f.read()) 110 | 111 | print '> total rirs info [asn: %d, ipv4: %d, ipv6: %d]' % (len(db['asn']), len(db['ipv4']), len(db['ipv6'])) 112 | 113 | wnm_save(path_db, db) 114 | 115 | def db_update(download=1): 116 | if download: 117 | if db_download() < 0: 118 | return -1 119 | 120 | db_consolidate(default_db) 121 | return 0 122 | 123 | def db_load(): 124 | global db 125 | try: 126 | db1 = wnm_load(path_db) 127 | for type in ['asn', 'ipv4', 'ipv6']: 128 | for k in db1[type]: 129 | db[type][int(k)] = db1[type][k] 130 | print '> loaded db (asn: %d, ipv4: %d, ipv6: %d)' % (len(db['asn']), len(db['ipv4']), len(db['ipv6'])) 131 | return db 132 | except: 133 | print '[-] error loading database file' 134 | return default_db 135 | 136 | 137 | def lookup(type, val, warn=1): 138 | global keys_ipv4, keys_ipv6, keys_asn 139 | 140 | # build sorted key tables if needed 141 | if type == 'ipv4': 142 | if not len(keys_ipv4): 143 | keys_ipv4 = sorted(db['ipv4'].keys()) 144 | keys = keys_ipv4 145 | elif type == 'ipv6': 146 | if not len(keys_ipv6): 147 | keys_ipv6 = sorted(db['ipv6'].keys()) 148 | keys = keys_ipv6 149 | else: 150 | if not len(keys_asn): 151 | keys_asn = sorted(db['asn'].keys()) 152 | keys = keys_asn 153 | 154 | # convert to decimal 155 | intval = val_to_int(type, val) 156 | 157 | # lookup 158 | ind = bisect.bisect_left(keys, intval) 159 | if ind < len(keys): 160 | if keys[ind] == intval: 161 | return db[type][keys[ind]] 162 | else: 163 | e = db[type][keys[ind-1]] 164 | if intval >= e['intstart'] and intval < e['intstart'] + e['count']: 165 | return e 166 | 167 | if warn: 168 | print '[-] cant resolve [%s:%s]' % (type, val) 169 | return None 170 | 171 | def lookup_ipv4(ipv4): 172 | return lookup('ipv4', ipv4) 173 | 174 | def lookup_ipv6(ipv6): 175 | return lookup('ipv6', ipv6) 176 | 177 | def lookup_asn(asn): 178 | return lookup('asn', asn) 179 | 180 | 181 | # ================================= 182 | # main 183 | # ================================= 184 | def usage(): 185 | print '%s usage ' % sys.argv[0] 186 | print '' 187 | print 'commands:' 188 | print ' download -- download nics database' 189 | print ' rebuild -- rebuild local json database' 190 | print ' update -- download & rebuild' 191 | print ' lookup ipv4_addr -- lookup an ipv4 address' 192 | print ' lookup ipv4 addr -- lookup an ipv4 address' 193 | print ' lookup ipv6 addr -- lookup an ipv6 address' 194 | print ' lookup asn addr -- lookup an AS number' 195 | print 196 | sys.exit(1) 197 | 198 | if __name__ == '__main__': 199 | if not os.path.exists(path_nic): 200 | os.mkdir(path_nic) 201 | 202 | if not os.path.exists(path_data): 203 | os.mkdir(path_data) 204 | 205 | if len(sys.argv) < 2: 206 | usage() 207 | sys.exit(-1) 208 | 209 | action = sys.argv[1] 210 | if action == 'download': 211 | if db_download(): 212 | sys.exit(-1) 213 | elif action == 'update': 214 | if db_update(): 215 | sys.exit(-1) 216 | elif action == 'rebuild': 217 | if db_update(0): 218 | sys.exit(-1) 219 | elif action == 'lookup': 220 | if len(sys.argv) == 3: 221 | type = 'ipv4' 222 | val = sys.argv[2] 223 | elif len(sys.argv) == 4: 224 | type = sys.argv[2] 225 | val = sys.argv[3] 226 | else: 227 | usage() 228 | db_load() 229 | print json.dumps(lookup(type, val), indent=4) 230 | else: 231 | print '[-] unknown command "%s"' % action 232 | sys.exit(-1) 233 | 234 | sys.exit(0) 235 | 236 | 237 | 238 | -------------------------------------------------------------------------------- /wnm/wnm_scd.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | import sys, os, json, re, glob, datetime 4 | from wnm_utils import * 5 | 6 | # submarine cable database 7 | url = 'https://github.com/telegeography/www.submarinecablemap.com/archive/master.zip' 8 | path_raw = './data-raw/' 9 | path_tmp = path_raw + 'cables.zip' 10 | path_db = path_raw + 'cables' 11 | path_out = './data/scdb.json' 12 | 13 | def scd_download(): 14 | print '=' * 40 15 | print '> updating submarine cable database' 16 | print '=' * 40 17 | if not get_url(url, path_tmp, chunk_size=512): 18 | # retry once, github can be annoying sometimes 19 | if not get_url(url, path_tmp, chunk_size=512): 20 | return -1 21 | 22 | if os.system('unzip -q -d %s %s' % (path_raw, path_tmp)): 23 | print '> failed to unzip %s' % path_tmp 24 | return -1 25 | 26 | if os.system('mv %s/www.submarinecablemap.com-master/public/api/v1/ %s' % (path_raw, path_db)): 27 | return -1 28 | 29 | if os.system('rm -rf %s/www.submarinecablemap.com-master' % path_raw): 30 | return -1 31 | 32 | if os.system('rm -rf %s' % path_tmp): 33 | return -1 34 | 35 | return 0 36 | 37 | def scd_rebuild(): 38 | 39 | def build_db_from_dir(path): 40 | data = [] 41 | for f in glob.glob('%s/*' % path): 42 | if 'all.json' in f: 43 | continue 44 | if 'search.json' in f: 45 | continue 46 | with open(f, 'rb') as f: 47 | data += [ json.load(f) ] 48 | return data 49 | 50 | # country name to iso2 country code 51 | ccmap = {"Andorra":"AD","United Arab Emirates":"AE","Afghanistan":"AF","Antigua and Barbuda":"AG","Anguilla":"AI","Albania":"AL","Armenia":"AM","Netherlands Antilles":"AN","Angola":"AO","Antarctica":"AQ","Argentina":"AR","American Samoa":"AS","Austria":"AT","Australia":"AU","Aruba":"AW","ALA Aland Islands":"AX","Azerbaijan":"AZ","Bosnia and Herzegovina":"BA","Barbados":"BB","Bangladesh":"BD","Belgium":"BE","Burkina Faso":"BF","Bulgaria":"BG","Bahrain":"BH","Burundi":"BI","Benin":"BJ","Saint-Barthélemy":"BL","Saint Barthélemy":"BL","Bermuda":"BM","Brunei Darussalam":"BN","Brunei":"BN","Bolivia":"BO","Bonaire":"BQ","Sint Eustatius and Saba":"BQ","Brazil":"BR","Bahamas":"BS","Bhutan":"BT","Bouvet Island":"BV","Botswana":"BW","Belarus":"BY","Belize":"BZ","Canada":"CA","Cocos (Keeling) Islands":"CC","Congo, (Kinshasa)":"CD","Congo, Rep.":"CD","Congo, Dem. Rep.":"CD","Central African Republic":"CF","Congo (Brazzaville)":"CG","Switzerland":"CH","Côte d'Ivoire":"CI","Cook Islands":"CK","Chile":"CL","Cameroon":"CM","China":"CN","Colombia":"CO","Costa Rica":"CR","Cuba":"CU","Cape Verde":"CV","Curaçao":"CW","Christmas Island":"CX","Cyprus":"CY","Czech Republic":"CZ","Germany":"DE","Djibouti":"DJ","Denmark":"DK","Tyra":"DK","Valdemar":"DK","South Arne":"DK","Dominica":"DM","Dominican Republic":"DO","Algeria":"DZ","Ecuador":"EC","Estonia":"EE","Egypt":"EG","Western Sahara":"EH","Eritrea":"ER","Spain":"ES","Ethiopia":"ET","Europe":"EU","Finland":"FI","Fiji":"FJ","Falkland Islands (Malvinas)":"FK","Micronesia, Federated States of":"FM","Federated States of Micronesia":"FM","Chuuk":"FM","Faroe Islands":"FO","Faeroe Islands":"FO","France":"FR","Gabon":"GA","United Kingdom":"GB","Grenada":"GD","Georgia":"GE","French Guiana":"GF","Guernsey":"GG","Ghana":"GH","Gibraltar":"GI","Greenland":"GL","Gambia":"GM","Guinea":"GN","Guadeloupe":"GP","Equatorial Guinea":"GQ","Greece":"GR","South Georgia and the South Sandwich Islands":"GS","Guatemala":"GT","Guam":"GU","Guinea-Bissau":"GW","Guyana":"GY","Hong Kong, SAR China":"HK","Heard and Mcdonald Islands":"HM","Honduras":"HN","Croatia":"HR","Haiti":"HT","Hungary":"HU","Indonesia":"ID","Ireland":"IE","Israel":"IL","Isle of Man":"IM","India":"IN","British Indian Ocean Territory":"IO","Iraq":"IQ","Iran, Islamic Republic of":"IR","Iran":"IR","Iceland":"IS","Italy":"IT","Jersey":"JE","Jamaica":"JM","Jordan":"JO","Japan":"JP","Kenya":"KE","Kyrgyzstan":"KG","Cambodia":"KH","Kiribati":"KI","Comoros":"KM","Saint Kitts and Nevis":"KN","Korea (North)":"KP","Korea (South)":"KR","Korea, Rep.":"KR","Kuwait":"KW","Cayman Islands":"KY","Kazakhstan":"KZ","Lao PDR":"LA","Lebanon":"LB","Saint Lucia":"LC","Liechtenstein":"LI","Sri Lanka":"LK","Liberia":"LR","Lesotho":"LS","Lithuania":"LT","Luxembourg":"LU","Latvia":"LV","Libya":"LY","Morocco":"MA","Monaco":"MC","Moldova":"MD","Montenegro":"ME","Saint-Martin (French part)":"MF","Saint Martin":"MF","SaintMartin":"MF","Madagascar":"MG","Marshall Islands":"MH","Republic of Marshall Islands":"MH","Macedonia, Republic of":"MK","Mali":"ML","Myanmar":"MM","Mongolia":"MN","Macao, SAR China":"MO","Northern Mariana Islands":"MP","Saipan":"MP","Martinique":"MQ","Mauritania":"MR","Montserrat":"MS","Malta":"MT","Mauritius":"MU","Maldives":"MV","Malawi":"MW","Mexico":"MX","Malaysia":"MY","Mozambique":"MZ","Namibia":"NA","New Caledonia":"NC","Niger":"NE","Norfolk Island":"NF","Nigeria":"NG","Nicaragua":"NI","Netherlands":"NL","Norway":"NO","Nepal":"NP","Nauru":"NR","Niue":"NU","New Zealand":"NZ","Oman":"OM","Panama":"PA","Peru":"PE","French Polynesia":"PF","Papua New Guinea":"PG","Philippines":"PH","Pakistan":"PK","Poland":"PL","Saint Pierre and Miquelon":"PM","Pitcairn":"PN","Puerto Rico":"PR","Palestinian Territory":"PS","Portugal":"PT","Palau":"PW","Paraguay":"PY","Qatar":"QA","Réunion":"RE","Romania":"RO","Serbia":"RS","Russian Federation":"RU","Russia":"RU","Rwanda":"RW","Saudi Arabia":"SA","Solomon Islands":"SB","Seychelles":"SC","Sudan":"SD","Sweden":"SE","Singapore":"SG","Saint Helena":"SH","Slovenia":"SI","Svalbard and Jan Mayen Islands":"SJ","Slovakia":"SK","Sierra Leone":"SL","San Marino":"SM","Senegal":"SN","Somalia":"SO","Suriname":"SR","South Sudan":"SS","Sao Tome and Principe":"ST","El Salvador":"SV","Sint Maarten":"SX","Syrian Arab Republic":"SY","Syria":"SY","Swaziland":"SZ","Turks and Caicos Islands":"TC","Chad":"TD","French Southern Territories":"TF","Togo":"TG","Thailand":"TH","Tajikistan":"TJ","Tokelau":"TK","Timor-Leste":"TL","Turkmenistan":"TM","Tunisia":"TN","Tonga":"TO","Turkey":"TR","Trinidad and Tobago":"TT","Tuvalu":"TV","Taiwan, Republic of China":"TW","Taiwan":"TW","Tanzania, United Republic of":"TZ","Tanzania":"TZ","Ukraine":"UA","Uganda":"UG","US Minor Outlying Islands":"UM","United States of America":"US","United States":"US","Uruguay":"UY","Uzbekistan":"UZ","Holy See (Vatican City State)":"VA","Saint Vincent and Grenadines":"VC","Saint Vincent and the Grenadines":"VC","Venezuela":"VE","British Virgin Islands":"VG","Virgin Islands (U.K.)":"VG","Virgin Islands, US":"VI","Virgin Islands (U.S.)":"VI","Viet Nam":"VN","Vietnam":"VN","Vanuatu":"VU","Wallis and Futuna Islands":"WF","Wallis and Futuna":"WF","Samoa":"WS","Kosovo":"XK","Yemen":"YE","Mayotte":"YT","South Africa":"ZA","Zambia":"ZM","Zimbabwe":"ZW","Canary Islands":"ES"} 52 | 53 | countries = {} 54 | landing2cc = {} 55 | for cn in build_db_from_dir(path_db + '/country/'): 56 | n = cn['name'].encode('utf-8') 57 | c = {} 58 | c['cc'] = ccmap[n] 59 | c['name'] = n 60 | c['cables'] = [ str(_['cable_id']) for _ in cn['cables'] ] 61 | c['landings'] = [ str(_['landing_point_id']) for _ in cn['landing_points'] ] 62 | 63 | countries[c['cc']] = c 64 | 65 | for lid in c['landings']: 66 | if lid in landing2cc and landing2cc[lid] != c['cc']: 67 | print '> warning landing-point %s already mapped to %s (and %s??)' % (lid, landing2cc[lid], c['cc']) 68 | else: 69 | landing2cc[lid] = c['cc'] 70 | 71 | landings = {} 72 | for ld in build_db_from_dir(path_db + '/landing-point/'): 73 | l = {} 74 | lid = str(ld['city_id']) 75 | l['cc'] = landing2cc[lid] 76 | l['id'] = lid 77 | l['name'] = ld['name'] 78 | l['lat'] = ld['latitude'] 79 | l['lng'] = ld['longitude'] 80 | l['cables'] = [ str(_['cable_id']) for _ in ld['cables'] ] 81 | 82 | landings[lid] = l 83 | 84 | cables = {} 85 | for ca in build_db_from_dir(path_db + '/cable/'): 86 | cid = str(ca['cable_id']) 87 | 88 | def _length(s): 89 | if s == 'n.a.': 90 | return -1 91 | return float(s.replace(',', '').replace(' ', '').replace('km', '')) 92 | 93 | c = {} 94 | c['id'] = cid 95 | c['name'] = ca['name'].encode('utf-8') 96 | c['rfs'] = ca['rfs'].split(' ')[-1] 97 | c['length'] = _length(ca['length']) 98 | c['url'] = ca['url'] if ca['url'] is not None else '' 99 | c['notes'] = ca['notes'] if ca['notes'] is not None else '' 100 | c['landings'] = [ str(_['landing_point_id']) for _ in ca['landing_points'] ] 101 | c['countries'] = list(set([ landings[i]['cc'] for i in c['landings'] if i in landings ])) 102 | 103 | own = ca['owners'].replace(',', ', ') 104 | while own.find(' ') >= 0: 105 | own = own.replace(' ', ' ') 106 | c['owners'] = own.split(', ') 107 | 108 | # is cable ready for service? 109 | month = ca['rfs'].split(' ')[-2] if len(ca['rfs'].split(' ')) > 1 else 'December' 110 | months = { 'January':1, 'February':2, 'March':3, 'April':4, 'May':5, 'June':6, 'July':7, 'August':8, 'September':9, 'October':10, 'November':11, 'December':12, 'Q1':3, 'Q2':6, 'Q3':9, 'Q4': 12 } 111 | if month not in months.keys(): 112 | month = 'December' 113 | try: 114 | m = months[month] 115 | y = int(c['rfs']) 116 | today = datetime.date.today() 117 | 118 | if today.year > y: 119 | c['ready'] = 1 120 | elif today.year < y: 121 | c['ready'] = 0 122 | else: 123 | c['ready'] = 1 if m < today.month else 0 124 | except: 125 | c['ready'] = 0 126 | 127 | cables[cid] = c 128 | 129 | # sanity check 130 | for cc in countries: 131 | for lid in countries[cc]['landings']: 132 | if lid not in landings: 133 | print '> unknown landing %s for country %s' % (lid, cc) 134 | countries[cc]['landings'] = [ lid for lid in countries[cc]['landings'] if lid in landings ] 135 | for cid in cables: 136 | for lid in cables[cid]['landings']: 137 | if lid not in landings: 138 | print '> unknown landing %s for cable %s' % (lid, cid) 139 | cables[cid]['landings'] = [ lid for lid in cables[cid]['landings'] if lid in landings ] 140 | 141 | scdb = { 142 | 'countries': countries, 143 | 'landings': landings, 144 | 'cables': cables, 145 | } 146 | 147 | wnm_save(path_out, scdb) 148 | 149 | return 0 150 | 151 | 152 | # ================================= 153 | # main 154 | # ================================= 155 | def usage(): 156 | print '%s usage ' % sys.argv[0] 157 | print '' 158 | print 'commands:' 159 | print ' download -- download submarinecablemap database' 160 | print ' rebuild -- rebuild local json database' 161 | print ' update -- download & rebuild' 162 | print 163 | sys.exit(1) 164 | 165 | if __name__ == '__main__': 166 | 167 | if len(sys.argv) < 2: 168 | usage() 169 | sys.exit(-1) 170 | 171 | action = sys.argv[1] 172 | 173 | if action == 'download': 174 | scd_download() 175 | 176 | elif action == 'update': 177 | if scd_download(): 178 | sys.exit(-1) 179 | if scd_rebuild(): 180 | sys.exit(-1) 181 | 182 | elif action == 'rebuild': 183 | if scd_rebuild(): 184 | sys.exit(-1) 185 | 186 | else: 187 | print '[-] unknown command "%s"' % action 188 | sys.exit(-1) 189 | 190 | sys.exit(0) 191 | 192 | -------------------------------------------------------------------------------- /wnm/wnm_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | import os, sys, json, struct, urllib, urllib2, socket, time, string, gzip 4 | 5 | def h2bin(x): 6 | return x.replace(' ', '').replace('\n', '').decode('hex') 7 | 8 | def hdstr(x): 9 | return ''.join( [ '%.2x' % ord(c) for c in x] ) 10 | 11 | def ascii_clean(s): 12 | return filter(lambda x: x in string.printable, s) 13 | 14 | def get_url(url, fname, chunk_size=8192*16): 15 | def report(size_dled, size_tot, error=0): 16 | sys.stdout.write(' ' * 80 + '\r') 17 | if error: 18 | sys.stdout.write('> error downloading %s' % fname + '\n') 19 | elif size_dled >= size_tot: 20 | te = time.time() 21 | sys.stdout.write('> downloaded %s in %.1f secs' % (fname, te-ts) + '\n') 22 | else: 23 | sys.stdout.write('> downloading %s progress %.1f%%\r' % (fname, 100.0 * float(size_dled) / size_tot)) 24 | sys.stdout.flush() 25 | 26 | try: 27 | print '> %s' % url 28 | ts = time.time() 29 | f = open(fname, 'wb+') 30 | ans = urllib2.urlopen(url); 31 | size_tot = int(ans.info().getheader('Content-Length').strip()) 32 | size_dled = 0 33 | while 1: 34 | chunk = ans.read(chunk_size) 35 | if not chunk: 36 | break 37 | size_dled += len(chunk) 38 | f.write(chunk) 39 | if size_dled < size_tot: 40 | report(size_dled, size_tot) 41 | report(size_dled, size_tot) 42 | f.close() 43 | return 1 44 | except: 45 | report(0, 0, 1) 46 | return 0 47 | 48 | def ipv4_to_int(ipstr): 49 | return struct.unpack('!I', socket.inet_pton(socket.AF_INET, ipstr))[0] 50 | 51 | def int_to_ipv4(ipint): 52 | return socket.inet_ntop(socket.AF_INET, struct.pack('!I', ipint)) 53 | 54 | def ipv6_to_int(ipstr): 55 | _str = socket.inet_pton(socket.AF_INET6, ipstr) 56 | a, b = struct.unpack('!2Q', _str) 57 | return (a << 64) | b 58 | 59 | def int_to_ipv6(ipint): 60 | a = ipint >> 64 61 | b = ipint & ((1 << 64) - 1) 62 | return socket.inet_ntop(socket.AF_INET6, struct.pack('!2Q', a, b)) 63 | 64 | def val_to_int(type, val): 65 | if type == 'ipv4': 66 | return ipv4_to_int(val) 67 | elif type == 'ipv6': 68 | return ipv6_to_int(val) 69 | else: 70 | return int(val) 71 | 72 | def wnm_save(path, obj): 73 | print '> saving to %s.gz (%d items)\n' % (path, len(obj)) 74 | with gzip.open(path + '.gz', 'wb+') as f: 75 | json.dump(obj, f, indent=4, sort_keys=1) 76 | 77 | def wnm_load(path, default=None): 78 | try: 79 | print '> loading data from %s.gz' % path 80 | with gzip.open(path + '.gz', 'rb') as f: 81 | return json.load(f) 82 | except: 83 | print '> error loading %s.gz' % path 84 | return default 85 | 86 | # opencagedata.com to resolve address to gps coords 87 | # OCD_KEY=4b8c2a0d3c624c0a13c276f9125b14aa 88 | api_warn = 0 89 | def geo_lookup(place): 90 | if not 'OCD_KEY' in os.environ: 91 | global api_warn 92 | if not api_warn: 93 | print '> warn: set OpenCageData API key (export OCD_KEY=...) for accurate geolocation' 94 | api_warn = 1 95 | raise Exception('') 96 | 97 | API_KEY = os.environ['OCD_KEY'] 98 | u = 'http://api.opencagedata.com/geocode/v1/json?q=%s&key=%s' % (urllib.quote_plus(place), API_KEY) 99 | a = urllib2.urlopen(u) 100 | r = json.loads(a.read()) 101 | 102 | # debug 103 | # print '> %s' % place 104 | # print '> %s' % u 105 | # print json.dumps(r, indent=4) 106 | 107 | lat = r['results'][0]['geometry']['lat'] 108 | lng = r['results'][0]['geometry']['lng'] 109 | return (lat, lng) 110 | -------------------------------------------------------------------------------- /wnm/wnm_world.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | import sys, os, json, csv, zipfile 4 | from wnm_utils import * 5 | 6 | url = 'http://download.geonames.org/export/dump/countryInfo.txt' 7 | 8 | path_raw = './data-raw/worldinfo.txt' 9 | path_out = './data/worldinfo.json' 10 | 11 | def download(): 12 | print '=' * 40 13 | print '> updating country database' 14 | print '=' * 40 15 | 16 | if not get_url(url, path_raw, chunk_size=2048): 17 | return -1 18 | 19 | return 0 20 | 21 | def rebuild(): 22 | data = {} 23 | 24 | with open(path_raw, 'rb') as f: 25 | text = f.read() 26 | 27 | for l in text.splitlines(): 28 | if l[0] == '#': 29 | hdr = l[1:] 30 | continue 31 | 32 | c = dict(zip(hdr.split('\t'), l.split('\t'))) 33 | data[c['ISO']] = c 34 | 35 | wnm_save(path_out, data) 36 | 37 | return 0 38 | 39 | 40 | # ================================= 41 | # main 42 | # ================================= 43 | def usage(): 44 | print '%s usage ' % sys.argv[0] 45 | print '' 46 | print 'commands:' 47 | print ' download -- download world info databases' 48 | print ' rebuild -- rebuild local json database' 49 | print ' update -- download & rebuild' 50 | print 51 | sys.exit(1) 52 | 53 | if __name__ == '__main__': 54 | 55 | if len(sys.argv) < 2: 56 | usage() 57 | sys.exit(-1) 58 | 59 | action = sys.argv[1] 60 | 61 | if action == 'download': 62 | download() 63 | 64 | elif action == 'update': 65 | if download(): 66 | sys.exit(-1) 67 | if rebuild(): 68 | sys.exit(-1) 69 | 70 | elif action == 'rebuild': 71 | if rebuild(): 72 | sys.exit(-1) 73 | 74 | else: 75 | print '[-] unknown command "%s"' % action 76 | sys.exit(-1) 77 | 78 | sys.exit(0) 79 | 80 | --------------------------------------------------------------------------------