├── static ├── img │ ├── dash.png │ ├── world.png │ ├── bitcoin.png │ ├── bitcoin-sv.png │ ├── litecoin.png │ ├── bitcoin-cash.png │ └── network.svg ├── flags │ ├── blank.gif │ ├── flags.png │ ├── iso_countries.json │ └── flags.min.css ├── css │ ├── images │ │ ├── ui-bg_glass_100_f6f6f6_1x400.png │ │ ├── ui-bg_glass_100_fdf5ce_1x400.png │ │ ├── ui-bg_glass_65_ffffff_1x400.png │ │ └── ui-bg_highlight-soft_100_eeeeee_1x100.png │ ├── dc.min.css │ ├── custom.css │ └── jquery-ui.min.css └── js │ └── crossfilter.min.js ├── requirements.txt ├── .dockerignore ├── geoip └── README.txt ├── .env.dist ├── Dockerfile ├── templates ├── footer.html ├── docs.html ├── about.html └── base.html ├── LICENSE ├── docker-compose.yml ├── README.md ├── config.py ├── crawler_config.yml ├── autodoc.py ├── models.py ├── server.py ├── crawler.py └── protocol.py /static/img/dash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blakebjorn/opennodes/master/static/img/dash.png -------------------------------------------------------------------------------- /static/img/world.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blakebjorn/opennodes/master/static/img/world.png -------------------------------------------------------------------------------- /static/flags/blank.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blakebjorn/opennodes/master/static/flags/blank.gif -------------------------------------------------------------------------------- /static/flags/flags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blakebjorn/opennodes/master/static/flags/flags.png -------------------------------------------------------------------------------- /static/img/bitcoin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blakebjorn/opennodes/master/static/img/bitcoin.png -------------------------------------------------------------------------------- /static/img/bitcoin-sv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blakebjorn/opennodes/master/static/img/bitcoin-sv.png -------------------------------------------------------------------------------- /static/img/litecoin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blakebjorn/opennodes/master/static/img/litecoin.png -------------------------------------------------------------------------------- /static/img/bitcoin-cash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blakebjorn/opennodes/master/static/img/bitcoin-cash.png -------------------------------------------------------------------------------- /static/css/images/ui-bg_glass_100_f6f6f6_1x400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blakebjorn/opennodes/master/static/css/images/ui-bg_glass_100_f6f6f6_1x400.png -------------------------------------------------------------------------------- /static/css/images/ui-bg_glass_100_fdf5ce_1x400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blakebjorn/opennodes/master/static/css/images/ui-bg_glass_100_fdf5ce_1x400.png -------------------------------------------------------------------------------- /static/css/images/ui-bg_glass_65_ffffff_1x400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blakebjorn/opennodes/master/static/css/images/ui-bg_glass_65_ffffff_1x400.png -------------------------------------------------------------------------------- /static/css/images/ui-bg_highlight-soft_100_eeeeee_1x100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blakebjorn/opennodes/master/static/css/images/ui-bg_highlight-soft_100_eeeeee_1x100.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | wheel 2 | geoip2 3 | ipaddress 4 | PySocks 5 | requests 6 | sqlalchemy 7 | flask_sqlalchemy 8 | numpy 9 | pandas 10 | pyyaml 11 | waitress 12 | psycopg2-binary 13 | python-dotenv -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | *.db 2 | venv 3 | __pycache__ 4 | .idea 5 | .ipynb_checkpoints 6 | *.pyc 7 | .git 8 | dash 9 | geoip 10 | db_cache 11 | postgres 12 | static 13 | **/temp/ 14 | **/logs/ 15 | **/geoip/ 16 | node_modules 17 | -------------------------------------------------------------------------------- /geoip/README.txt: -------------------------------------------------------------------------------- 1 | Download the ASN, City, and Country .mmdb files from the geolite page on the maxmind site and 2 | place them in this directory, ensuring the following names: 3 | 4 | ``` 5 | GeoLite2-ASN.mmdb 6 | GeoLite2-City.mmdb 7 | GeoLite2-Country.mmdb 8 | ``` -------------------------------------------------------------------------------- /.env.dist: -------------------------------------------------------------------------------- 1 | DB_TYPE=postgresql 2 | DB_USER=postgres 3 | DB_PASS=password 4 | DB_HOST=opennodes_postgres 5 | DB_PORT=5432 6 | DB_NAME=opennodes 7 | 8 | SERVER_HOST=0.0.0.0 9 | SERVER_PORT=8888 10 | 11 | DASH_RPC_URI=http://127.0.0.1:9998 12 | DASH_RPC_USER=myrpcuser 13 | DASH_RPC_PASS=myrpcpassword -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Base Image - can't use python:alpine-3.7 because MSSQL drivers don't support alpine linux 2 | FROM python:3.9-alpine 3 | ENV PYTHONUNBUFFERED=1 4 | 5 | RUN apk --no-cache add g++ postgresql-dev 6 | 7 | COPY requirements.txt requirements.txt 8 | RUN pip install --no-cache-dir -r requirements.txt 9 | 10 | COPY . . 11 | CMD ['python', 'daemon.py'] 12 | -------------------------------------------------------------------------------- /templates/footer.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Opennodes / blakebjorn 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /static/img/network.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 15 | 16 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.3' 2 | 3 | services: 4 | opennodes_crawler: 5 | restart: always 6 | network_mode: "host" 7 | depends_on: 8 | - opennodes_postgres 9 | - opennodes_dashd 10 | container_name: opennodes-crawler 11 | build: 12 | context: "" 13 | dockerfile: Dockerfile 14 | volumes: 15 | - ./static:/static 16 | - ./geoip:/geoip 17 | - ./db_cache:/db_cache 18 | entrypoint: 19 | - python 20 | - crawler.py 21 | - --daemon 22 | logging: 23 | driver: "json-file" 24 | options: 25 | max-size: "2m" 26 | max-file: "1" 27 | opennodes_postgres: 28 | restart: always 29 | container_name: opennodes-postgres 30 | image: postgres:alpine 31 | environment: 32 | POSTGRES_DB: opennodes 33 | POSTGRES_PASSWORD: "${DB_PASS}" 34 | volumes: 35 | - ./postgres/lib:/var/lib/postgresql 36 | - ./postgres/data:/var/lib/postgresql/data 37 | mem_limit: 2g 38 | memswap_limit: 3g 39 | ports: 40 | - "127.0.0.1:5432:5432" 41 | logging: 42 | driver: "json-file" 43 | options: 44 | max-size: "2m" 45 | max-file: "1" 46 | opennodes_dashd: 47 | restart: always 48 | container_name: opennodes-dashd 49 | build: https://github.com/dashpay/docker-dashd.git 50 | ports: 51 | - "9999:9999" 52 | - "127.0.0.1:9998:9998" 53 | volumes: 54 | - ./dash:/dash 55 | environment: 56 | DISABLEWALLET: 1 57 | PRINTTOCONSOLE: 1 58 | RPCUSER: "${DASH_RPC_USER}" 59 | RPCPASSWORD: "${DASH_RPC_PASS}" 60 | logging: 61 | driver: "json-file" 62 | options: 63 | max-size: "2m" 64 | max-file: "1" 65 | opennodes_site: 66 | restart: always 67 | depends_on: 68 | - opennodes_postgres 69 | - opennodes_dashd 70 | container_name: opennodes-site 71 | build: 72 | context: "" 73 | dockerfile: Dockerfile 74 | ports: 75 | - "8888:8888" 76 | volumes: 77 | - ./static:/static 78 | - ./geoip:/geoip 79 | entrypoint: 80 | - python 81 | - server.py 82 | - --prod 83 | environment: 84 | DB_HOST_OVERRIDE: "opennodes_postgres" 85 | logging: 86 | driver: "json-file" 87 | options: 88 | max-size: "2m" 89 | max-file: "1" 90 | -------------------------------------------------------------------------------- /templates/docs.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block content %} 3 | 4 | API Documentation 5 | 6 | 50 | 51 | 52 | 53 | 54 |

API Reference

55 | 56 | {% for doc in autodoc %} 57 |
58 |

{{ doc.rule|escape }}

 59 | 60 |

{{ doc.docstring|urlize|nl2br }}

61 | 62 | 73 | 74 | {% if doc.args %} 75 | 80 | {% endif %} 81 | 82 | {% if doc.returns %} 83 | Returns: {{ doc.returns[0][1] }} 84 | {% endif %} 85 |
86 | {% endfor %} 87 | 88 | 89 | {% endblock %} -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Open Nodes 2 | 3 | Open Nodes is a crawler that attempts to map out all nodes of crypto currencies based on the bitcoin protocol. 4 | 5 | A flask web server is included to display the data. 6 | 7 | ### Setup 8 | You will need to download the 3 geoip files (cities, countries, and ASN) from the maxmind site - 9 | this requires registration. 10 | Copy the .mmdb files into the `geoip` directory otherwise no country/ASN info will be populated 11 | 12 | copy `.env.dist` to `.env` and change any values as necessary. Delete all database entries to use sqlite 13 | 14 | ### Usage (Docker) 15 | The crawler runs in the `host` network mode to alleviate any issues connecting to external ipv6 addresses. You will need Tor installed on the host machine if you want to crawl onion nodes. 16 | ``` 17 | cp .env.dist .env 18 | docker-compose up 19 | ``` 20 | 21 | ### Usage (Manual Installation) 22 | ``` 23 | # Install python virtual environment 24 | apt install python3-pip python3-venv 25 | python3 -m venv venv 26 | source venv/bin/activate 27 | 28 | # Install packages (Python 3) 29 | # psycopg2-binary is required for postgres support 30 | pip3 install -r requirements.txt 31 | 32 | # run crawler. 33 | python3 crawler.py --seed --crawl --dump 34 | 35 | # run flask server 36 | python3 app.py 37 | ``` 38 | 39 | The `--seed` parameter is only needed for the first run or when adding a new network. It will hit all the DNS seeds specified in the config file, as well as all individual seeder nodes (if applicable) 40 | 41 | The `--crawl` parameter iterates through all known nodes and stores them in the specified database 42 | 43 | The `--dump` parameter writes all data to disk in json, csv, and txt format for ingestion by the webserver 44 | 45 | The `--prune` parameter removes old entries from the DB and writes them to gzipped CSVs on disk 46 | 47 | The `--daemon` parameter does all of the above in an endless loop based on configuration 48 | 49 | IPv6 Nodes will only be reachable if you have IPv6 Routing available. To set up IPv6 routing on an AWS deployment see [here](https://www.dogsbody.com/blog/setting-up-ipv6-on-your-ec2/) 50 | 51 | Onion Nodes will only be reachable if you have a Tor server running (`apt install tor`) 52 | 53 | ### Deployment 54 | The crawler can be set up as a service or via cron jobs, `--dump` instances should be scheduled separately 55 | from `--crawl` jobs, as they are slow to run as the DB size grows. `flock` can be used to prevent multiple 56 | instances from running concurrently. For production use, the default database (sqlite) should not be used, 57 | as the file lock timeout will prevent simultaneous crawling, dumping, and reporting/api calls. 58 | 59 | The server runs in debug mode by default, if you pass `--prod` as an argument to `server.py` it will instead 60 | run via waitress, and should be reverse proxied to via nginx/apache 61 | 62 | ##### Example service: 63 | ``` 64 | /etc/systemd/system/opennodes.service 65 | ``` 66 | 67 | ``` 68 | [Unit] 69 | Description=OpenNodes Server 70 | After=network.target 71 | 72 | [Service] 73 | User= {{ USERNAME }} 74 | Group=www-data 75 | WorkingDirectory=/home/{{ PROJECT ROOT }} 76 | Environment="PATH=/home/{{ PATH TO PYTHON/VENV BIN DIR }}" 77 | ExecStart=/home/{{ PATH TO PYTHON/VENV BIN DIR }}/python3 server.py --prod 78 | 79 | [Install] 80 | WantedBy=multi-user.target 81 | ``` 82 | ##### Example nginx config: 83 | ``` 84 | /etc/nginx/sites-enabled/opennodes 85 | ``` 86 | ``` 87 | server { 88 | listen 80; 89 | server_name {{ SERVER DOMAIN OR IP }}; 90 | 91 | location / { 92 | # Set proxy headers 93 | proxy_set_header Host $host; 94 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 95 | proxy_set_header X-Forwarded-Proto $scheme; 96 | 97 | proxy_pass http://localhost:8888; 98 | } 99 | } -------------------------------------------------------------------------------- /static/css/dc.min.css: -------------------------------------------------------------------------------- 1 | .dc-chart path.dc-symbol,.dc-legend g.dc-legend-item.fadeout{fill-opacity:.5;stroke-opacity:.5}div.dc-chart{float:left}.dc-chart rect.bar{stroke:none;cursor:pointer}.dc-chart rect.bar:hover{fill-opacity:.5}.dc-chart rect.deselected{stroke:none;fill:#ccc}.dc-chart .pie-slice{fill:#fff;font-size:12px;cursor:pointer}.dc-chart .pie-slice.external{fill:#000}.dc-chart .pie-slice :hover,.dc-chart .pie-slice.highlight{fill-opacity:.8}.dc-chart .pie-path{fill:none;stroke-width:2px;stroke:#000;opacity:.4}.dc-chart .selected circle,.dc-chart .selected path{stroke-width:3;stroke:#ccc;fill-opacity:1}.dc-chart .deselected circle,.dc-chart .deselected path{stroke:none;fill-opacity:.5;fill:#ccc}.dc-chart .axis line,.dc-chart .axis path{fill:none;stroke:#000;shape-rendering:crispEdges}.dc-chart .axis text{font:10px sans-serif}.dc-chart .axis .grid-line,.dc-chart .axis .grid-line line,.dc-chart .grid-line,.dc-chart .grid-line line{fill:none;stroke:#ccc;opacity:.5;shape-rendering:crispEdges}.dc-chart .brush rect.selection{fill:#4682b4;fill-opacity:.125}.dc-chart .brush .custom-brush-handle{fill:#eee;stroke:#666;cursor:ew-resize}.dc-chart path.line{fill:none;stroke-width:1.5px}.dc-chart path.area{fill-opacity:.3;stroke:none}.dc-chart path.highlight{stroke-width:3;fill-opacity:1;stroke-opacity:1}.dc-chart g.state{cursor:pointer}.dc-chart g.state :hover{fill-opacity:.8}.dc-chart g.state path{stroke:#fff}.dc-chart g.deselected path{fill:grey}.dc-chart g.deselected text{display:none}.dc-chart g.row rect{fill-opacity:.8;cursor:pointer}.dc-chart g.row rect:hover{fill-opacity:.6}.dc-chart g.row text{fill:#fff;font-size:12px;cursor:pointer}.dc-chart g.dc-tooltip path{fill:none;stroke:grey;stroke-opacity:.8}.dc-chart g.county path{stroke:#fff;fill:none}.dc-chart g.debug rect{fill:#00f;fill-opacity:.2}.dc-chart g.axis text{-webkit-touch-callout:none;-webkit-user-select:none;-khtml-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;pointer-events:none}.dc-chart .node{font-size:.7em;cursor:pointer}.dc-chart .node :hover{fill-opacity:.8}.dc-chart .bubble{stroke:none;fill-opacity:.6}.dc-chart .highlight{fill-opacity:1;stroke-opacity:1}.dc-chart .fadeout{fill-opacity:.2;stroke-opacity:.2}.dc-chart .box text{font:10px sans-serif;-webkit-touch-callout:none;-webkit-user-select:none;-khtml-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;pointer-events:none}.dc-chart .box line{fill:#fff}.dc-chart .box circle,.dc-chart .box line,.dc-chart .box rect{stroke:#000;stroke-width:1.5px}.dc-chart .box .center{stroke-dasharray:3,3}.dc-chart .box .data{stroke:none;stroke-width:0}.dc-chart .box .outlier{fill:none;stroke:#ccc}.dc-chart .box .outlierBold{fill:red;stroke:none}.dc-chart .box.deselected{opacity:.5}.dc-chart .box.deselected .box{fill:#ccc}.dc-chart .symbol{stroke:none}.dc-chart .heatmap .box-group.deselected rect{stroke:none;fill-opacity:.5;fill:#ccc}.dc-chart .heatmap g.axis text{pointer-events:all;cursor:pointer}.dc-chart .empty-chart .pie-slice{cursor:default}.dc-chart .empty-chart .pie-slice path{fill:#fee;cursor:default}.dc-data-count{float:right;margin-top:15px;margin-right:15px}.dc-data-count .filter-count,.dc-data-count .total-count{color:#3182bd;font-weight:700}.dc-legend{font-size:11px}.dc-legend .dc-legend-item{cursor:pointer}.dc-hard .number-display{float:none}div.dc-html-legend{overflow-y:auto;overflow-x:hidden;height:inherit;float:right;padding-right:2px}div.dc-html-legend .dc-legend-item-horizontal{display:inline-block;margin-left:5px;margin-right:5px;cursor:pointer}div.dc-html-legend .dc-legend-item-horizontal.selected{background-color:#3182bd;color:#fff}div.dc-html-legend .dc-legend-item-vertical{display:block;margin-top:5px;padding-top:1px;padding-bottom:1px;cursor:pointer}div.dc-html-legend .dc-legend-item-vertical.selected{background-color:#3182bd;color:#fff}div.dc-html-legend .dc-legend-item-color{display:table-cell;width:12px;height:12px}div.dc-html-legend .dc-legend-item-label{line-height:12px;display:table-cell;vertical-align:middle;padding-left:3px;padding-right:3px;font-size:.75em}.dc-html-legend-container{height:inherit} -------------------------------------------------------------------------------- /templates/about.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block content %} 3 |
4 |
5 |
6 |

Open Nodes

7 |

Purpose

8 |

9 | Open Nodes aims to map out and compare the status of several Satoshi-based coin networks. Despite 10 | the fact that these coins share the same protocol, there currently exists no easy way to directly 11 | compare these coins in a single location. Additionally, no data is currently available for Dash 12 | full nodes, only masternodes. These statistics are essential to evaluate the relative health of each 13 | network. 14 |

15 | 16 |

Overview

17 |

18 | Open Nodes maps out the network by recursively connecting to nodes and issuing getaddr 19 | commands. Any connected nodes that have been online in the past 24 hours are then added to the 20 | crawl. Only nodes which are configured to allow remote connections and accept the handshake from the 21 | crawler are included in the presented dataset. Nodes with a version number below the minimum are 22 | also excluded. All nodes are designated as "Active" or "Inactive" based on the last reported block 23 | height. If the node's block height is more than an hour out of consensus with the network median 24 | then it is flagged as inactive. 25 |

26 | 27 |
The following constants are used for the handshake:
28 |
    29 |
  • Bitcoin: 30 |
      31 |
    • Magic number: f9beb4d9
      • 32 |
    • Protocol version: 70015
      • 33 |
    • Minimum version: 70001
      • 34 |
    35 |
    • 36 |
  • Bitcoin Cash + SV: 37 |
      38 |
    • Magic number: e3e1f3e8
      • 39 |
    • Protocol version: 70015
      • 40 |
    • Minimum version: 70001
      • 41 |
    42 |
    • 43 |
  • Dash: 44 |
      45 |
    • Magic number: bf0c6bbd
      • 46 |
    • Protocol version: 70209
      • 47 |
    • Minimum version: 70001
      • 48 |
    49 |
    • 50 |
  • Litecoin: 51 |
      52 |
    • Magic number: fbc0b6db
      • 53 |
    • Protocol version: 70015
      • 54 |
    • Minimum version: 70001
      • 55 |
    56 |
    • 57 |
58 | 59 | Bitcoin SV is extracted from the Bitcoin Cash crawl set based on the user agent.

60 | 61 | Some user agents are excluded from the result set based on the following regular expressions: 62 |
    63 |
  • /Monoeci.*?
    • 64 |
  • /Binarium.*?
    • 65 |
  • /Qyno.*?
    • 66 |
  • /Monaco.*?
    • 67 |
  • /DigitalCoin.*?
    • 68 |
  • /FlashCoin.*?
    • 69 |
  • /WaterTechnology.*?
    • 70 |
  • /WorldCoin.*?
    • 71 |
  • /FeatherCoin.*?
    • 72 |
  • /CryptoCoin.*?
    • 73 |
  • /Sollida.*?
    • 74 |
75 |
76 |
77 | 78 |
79 | 80 | {% include 'footer.html' %} 81 | {% endblock %} -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Open Nodes configuration 3 | Copyright (c) 2018 Opennodes / Blake Bjorn Anderson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | """ 24 | 25 | import logging 26 | import os 27 | import re 28 | import requests 29 | import yaml 30 | import binascii 31 | import dotenv 32 | from ipaddress import ip_network 33 | 34 | dotenv.load_dotenv() 35 | 36 | if os.environ.get("DB_HOST"): 37 | DATABASE_URI = f"{os.environ['DB_TYPE']}://{os.environ['DB_USER']}:{os.environ['DB_PASS']}@" \ 38 | f"{os.environ.get('DB_HOST_OVERRIDE', os.environ['DB_HOST'])}:{os.environ['DB_PORT']}/" \ 39 | f"{os.environ['DB_NAME']}" 40 | else: 41 | DATABASE_URI = os.environ.get("DATABASE_URI", "sqlite:///nodes.sqlite") 42 | 43 | 44 | def list_excluded_networks(networks): 45 | """ 46 | Converts list of networks from configuration file into a list of tuples of 47 | network address and netmask to be excluded from the crawl. 48 | """ 49 | networks_out = set() 50 | for addr in networks: 51 | addr = str(addr).split('#')[0].strip() 52 | try: 53 | network = ip_network(addr) 54 | except ValueError: 55 | continue 56 | else: 57 | networks_out.add((int(network.network_address), int(network.netmask))) 58 | return networks_out 59 | 60 | 61 | def get_ipv4_bogons(): 62 | url = "http://www.team-cymru.org/Services/Bogons/fullbogons-ipv4.txt" 63 | try: 64 | response = requests.get(url, timeout=15) 65 | except requests.exceptions.RequestException as err: 66 | logging.warning(err) 67 | else: 68 | if response.status_code == 200: 69 | return response.content.decode("utf8").splitlines() 70 | return [] 71 | 72 | 73 | def load_config(): 74 | with open("crawler_config.yml", "r") as f: 75 | conf = yaml.load(f, yaml.SafeLoader) 76 | 77 | if os.path.isfile("crawler_user_config.yml"): 78 | with open("crawler_user_config.yml", "r") as f: 79 | conf.update(yaml.load(f, yaml.SafeLoader)) 80 | 81 | if 'networks' in conf: 82 | for network in conf['networks']: 83 | conf['networks'][network]['magic_number'] = binascii.unhexlify( 84 | conf['networks'][network]['magic_number']) 85 | 86 | if 'exclude_ipv4_bogons' in conf and conf['exclude_ipv4_bogons']: 87 | conf['exclude_ipv4_networks'] = list(set(conf['exclude_ipv4_networks'] + get_ipv4_bogons())) 88 | 89 | if 'exclude_ipv4_networks' in conf: 90 | conf['exclude_ipv4_networks'] = list_excluded_networks(conf['exclude_ipv4_networks']) 91 | 92 | if 'excluded_user_agents' in conf: 93 | conf['excluded_user_agents'] = [re.compile(x, re.IGNORECASE) for x in conf['excluded_user_agents']] 94 | 95 | if 'user_agent' in conf: 96 | conf['user_agent'] = conf['user_agent'].encode() 97 | 98 | if 'tor_proxy' in conf and conf['tor_proxy']: 99 | assert ":" in conf['tor_proxy'] 100 | port = int(conf['tor_proxy'].split(":")[-1]) 101 | address = conf['tor_proxy'].split(f":{port}")[0] 102 | conf['tor_proxy'] = (address, port) 103 | 104 | return conf 105 | -------------------------------------------------------------------------------- /static/css/custom.css: -------------------------------------------------------------------------------- 1 | .dc-pie-cont { 2 | min-width: 200px; 3 | max-width: 350px; 4 | } 5 | 6 | .dc-pie-cont-legend { 7 | min-width: 200px; 8 | max-width: 350px; 9 | } 10 | 11 | .dc-pie { 12 | min-width: 190px; 13 | max-width: 290px; 14 | } 15 | 16 | .dc-row-cont { 17 | min-width: 270px; 18 | max-height: 240px; 19 | max-width: 490px; 20 | } 21 | 22 | .dc-row { 23 | min-width: 260px; 24 | max-height: 240px; 25 | max-width: 480px; 26 | } 27 | 28 | .dc-row g.row text { 29 | fill: #fff; 30 | font-size: 9px; 31 | cursor: pointer; 32 | } 33 | 34 | .dc-row g.tick text { 35 | text-anchor: start !important; 36 | transform: rotate(45deg); 37 | } 38 | 39 | path { 40 | stroke-opacity: 0.3; 41 | stroke: black; 42 | stroke-width: 1px; 43 | } 44 | 45 | .dc-table td { 46 | padding-top: 4px; 47 | padding-bottom: 3px; 48 | font-size: 10pt; 49 | } 50 | 51 | .dc-table th { 52 | padding-top: 5px; 53 | padding-bottom: 4px; 54 | text-align: center; 55 | cursor: pointer; 56 | } 57 | 58 | .table-scroll { 59 | overflow-y: auto; 60 | max-height:300px; 61 | } 62 | 63 | .table-scroll-long { 64 | overflow-x: visible; 65 | overflow-y: auto; 66 | max-height:500px; 67 | min-width:260px; 68 | max-width: 270px; 69 | } 70 | 71 | .dc-table-small td { 72 | padding-top: 2px; 73 | padding-bottom: 1px; 74 | padding-right: 1px; 75 | padding-left: 1px; 76 | font-size: 9pt; 77 | } 78 | 79 | .dc-table-small th { 80 | font-size: 12pt; 81 | padding-top: 2px; 82 | padding-bottom: 1px; 83 | text-align: center; 84 | cursor: pointer; 85 | } 86 | 87 | .jq-slider .ui-slider-range { 88 | background: #6495ED; 89 | } 90 | 91 | .dc-nd { 92 | font-size: 28px; 93 | } 94 | 95 | html, body { 96 | max-width: 100%; 97 | overflow-x: hidden; 98 | } 99 | 100 | .dc-chart g.row text { 101 | fill: black; 102 | } 103 | 104 | .color-box { 105 | width: 10px; 106 | height: 10px; 107 | display: inline-block; 108 | background-color: #ccc; 109 | left: 5px; 110 | top: 5px; 111 | } 112 | 113 | #mouseover-tooltip { 114 | visibility: hidden; 115 | position: absolute; 116 | z-index: 99; 117 | background-color: #FFFFFF; 118 | border: 1px solid black; 119 | padding: 3px 3px 1px 3px; 120 | text-align: center; 121 | opacity: 0.9; 122 | font-weight: 500; 123 | } 124 | 125 | .loading_spinner { 126 | border: 16px solid #f3f3f3; /* Light grey */ 127 | border-top: 16px solid deepskyblue; /* Blue */ 128 | border-radius: 50%; 129 | width: 120px; 130 | height: 120px; 131 | animation: spin 2s linear infinite; 132 | } 133 | 134 | .spinner_small { 135 | width: 60px; 136 | height: 60px; 137 | } 138 | 139 | .card { 140 | max-width: 90%; 141 | } 142 | 143 | @keyframes spin { 144 | 0% { transform: rotate(0deg); } 145 | 100% { transform: rotate(360deg); } 146 | } 147 | 148 | .i { 149 | border: solid black; 150 | border-width: 0 3px 3px 0; 151 | display: inline-block; 152 | padding: 3px; 153 | } 154 | 155 | .right { 156 | transform: rotate(-45deg); 157 | -webkit-transform: rotate(-45deg); 158 | } 159 | 160 | .left { 161 | transform: rotate(135deg); 162 | -webkit-transform: rotate(135deg); 163 | } 164 | 165 | .arrow-btn { 166 | border: none; 167 | background: none; 168 | } 169 | 170 | .line { 171 | fill: none; 172 | stroke: #6495ED; 173 | stroke-opacity: 0.8; 174 | stroke-width: 2; 175 | } 176 | 177 | #map-chart-container { 178 | max-width: 680px; 179 | /*padding-left: -40px;*/ 180 | /*margin-right: 100px;*/ 181 | } 182 | 183 | .svg-container { 184 | display: inline-block; 185 | position: relative; 186 | width: 100%; 187 | padding-bottom: 60%; /* aspect ratio */ 188 | vertical-align: top; 189 | overflow: hidden; 190 | max-height: 600px; 191 | } 192 | 193 | .svg-content-responsive { 194 | display: inline-block; 195 | position: absolute; 196 | top: 10px; 197 | left: 0; 198 | 199 | } 200 | 201 | @media only screen and (max-width: 600px) { 202 | #summary-table td{ 203 | font-size:8pt; 204 | } 205 | } 206 | 207 | -------------------------------------------------------------------------------- /templates/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | Open Nodes 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 |
31 |
32 |
33 | 34 |
35 |
36 | 64 |
65 |
66 |
67 |
68 |
69 | 70 |
71 | 72 | {% with messages = get_flashed_messages() %} 73 | {% if messages %} 74 | {% for message in messages %} 75 |
76 | {{ message }} 77 |
78 | {% endfor %} 79 | {% endif %} 80 | {% endwith %} 81 | 82 | {% block content %} 83 | {% endblock %} 84 | 85 | 86 | -------------------------------------------------------------------------------- /static/flags/iso_countries.json: -------------------------------------------------------------------------------- 1 | { 2 | "Afghanistan": "AF", 3 | "\u00c5land Islands": "AX", 4 | "Albania": "AL", 5 | "Algeria": "DZ", 6 | "American Samoa": "AS", 7 | "Andorra": "AD", 8 | "Angola": "AO", 9 | "Anguilla": "AI", 10 | "Antarctica": "AQ", 11 | "Antigua and Barbuda": "AG", 12 | "Argentina": "AR", 13 | "Armenia": "AM", 14 | "Aruba": "AW", 15 | "Australia": "AU", 16 | "Austria": "AT", 17 | "Azerbaijan": "AZ", 18 | "Bahamas": "BS", 19 | "Bahrain": "BH", 20 | "Bangladesh": "BD", 21 | "Barbados": "BB", 22 | "Belarus": "BY", 23 | "Belgium": "BE", 24 | "Belize": "BZ", 25 | "Benin": "BJ", 26 | "Bermuda": "BM", 27 | "Bhutan": "BT", 28 | "Bolivia": "BO", 29 | "Bonaire, Sint Eustatius and Saba": "BQ", 30 | "Bosnia and Herzegovina": "BA", 31 | "Botswana": "BW", 32 | "Bouvet Island": "BV", 33 | "Brazil": "BR", 34 | "British Indian Ocean Territory": "IO", 35 | "Brunei Darussalam": "BN", 36 | "Bulgaria": "BG", 37 | "Burkina Faso": "BF", 38 | "Burundi": "BI", 39 | "Cambodia": "KH", 40 | "Cameroon": "CM", 41 | "Canada": "CA", 42 | "Cape Verde": "CV", 43 | "Cayman Islands": "KY", 44 | "Central African Republic": "CF", 45 | "Chad": "TD", 46 | "Chile": "CL", 47 | "China": "CN", 48 | "Christmas Island": "CX", 49 | "Cocos (Keeling) Islands": "CC", 50 | "Colombia": "CO", 51 | "Comoros": "KM", 52 | "Congo": "CG", 53 | "Congo, the Democratic Republic of the": "CD", 54 | "Cook Islands": "CK", 55 | "Costa Rica": "CR", 56 | "C\u00f4te d'Ivoire": "CI", 57 | "Croatia": "HR", 58 | "Cuba": "CU", 59 | "Cura\u00e7ao": "CW", 60 | "Cyprus": "CY", 61 | "Czechia": "CZ", 62 | "Denmark": "DK", 63 | "Djibouti": "DJ", 64 | "Dominica": "DM", 65 | "Dominican Republic": "DO", 66 | "Ecuador": "EC", 67 | "Egypt": "EG", 68 | "El Salvador": "SV", 69 | "Equatorial Guinea": "GQ", 70 | "Eritrea": "ER", 71 | "Estonia": "EE", 72 | "Ethiopia": "ET", 73 | "Falkland Islands (Malvinas)": "FK", 74 | "Faroe Islands": "FO", 75 | "Fiji": "FJ", 76 | "Finland": "FI", 77 | "France": "FR", 78 | "French Guiana": "GF", 79 | "French Polynesia": "PF", 80 | "French Southern Territories": "TF", 81 | "Gabon": "GA", 82 | "Gambia": "GM", 83 | "Georgia": "GE", 84 | "Germany": "DE", 85 | "Ghana": "GH", 86 | "Gibraltar": "GI", 87 | "Greece": "GR", 88 | "Greenland": "GL", 89 | "Grenada": "GD", 90 | "Guadeloupe": "GP", 91 | "Guam": "GU", 92 | "Guatemala": "GT", 93 | "Guernsey": "GG", 94 | "Guinea": "GN", 95 | "Guinea-Bissau": "GW", 96 | "Guyana": "GY", 97 | "Haiti": "HT", 98 | "Heard Island and McDonald Islands": "HM", 99 | "Holy See (Vatican City State)": "VA", 100 | "Honduras": "HN", 101 | "Hong Kong": "HK", 102 | "Hungary": "HU", 103 | "Iceland": "IS", 104 | "India": "IN", 105 | "Indonesia": "ID", 106 | "Iran": "IR", 107 | "Iraq": "IQ", 108 | "Ireland": "IE", 109 | "Isle of Man": "IM", 110 | "Israel": "IL", 111 | "Italy": "IT", 112 | "Jamaica": "JM", 113 | "Japan": "JP", 114 | "Jersey": "JE", 115 | "Jordan": "JO", 116 | "Kazakhstan": "KZ", 117 | "Kenya": "KE", 118 | "Kiribati": "KI", 119 | "Democratic People's Republic of Korea": "KP", 120 | "Republic of Korea": "KR", 121 | "Kuwait": "KW", 122 | "Kyrgyzstan": "KG", 123 | "Lao People's Democratic Republic": "LA", 124 | "Latvia": "LV", 125 | "Lebanon": "LB", 126 | "Lesotho": "LS", 127 | "Liberia": "LR", 128 | "Libya": "LY", 129 | "Liechtenstein": "LI", 130 | "Republic of Lithuania": "LT", 131 | "Luxembourg": "LU", 132 | "Macao": "MO", 133 | "Macedonia": "MK", 134 | "Madagascar": "MG", 135 | "Malawi": "MW", 136 | "Malaysia": "MY", 137 | "Maldives": "MV", 138 | "Mali": "ML", 139 | "Malta": "MT", 140 | "Marshall Islands": "MH", 141 | "Martinique": "MQ", 142 | "Mauritania": "MR", 143 | "Mauritius": "MU", 144 | "Mayotte": "YT", 145 | "Mexico": "MX", 146 | "Micronesia, Federated States of": "FM", 147 | "Republic of Moldova": "MD", 148 | "Monaco": "MC", 149 | "Mongolia": "MN", 150 | "Montenegro": "ME", 151 | "Montserrat": "MS", 152 | "Morocco": "MA", 153 | "Mozambique": "MZ", 154 | "Myanmar": "MM", 155 | "Namibia": "NA", 156 | "Nauru": "NR", 157 | "Nepal": "NP", 158 | "Netherlands": "NL", 159 | "New Caledonia": "NC", 160 | "New Zealand": "NZ", 161 | "Nicaragua": "NI", 162 | "Niger": "NE", 163 | "Nigeria": "NG", 164 | "Niue": "NU", 165 | "Norfolk Island": "NF", 166 | "Northern Mariana Islands": "MP", 167 | "Norway": "NO", 168 | "Oman": "OM", 169 | "Pakistan": "PK", 170 | "Palau": "PW", 171 | "Palestine, State of": "PS", 172 | "Panama": "PA", 173 | "Papua New Guinea": "PG", 174 | "Paraguay": "PY", 175 | "Peru": "PE", 176 | "Philippines": "PH", 177 | "Pitcairn": "PN", 178 | "Poland": "PL", 179 | "Portugal": "PT", 180 | "Puerto Rico": "PR", 181 | "Qatar": "QA", 182 | "Réunion": "RE", 183 | "Romania": "RO", 184 | "Russia": "RU", 185 | "Rwanda": "RW", 186 | "Saint Barth\u00e9lemy": "BL", 187 | "Saint Helena, Ascension and Tristan da Cunha": "SH", 188 | "Saint Kitts and Nevis": "KN", 189 | "Saint Lucia": "LC", 190 | "Saint Martin (French part)": "MF", 191 | "Saint Pierre and Miquelon": "PM", 192 | "Saint Vincent and the Grenadines": "VC", 193 | "Samoa": "WS", 194 | "San Marino": "SM", 195 | "Sao Tome and Principe": "ST", 196 | "Saudi Arabia": "SA", 197 | "Senegal": "SN", 198 | "Serbia": "RS", 199 | "Seychelles": "SC", 200 | "Sierra Leone": "SL", 201 | "Singapore": "SG", 202 | "Sint Maarten (Dutch part)": "SX", 203 | "Slovakia": "SK", 204 | "Slovenia": "SI", 205 | "Solomon Islands": "SB", 206 | "Somalia": "SO", 207 | "South Africa": "ZA", 208 | "South Georgia and the South Sandwich Islands": "GS", 209 | "South Sudan": "SS", 210 | "Spain": "ES", 211 | "Sri Lanka": "LK", 212 | "Sudan": "SD", 213 | "Suriname": "SR", 214 | "Svalbard and Jan Mayen": "SJ", 215 | "Swaziland": "SZ", 216 | "Sweden": "SE", 217 | "Switzerland": "CH", 218 | "Syrian Arab Republic": "SY", 219 | "Taiwan": "TW", 220 | "Tajikistan": "TJ", 221 | "Tanzania, United Republic of": "TZ", 222 | "Thailand": "TH", 223 | "Timor-Leste": "TL", 224 | "Togo": "TG", 225 | "Tokelau": "TK", 226 | "Tonga": "TO", 227 | "Trinidad and Tobago": "TT", 228 | "Tunisia": "TN", 229 | "Turkey": "TR", 230 | "Turkmenistan": "TM", 231 | "Turks and Caicos Islands": "TC", 232 | "Tuvalu": "TV", 233 | "Uganda": "UG", 234 | "Ukraine": "UA", 235 | "United Arab Emirates": "AE", 236 | "United Kingdom": "GB", 237 | "United States": "US", 238 | "United States Minor Outlying Islands": "UM", 239 | "Uruguay": "UY", 240 | "Uzbekistan": "UZ", 241 | "Vanuatu": "VU", 242 | "Venezuela, Bolivarian Republic of": "VE", 243 | "Vietnam": "VN", 244 | "British Virgin Islands": "VG", 245 | "Virgin Islands, U.S.": "VI", 246 | "Wallis and Futuna": "WF", 247 | "Western Sahara": "EH", 248 | "Yemen": "YE", 249 | "Zambia": "ZM", 250 | "Zimbabwe": "ZW" 251 | } -------------------------------------------------------------------------------- /crawler_config.yml: -------------------------------------------------------------------------------- 1 | ### Anything added to crawler_user_config.yml will overwrite these settings 2 | daemon: 3 | crawl_interval: 15 4 | dump_interval: 60 5 | prune_interval: 10080 6 | 7 | # Max concurrent threadpool workers - setting this too high may cause connections to be dropped 8 | threads: 1000 9 | 10 | # Max tasks in queue - higher will minimize database IO at cost of increased memory usage 11 | max_queue: 20000 12 | 13 | # Minimum time between node visitations (minutes) 14 | crawl_interval: 15 15 | 16 | # Proportion of connections which are followed up with a getaddr call. (0.0-1.0) 17 | # After initial seeding it isn't necessary for this to be very high to discover all new nodes 18 | # When --seed parameter is passed this is ignored 19 | getaddr_prop: 0.15 20 | 21 | # How much the block height has to differ from the median to be marked inactive 22 | # Dash and Litecoin have faster block times so this is set slightly higher 23 | inactive_threshold: 24 | default: 12 25 | dash: 48 26 | litecoin: 48 27 | 28 | # Attempt to establish connection with IPv6 nodes 29 | ipv6: True 30 | 31 | # Attempt to establish connection with .onion nodes 32 | onion: True 33 | 34 | # Tor proxy is required to connect to .onion address 35 | tor_proxy: 127.0.0.1:9050 36 | 37 | # order with which to process the networks - bch should be processed first, as all bch nodes need to be mapped so 38 | # they are not pinged while crawling another network. Connections to BCH nodes with the wrong network-magic will 39 | # trigger a 24 hour IP ban. 40 | crawl_order: 41 | - bitcoin-cash 42 | - dash 43 | - litecoin 44 | - bitcoin 45 | 46 | # Locks database table and updates Last_Checked BEFORE crawling nodes 47 | # Performance overhead, but necessary if multiple processes are crawling concurrently using the same database 48 | database_concurrency: False 49 | 50 | # Socket timeout (seconds) 51 | socket_timeout: 20 52 | 53 | # Retries 54 | retries: 1 55 | 56 | # Retry only nodes seen in past X hours 57 | retry_threshold: 12 58 | 59 | # Used during connection handshake 60 | user_agent: /open-nodes:0.1/ 61 | source_address: 0.0.0.0 62 | 63 | # How many days to continue checking a node that has not yet been seen 64 | min_pruning_age: 2 65 | 66 | # How many days to continue checking a node that has previously been seen 67 | max_pruning_age: 0 68 | 69 | # If a seen node has been pruned from the database due to max_pruning_age, remove its visitation history as well 70 | prune_visitations: False 71 | 72 | # Active nodes are defined as nodes within +/- [inactive_threshold] of the median block height. 73 | # If false inactive nodes will not be reported in the dumped files 74 | export_inactive_nodes: True 75 | 76 | # Excluded user agents - regex (see https://docs.python.org/3/library/re.html) 77 | excluded_user_agents: 78 | - /monoeci* 79 | - /binarium* 80 | - /qyno* 81 | - /monaco* 82 | - /digitalcoin* 83 | - /flashcoin* 84 | - /watertechnology* 85 | - /worldcoin* 86 | - /feathercoin* 87 | - /cryptocoin* 88 | - /sollida* 89 | - /lkscoin* 90 | - /futurebit* 91 | - /desire* 92 | 93 | # Lookup whether a node is a Dash masternode or not 94 | get_dash_masternodes: True 95 | 96 | # dash-masternodes-api-endpoint - used as a fallback if local node is not running 97 | # GET/POST request should return a json array of all active dash masternodes 98 | # e.g. https://running-opennodes-webserver.com/api/get_dash_masternodes 99 | dash_masternodes_api: '' 100 | 101 | # dash-cli executable directory 102 | dash_cli_path: /mnt/HC_Volume_2167606/dashcore-0.13.2/bin/ 103 | 104 | # historical data reporting interval (hours) 105 | historic_interval: 4 106 | 107 | # Protocol info for crawled networks 108 | networks: 109 | bitcoin: 110 | services: 0 111 | magic_number: f9beb4d9 112 | port: 8333 113 | protocol_version: 70015 114 | min_protocol_version: 70001 115 | dns_seeds: 116 | - dnsseed.bitcoin.dashjr.org 117 | - dnsseed.bluematt.me 118 | - seed.bitcoin.jonasschnelli.ch 119 | - seed.bitcoin.sipa.be 120 | - seed.bitcoinstats.com 121 | - seed.btc.petertodd.org 122 | - seed.bitnodes.io 123 | - seed.bitcoinabc.org 124 | - seed-abc.bitcoinforks.org 125 | - seed.bitcoinunlimited.info 126 | - btccash-seeder.bitcoinunlimited.info 127 | - seed.bitprim.org 128 | - seed.deadalnix.me 129 | - seeder.criptolayer.net 130 | address_seeds: 131 | - wioaxcgen3qvoqbf.onion 132 | - hpquscklwzoiw7qv.onion 133 | - 2zfhqfr4buzvrl5y.onion 134 | - wgjotdhmpcj2kyq2.onion 135 | - l4xfmcziytzeehcz.onion 136 | - 36xxwca2o6gz2h2b.onion 137 | - 7ejgnxi5z4w6tcgc.onion 138 | - rlxhnlonhjiyzhjb.onion 139 | - 62au2pzqah7cnrrw.onion 140 | - 5difox54jm5hplrw.onion 141 | - jwplkr5q3fivotmz.onion 142 | - ndndword5lpb7eex.onion 143 | dash: 144 | services: 0 145 | magic_number: bf0c6bbd 146 | port: 9999 147 | protocol_version: 70215 148 | min_protocol_version: 70001 149 | dns_seeds: 150 | - dnsseed.darkcoin.io 151 | - dnsseed.dashdot.io 152 | - dnsseed.masternode.io 153 | - dnsseed.dashpay.io 154 | - dnsseed.dash.org 155 | address_seeds: 156 | - 128.199.62.168 157 | - 31.220.7.131 158 | - 89.38.144.71 159 | - 188.166.9.179 160 | - 45.32.114.160 161 | - 37.59.247.129 162 | - 149.56.66.236 163 | - 159.65.84.39 164 | - 185.158.152.60 165 | - 45.32.20.140 166 | - 52.79.197.66 167 | - 51.15.88.43 168 | litecoin: 169 | services: 0 170 | magic_number: fbc0b6db 171 | port: 9333 172 | protocol_version: 70015 173 | min_protocol_version: 70001 174 | dns_seeds: 175 | - dnsseed.litecointools.com 176 | - dnsseed.litecoinpool.org 177 | - dnsseed.ltc.xurious.com 178 | - dnsseed.koin-project.com 179 | - seed-a.litecoin.loshan.co.uk 180 | - dnsseed.thrasher.io 181 | address_seeds: 182 | - phrj27hskw3gq4b5.onion 183 | - 2oqq4rydtti3xq4n.onion 184 | - 4xjze2q2ztn6l4ce.onion 185 | - 5zkvemmfa7ylr2qs.onion 186 | - mqgmba4o453jwdjd.onion 187 | - czahaqs6fhwr3jmw.onion 188 | bitcoin-cash: 189 | services: 0 190 | magic_number: e3e1f3e8 191 | port: 8333 192 | protocol_version: 70015 193 | min_protocol_version: 70001 194 | dns_seeds: 195 | - seed.bitcoinabc.org 196 | - seed-abc.bitcoinforks.org 197 | - seed.bitcoinunlimited.info 198 | - btccash-seeder.bitcoinunlimited.info 199 | - seed.bitprim.org 200 | - seed.deadalnix.me 201 | - seeder.criptolayer.net 202 | address_seeds: 203 | - li2mrdnveswxiwpe.onion 204 | - bchponzidlqjpsqp.onion 205 | - kister7332my5jka.onion 206 | - nld6rvbglzbbf7av.onion 207 | - wxjlz4avds42d42o.onion 208 | - wxh5kn2zjkcptpvo.onion 209 | - zndv4khma6ikx7o3.onion 210 | - kisternetg2pq7wx.onion 211 | 212 | exclude_ipv4_networks: 213 | - 0.0.0.0/8 214 | - 10.0.0.0/8 215 | - 100.64.0.0/10 216 | - 127.0.0.0/8 217 | - 169.254.0.0/16 218 | - 172.16.0.0/12 219 | - 192.0.0.0/24 220 | - 192.0.0.0/29 221 | - 192.0.0.170/32 222 | - 192.0.0.171/32 223 | - 192.0.0.8/32 224 | - 192.0.2.0/24 225 | - 192.168.0.0/16 226 | - 192.175.48.0/24 227 | - 192.31.196.0/24 228 | - 192.52.193.0/24 229 | - 192.88.99.0/24 230 | - 198.18.0.0/15 231 | - 198.51.100.0/24 232 | - 203.0.113.0/24 233 | - 240.0.0.0/4 234 | - 255.255.255.255/32 235 | 236 | # Set to 1 to receive all txs (unused) 237 | relay: 0 238 | 239 | # List of excluded ASNs 240 | exclude_asns: [] 241 | 242 | # List of excluded IPv6 networks 243 | exclude_ipv6_networks: [] 244 | 245 | # Bogons are addresses outside the range of valid ipv4 addresses 246 | exclude_ipv4_bogons: True 247 | 248 | # Use median percentile +/- N interquartile ranges in place of block height to determine activity 249 | inactive_use_iqr: False 250 | 251 | 252 | -------------------------------------------------------------------------------- /autodoc.py: -------------------------------------------------------------------------------- 1 | """ 2 | Automatic documentation generator for Flask 3 | 4 | Modified from https://github.com/acoomans/flask-autodoc 5 | Copyright (c) 2013 Arnaud Coomans 6 | Copyright (c) 2018 Opennodes / Blake Bjorn Anderson 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining 9 | a copy of this software and associated documentation files (the 10 | "Software"), to deal in the Software without restriction, including 11 | without limitation the rights to use, copy, modify, merge, publish, 12 | distribute, sublicense, and/or sell copies of the Software, and to 13 | permit persons to whom the Software is furnished to do so, subject to 14 | the following conditions: 15 | 16 | The above copyright notice and this permission notice shall be 17 | included in all copies or substantial portions of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 22 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 23 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 24 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | """ 27 | import inspect 28 | import json 29 | import os 30 | import re 31 | import sys 32 | from collections import defaultdict 33 | from operator import attrgetter 34 | from typing import Callable, Optional, Any, Dict 35 | from flask import current_app, render_template, render_template_string 36 | from flask.app import Flask 37 | from jinja2 import evalcontextfilter 38 | 39 | try: 40 | from flask import _app_ctx_stack as stack 41 | except ImportError: 42 | from flask import _request_ctx_stack as stack 43 | 44 | if sys.version < '3': 45 | get_function_code = attrgetter('func_code') 46 | else: 47 | get_function_code = attrgetter('__code__') 48 | 49 | 50 | class Autodoc(object): 51 | def __init__(self, app: Optional[Flask] = None) -> None: 52 | self.app = app 53 | self.func_groups = defaultdict(set) 54 | self.func_locations = defaultdict(dict) 55 | if app is not None: 56 | self.init_app(app) 57 | 58 | def init_app(self, app: Flask) -> None: 59 | if hasattr(app, 'teardown_appcontext'): 60 | app.teardown_appcontext(self.teardown) 61 | else: 62 | app.teardown_request(self.teardown) 63 | self.add_custom_template_filters(app) 64 | 65 | def teardown(self, exception: None) -> None: 66 | ctx = stack.top 67 | 68 | def add_custom_template_filters(self, app: Flask) -> None: 69 | """Add custom filters to jinja2 templating engine""" 70 | self.add_custom_nl2br_filters(app) 71 | 72 | def add_custom_nl2br_filters(self, app: Flask) -> None: 73 | """Add a custom filter nl2br to jinja2 74 | Replaces all newline to
75 | """ 76 | _paragraph_re = re.compile(r'(?:\r\n|\r|\n){3,}') 77 | 78 | @app.template_filter() 79 | @evalcontextfilter 80 | def nl2br(eval_ctx, value): 81 | result = '\n\n'.join('%s' % p.replace('\n', '
\n') 82 | for p in _paragraph_re.split(value)) 83 | return result 84 | 85 | def doc(self, groups: None = None) -> Callable: 86 | """Add flask route to autodoc for automatic documentation 87 | 88 | Any route decorated with this method will be added to the list of 89 | routes to be documented by the generate() or html() methods. 90 | 91 | By default, the route is added to the 'all' group. 92 | By specifying group or groups argument, the route can be added to one 93 | or multiple other groups as well, besides the 'all' group. 94 | """ 95 | 96 | def decorator(f): 97 | # Set group[s] 98 | if type(groups) is list: 99 | groupset = set(groups) 100 | else: 101 | groupset = set() 102 | if type(groups) is str: 103 | groupset.add(groups) 104 | groupset.add('all') 105 | self.func_groups[f] = groupset 106 | 107 | # Set location 108 | caller_frame = inspect.stack()[1] 109 | self.func_locations[f] = { 110 | 'filename': caller_frame[1], 111 | 'line': caller_frame[2], 112 | } 113 | 114 | return f 115 | 116 | return decorator 117 | 118 | def deconstruct_docstring(self, docstring): 119 | docstring = str(docstring) 120 | 121 | params = re.findall("(\:param )(.*?\: )(.*)", docstring) 122 | returns = re.findall("(\:return: )(.*)", docstring) 123 | 124 | if params: 125 | docstring = docstring.split("".join(params[0]))[0].strip() 126 | elif returns: 127 | docstring = docstring.split("".join(returns[0]))[0].strip() 128 | return docstring, params, returns 129 | 130 | def generate(self, groups='all', sort=None): 131 | """Return a list of dict describing the routes specified by the 132 | doc() method 133 | 134 | Each dict contains: 135 | - methods: the set of allowed methods (ie ['GET', 'POST']) 136 | - rule: relative url (ie '/user/') 137 | - endpoint: function name (ie 'show_user') 138 | - doc: docstring of the function 139 | - args: function arguments 140 | - defaults: defaults values for the arguments 141 | 142 | By specifying the group or groups arguments, only routes belonging to 143 | those groups will be returned. 144 | 145 | Routes are sorted alphabetically based on the rule. 146 | """ 147 | groups_to_generate = list() 148 | if type(groups) is list: 149 | groups_to_generate = groups 150 | elif type(groups) is str: 151 | groups_to_generate.append(groups) 152 | 153 | links = [] 154 | for rule in current_app.url_map.iter_rules(): 155 | if rule.endpoint == 'static': 156 | continue 157 | func = current_app.view_functions[rule.endpoint] 158 | func_groups = self.func_groups[func] 159 | location = self.func_locations.get(func, None) 160 | 161 | if func_groups.intersection(groups_to_generate): 162 | docstring, arguments, returns = self.deconstruct_docstring(func.__doc__) 163 | 164 | if isinstance(arguments, set): 165 | arguments = list(arguments) 166 | arguments = arguments if len(arguments) >= 1 and arguments[0] != "None" else None 167 | links.append( 168 | dict( 169 | methods=sorted([x for x in rule.methods if x not in ("HEAD", "OPTIONS")]), 170 | rule="%s" % rule, 171 | endpoint=rule.endpoint, 172 | docstring=docstring, 173 | args=arguments, 174 | defaults=rule.defaults, 175 | location=location, 176 | returns=returns 177 | ) 178 | ) 179 | if sort: 180 | return sort(links) 181 | else: 182 | return sorted(links, key=lambda x: x['rule'].lower()) 183 | 184 | def html(self, groups='all', template=None, **context): 185 | """Return an html string of the routes specified by the doc() method 186 | 187 | A template can be specified. A list of routes is available under the 188 | 'autodoc' value (refer to the documentation for the generate() for a 189 | description of available values). If no template is specified, a 190 | default template is used. 191 | 192 | By specifying the group or groups arguments, only routes belonging to 193 | those groups will be returned. 194 | """ 195 | if template: 196 | return render_template(template, 197 | autodoc=self.generate(groups=groups), 198 | **context) 199 | else: 200 | filename = os.path.join( 201 | os.path.dirname(__file__), 202 | 'templates', 203 | 'docs.html' 204 | ) 205 | with open(filename) as file: 206 | content = file.read() 207 | with current_app.app_context(): 208 | return render_template_string( 209 | content, 210 | autodoc=self.generate(groups=groups), 211 | **context) 212 | -------------------------------------------------------------------------------- /models.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright (c) 2018 Opennodes / blakebjorn 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining 6 | # a copy of this software and associated documentation files (the 7 | # "Software"), to deal in the Software without restriction, including 8 | # without limitation the rights to use, copy, modify, merge, publish, 9 | # distribute, sublicense, and/or sell copies of the Software, and to 10 | # permit persons to whom the Software is furnished to do so, subject to 11 | # the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be 14 | # included in all copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | from sqlalchemy.ext.declarative import declarative_base 25 | from sqlalchemy.orm import validates, sessionmaker 26 | from sqlalchemy import Column, Integer, String, DateTime, Boolean, Index, BIGINT, create_engine 27 | import datetime 28 | 29 | import config 30 | 31 | Base = declarative_base() 32 | 33 | 34 | class Node(Base): 35 | __tablename__ = 'nodes' 36 | 37 | id = Column(Integer, primary_key=True) 38 | network = Column(String(20), nullable=False) 39 | address = Column(String(50), nullable=False) 40 | port = Column(Integer, nullable=False) 41 | first_seen = Column(DateTime, nullable=True) 42 | last_seen = Column(DateTime, nullable=True) 43 | first_checked = Column(DateTime, nullable=True) 44 | last_checked = Column(DateTime, nullable=True) 45 | seen = Column(Boolean, default=False, nullable=False) 46 | last_height = Column(BIGINT, nullable=True) 47 | version = Column(Integer, nullable=True) 48 | user_agent = Column(String(75), nullable=True) 49 | services = Column(BIGINT, default=0, nullable=False) 50 | country = Column(String(60), nullable=True) 51 | city = Column(String(60), nullable=True) 52 | asn = Column(Integer, nullable=True) 53 | aso = Column(String(100), nullable=True) 54 | is_masternode = Column(Boolean, default=False, nullable=False) 55 | 56 | Index('idx_node', 'network', 'address', 'port', unique=True) 57 | 58 | def to_dict(self): 59 | return { 60 | "id": self.id, 61 | "network": self.network, 62 | "address": self.address, 63 | "port": self.port, 64 | "first_seen": self.first_seen, 65 | "last_seen": self.last_seen, 66 | "first_checked": self.first_checked, 67 | "last_checked": self.last_checked, 68 | "seen": self.seen, 69 | "last_height": self.last_height, 70 | "version": self.version, 71 | "user_agent": self.user_agent, 72 | "services": self.services, 73 | "country": self.country, 74 | "city": self.city, 75 | "asn": self.asn, 76 | "aso": self.aso, 77 | "is_masternode": self.is_masternode 78 | } 79 | 80 | def from_dict(self, d): 81 | self.id = d['id'] 82 | self.network = d["network"] 83 | self.address = d["address"] 84 | self.port = d['port'] 85 | self.first_seen = d["first_seen"] 86 | self.last_seen = d["last_seen"] 87 | self.first_checked = d['first_checked'] 88 | self.last_checked = d["last_checked"] 89 | self.seen = d["seen"] 90 | self.last_height = d["last_height"] 91 | self.version = d["version"] 92 | self.user_agent = d["user_agent"] 93 | self.services = d["services"] 94 | self.country = d['country'] 95 | self.city = d['city'] 96 | self.asn = d['asn'] 97 | self.aso = d['aso'] 98 | self.is_masternode = d['is_masternode'] 99 | 100 | @staticmethod 101 | def new_from_dict(d): 102 | obj = Node() 103 | obj.id = d['id'] if 'id' in d else None 104 | obj.network = d["network"] if "network" in d else None 105 | obj.address = d["address"] if "address" in d else None 106 | obj.port = d['port'] if 'port' in d else None 107 | obj.first_seen = d["first_seen"] if "first_seen" in d else None 108 | obj.last_seen = d["last_seen"] if "last_seen" in d else None 109 | obj.first_checked = d['first_checked'] if "first_checked" in d else None 110 | obj.last_checked = d["last_checked"] if "last_checked" in d else None 111 | obj.seen = d["seen"] if "seen" in d else None 112 | obj.last_height = d["last_height"] if "last_height" in d else None 113 | obj.version = d["version"] if "version" in d else None 114 | obj.user_agent = d["user_agent"] if "user_agent" in d else None 115 | obj.services = d["services"] if "services" in d else None 116 | obj.country = d['country'] if 'country' in d else None 117 | obj.city = d['city'] if 'city' in d else None 118 | obj.asn = d['asn'] if 'asn' in d else None 119 | obj.aso = d['aso'] if 'aso' in d else None 120 | obj.is_masternode = d['is_masternode'] if 'is_masternode' in d else None 121 | return obj 122 | 123 | def __repr__(self): 124 | return "".format(self.to_dict()) 125 | 126 | @validates('port', 'last_height', 'services', 'version', 'asn') 127 | def validate_integers(self, key, field): 128 | if field is not None: 129 | if field > 9223372036854775807: 130 | print("{}:{} is > SQLite Max Value. Truncating".format(key, field)) 131 | return 9223372036854775807 132 | return int(field) 133 | return None 134 | 135 | @validates('address', 'user_agent', 'country', 'city', 'aso') 136 | def validate_string(self, key, field): 137 | if field is not None: 138 | if key == 'address': 139 | if len(field) > 50: 140 | print(key, field, "over max len") 141 | return field[:50] 142 | elif key == "aso": 143 | if len(field) > 100: 144 | print(key, field, "over max len") 145 | return field[:100] 146 | elif key == "user_agent": 147 | if len(field) > 75: 148 | print(key, field, "over max len") 149 | return field[:75] 150 | elif len(field) > 60: 151 | print(key, field, "over max len") 152 | return field[:60] 153 | return field 154 | 155 | 156 | class CrawlSummary(Base): 157 | __tablename__ = 'crawl_summaries' 158 | 159 | id = Column(Integer, primary_key=True) 160 | timestamp = Column(DateTime, default=datetime.datetime.utcnow()) 161 | network = Column(String(255), nullable=False) 162 | node_count = Column(Integer, nullable=False) 163 | masternode_count = Column(Integer, nullable=False) 164 | lookback_hours = Column(Integer) 165 | 166 | def __repr__(self): 167 | return "".format( 168 | self.network, self.timestamp.isoformat(), self.node_count, self.masternode_count, self.lookback_hours) 169 | 170 | 171 | class UserAgent(Base): 172 | __tablename__ = 'user_agents' 173 | 174 | id = Column(Integer, primary_key=True) 175 | user_agent = Column(String(60)) 176 | 177 | Index('idx_user_agent', 'user_agent') 178 | 179 | 180 | class NodeVisitation(Base): 181 | __tablename__ = 'node_visitations' 182 | 183 | id = Column(Integer, primary_key=True) 184 | parent_id = Column(Integer) 185 | timestamp = Column(DateTime, default=datetime.datetime.utcnow()) 186 | success = Column(Boolean, default=False) 187 | height = Column(BIGINT, nullable=True) 188 | user_agent_id = Column(Integer) 189 | is_masternode = Column(Boolean, default=None) 190 | 191 | Index('idx_vis_timestamp', 'timestamp') 192 | 193 | def to_dict(self): 194 | return { 195 | "id": self.id, 196 | "parent_id": self.parent_id, 197 | "timestamp": self.timestamp, 198 | "success": self.success, 199 | "user_agent_id": self.user_agent_id, 200 | "is_masternode": self.is_masternode, 201 | "height": self.height 202 | } 203 | 204 | def from_dict(self, d): 205 | self.id = d['id'] 206 | self.parent_id = d['parent_id'] 207 | self.timestamp = d["timestamp"] 208 | self.success = d["success"] 209 | self.height = d["height"] 210 | self.is_masternode = d['is_masternode'] 211 | self.user_agent_id = d['user_agent_id'] 212 | 213 | @staticmethod 214 | def new_from_dict(d): 215 | obj = NodeVisitation() 216 | obj.id = d['id'] if 'id' in d else None 217 | obj.parent_id = d['parent_id'] if 'parent_id' in d else None 218 | obj.timestamp = d["timestamp"] if 'timestamp' in d else None 219 | obj.success = d["success"] if 'success' in d else None 220 | obj.height = d["height"] if 'height' in d else None 221 | obj.is_masternode = d['is_masternode'] if 'is_masternode' in d else None 222 | obj.user_agent_id = d['user_agent_id'] if 'user_agent_id' in d else None 223 | return obj 224 | 225 | @validates('user_agent') 226 | def validate_string(self, key, field): 227 | if field is not None and len(field) > 60: 228 | print(key, field, "over max len") 229 | return field[:60] 230 | return field 231 | 232 | def __repr__(self): 233 | return "".format(self.id, self.parent_id, self.timestamp, self.success) 234 | 235 | 236 | def init_db(): 237 | if "sqlite:/" in config.DATABASE_URI: 238 | engine = create_engine(config.DATABASE_URI, connect_args={'timeout': 15}, echo=False) 239 | else: 240 | engine = create_engine(config.DATABASE_URI, echo=False) 241 | Base.metadata.create_all(engine) 242 | return sessionmaker(bind=engine, autoflush=False)() 243 | 244 | 245 | session = init_db() 246 | -------------------------------------------------------------------------------- /static/flags/flags.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Generated with CSS Flag Sprite generator (https://www.flag-sprites.com/) 3 | */.flag{display:inline-block;width:24px;height:24px;background:url('flags.png') no-repeat} .flag.flag-ad{background-position:-24px 0} .flag.flag-ae{background-position:-48px 0} .flag.flag-af{background-position:-72px 0} .flag.flag-ag{background-position:-96px 0} .flag.flag-ai{background-position:-120px 0} .flag.flag-al{background-position:-144px 0} .flag.flag-am{background-position:-168px 0} .flag.flag-an{background-position:-192px 0} .flag.flag-ao{background-position:-216px 0} .flag.flag-ar{background-position:-240px 0} .flag.flag-as{background-position:-264px 0} .flag.flag-at{background-position:-288px 0} .flag.flag-au{background-position:-312px 0} .flag.flag-aw{background-position:-336px 0} .flag.flag-ax{background-position:-360px 0} .flag.flag-az{background-position:0 -24px} .flag.flag-ba{background-position:-24px -24px} .flag.flag-bb{background-position:-48px -24px} .flag.flag-bd{background-position:-72px -24px} .flag.flag-be{background-position:-96px -24px} .flag.flag-bf{background-position:-120px -24px} .flag.flag-bg{background-position:-144px -24px} .flag.flag-bh{background-position:-168px -24px} .flag.flag-bi{background-position:-192px -24px} .flag.flag-bj{background-position:-216px -24px} .flag.flag-bl{background-position:-240px -24px} .flag.flag-bm{background-position:-264px -24px} .flag.flag-bn{background-position:-288px -24px} .flag.flag-bo{background-position:-312px -24px} .flag.flag-br{background-position:-336px -24px} .flag.flag-bs{background-position:-360px -24px} .flag.flag-bt{background-position:0 -48px} .flag.flag-bw{background-position:-24px -48px} .flag.flag-by{background-position:-48px -48px} .flag.flag-bz{background-position:-72px -48px} .flag.flag-ca{background-position:-96px -48px} .flag.flag-cd{background-position:-120px -48px} .flag.flag-cf{background-position:-144px -48px} .flag.flag-cg{background-position:-168px -48px} .flag.flag-ch{background-position:-192px -48px} .flag.flag-ci{background-position:-216px -48px} .flag.flag-ck{background-position:-240px -48px} .flag.flag-cl{background-position:-264px -48px} .flag.flag-cm{background-position:-288px -48px} .flag.flag-cn{background-position:-312px -48px} .flag.flag-co{background-position:-336px -48px} .flag.flag-cr{background-position:-360px -48px} .flag.flag-cu{background-position:0 -72px} .flag.flag-cv{background-position:-24px -72px} .flag.flag-cw{background-position:-48px -72px} .flag.flag-cy{background-position:-72px -72px} .flag.flag-cz{background-position:-96px -72px} .flag.flag-de{background-position:-120px -72px} .flag.flag-dj{background-position:-144px -72px} .flag.flag-dk{background-position:-168px -72px} .flag.flag-dm{background-position:-192px -72px} .flag.flag-do{background-position:-216px -72px} .flag.flag-dz{background-position:-240px -72px} .flag.flag-ec{background-position:-264px -72px} .flag.flag-ee{background-position:-288px -72px} .flag.flag-eg{background-position:-312px -72px} .flag.flag-eh{background-position:-336px -72px} .flag.flag-er{background-position:-360px -72px} .flag.flag-es{background-position:0 -96px} .flag.flag-et{background-position:-24px -96px} .flag.flag-eu{background-position:-48px -96px} .flag.flag-fi{background-position:-72px -96px} .flag.flag-fj{background-position:-96px -96px} .flag.flag-fk{background-position:-120px -96px} .flag.flag-fm{background-position:-144px -96px} .flag.flag-fo{background-position:-168px -96px} .flag.flag-fr{background-position:-192px -96px} .flag.flag-ga{background-position:-216px -96px} .flag.flag-gb{background-position:-240px -96px} .flag.flag-gd{background-position:-264px -96px} .flag.flag-ge{background-position:-288px -96px} .flag.flag-gg{background-position:-312px -96px} .flag.flag-gh{background-position:-336px -96px} .flag.flag-gi{background-position:-360px -96px} .flag.flag-gl{background-position:0 -120px} .flag.flag-gm{background-position:-24px -120px} .flag.flag-gn{background-position:-48px -120px} .flag.flag-gq{background-position:-72px -120px} .flag.flag-gr{background-position:-96px -120px} .flag.flag-gs{background-position:-120px -120px} .flag.flag-gt{background-position:-144px -120px} .flag.flag-gu{background-position:-168px -120px} .flag.flag-gw{background-position:-192px -120px} .flag.flag-gy{background-position:-216px -120px} .flag.flag-hk{background-position:-240px -120px} .flag.flag-hn{background-position:-264px -120px} .flag.flag-hr{background-position:-288px -120px} .flag.flag-ht{background-position:-312px -120px} .flag.flag-hu{background-position:-336px -120px} .flag.flag-ic{background-position:-360px -120px} .flag.flag-id{background-position:0 -144px} .flag.flag-ie{background-position:-24px -144px} .flag.flag-il{background-position:-48px -144px} .flag.flag-im{background-position:-72px -144px} .flag.flag-in{background-position:-96px -144px} .flag.flag-iq{background-position:-120px -144px} .flag.flag-ir{background-position:-144px -144px} .flag.flag-is{background-position:-168px -144px} .flag.flag-it{background-position:-192px -144px} .flag.flag-je{background-position:-216px -144px} .flag.flag-jm{background-position:-240px -144px} .flag.flag-jo{background-position:-264px -144px} .flag.flag-jp{background-position:-288px -144px} .flag.flag-ke{background-position:-312px -144px} .flag.flag-kg{background-position:-336px -144px} .flag.flag-kh{background-position:-360px -144px} .flag.flag-ki{background-position:0 -168px} .flag.flag-km{background-position:-24px -168px} .flag.flag-kn{background-position:-48px -168px} .flag.flag-kp{background-position:-72px -168px} .flag.flag-kr{background-position:-96px -168px} .flag.flag-kw{background-position:-120px -168px} .flag.flag-ky{background-position:-144px -168px} .flag.flag-kz{background-position:-168px -168px} .flag.flag-la{background-position:-192px -168px} .flag.flag-lb{background-position:-216px -168px} .flag.flag-lc{background-position:-240px -168px} .flag.flag-li{background-position:-264px -168px} .flag.flag-lk{background-position:-288px -168px} .flag.flag-lr{background-position:-312px -168px} .flag.flag-ls{background-position:-336px -168px} .flag.flag-lt{background-position:-360px -168px} .flag.flag-lu{background-position:0 -192px} .flag.flag-lv{background-position:-24px -192px} .flag.flag-ly{background-position:-48px -192px} .flag.flag-ma{background-position:-72px -192px} .flag.flag-mc{background-position:-96px -192px} .flag.flag-md{background-position:-120px -192px} .flag.flag-me{background-position:-144px -192px} .flag.flag-mf{background-position:-168px -192px} .flag.flag-mg{background-position:-192px -192px} .flag.flag-mh{background-position:-216px -192px} .flag.flag-mk{background-position:-240px -192px} .flag.flag-ml{background-position:-264px -192px} .flag.flag-mm{background-position:-288px -192px} .flag.flag-mn{background-position:-312px -192px} .flag.flag-mo{background-position:-336px -192px} .flag.flag-mp{background-position:-360px -192px} .flag.flag-mq{background-position:0 -216px} .flag.flag-mr{background-position:-24px -216px} .flag.flag-ms{background-position:-48px -216px} .flag.flag-mt{background-position:-72px -216px} .flag.flag-mu{background-position:-96px -216px} .flag.flag-mv{background-position:-120px -216px} .flag.flag-mw{background-position:-144px -216px} .flag.flag-mx{background-position:-168px -216px} .flag.flag-my{background-position:-192px -216px} .flag.flag-mz{background-position:-216px -216px} .flag.flag-na{background-position:-240px -216px} .flag.flag-nc{background-position:-264px -216px} .flag.flag-ne{background-position:-288px -216px} .flag.flag-nf{background-position:-312px -216px} .flag.flag-ng{background-position:-336px -216px} .flag.flag-ni{background-position:-360px -216px} .flag.flag-nl{background-position:0 -240px} .flag.flag-no{background-position:-24px -240px} .flag.flag-np{background-position:-48px -240px} .flag.flag-nr{background-position:-72px -240px} .flag.flag-nu{background-position:-96px -240px} .flag.flag-nz{background-position:-120px -240px} .flag.flag-om{background-position:-144px -240px} .flag.flag-pa{background-position:-168px -240px} .flag.flag-pe{background-position:-192px -240px} .flag.flag-pf{background-position:-216px -240px} .flag.flag-pg{background-position:-240px -240px} .flag.flag-ph{background-position:-264px -240px} .flag.flag-pk{background-position:-288px -240px} .flag.flag-pl{background-position:-312px -240px} .flag.flag-pn{background-position:-336px -240px} .flag.flag-pr{background-position:-360px -240px} .flag.flag-ps{background-position:0 -264px} .flag.flag-pt{background-position:-24px -264px} .flag.flag-pw{background-position:-48px -264px} .flag.flag-py{background-position:-72px -264px} .flag.flag-qa{background-position:-96px -264px} .flag.flag-ro{background-position:-120px -264px} .flag.flag-rs{background-position:-144px -264px} .flag.flag-ru{background-position:-168px -264px} .flag.flag-rw{background-position:-192px -264px} .flag.flag-sa{background-position:-216px -264px} .flag.flag-sb{background-position:-240px -264px} .flag.flag-sc{background-position:-264px -264px} .flag.flag-sd{background-position:-288px -264px} .flag.flag-se{background-position:-312px -264px} .flag.flag-sg{background-position:-336px -264px} .flag.flag-sh{background-position:-360px -264px} .flag.flag-si{background-position:0 -288px} .flag.flag-sk{background-position:-24px -288px} .flag.flag-sl{background-position:-48px -288px} .flag.flag-sm{background-position:-72px -288px} .flag.flag-sn{background-position:-96px -288px} .flag.flag-so{background-position:-120px -288px} .flag.flag-sr{background-position:-144px -288px} .flag.flag-ss{background-position:-168px -288px} .flag.flag-st{background-position:-192px -288px} .flag.flag-sv{background-position:-216px -288px} .flag.flag-sy{background-position:-240px -288px} .flag.flag-sz{background-position:-264px -288px} .flag.flag-tc{background-position:-288px -288px} .flag.flag-td{background-position:-312px -288px} .flag.flag-tf{background-position:-336px -288px} .flag.flag-tg{background-position:-360px -288px} .flag.flag-th{background-position:0 -312px} .flag.flag-tj{background-position:-24px -312px} .flag.flag-tk{background-position:-48px -312px} .flag.flag-tl{background-position:-72px -312px} .flag.flag-tm{background-position:-96px -312px} .flag.flag-tn{background-position:-120px -312px} .flag.flag-to{background-position:-144px -312px} .flag.flag-tr{background-position:-168px -312px} .flag.flag-tt{background-position:-192px -312px} .flag.flag-tv{background-position:-216px -312px} .flag.flag-tw{background-position:-240px -312px} .flag.flag-tz{background-position:-264px -312px} .flag.flag-ua{background-position:-288px -312px} .flag.flag-ug{background-position:-312px -312px} .flag.flag-us{background-position:-336px -312px} .flag.flag-uy{background-position:-360px -312px} .flag.flag-uz{background-position:0 -336px} .flag.flag-va{background-position:-24px -336px} .flag.flag-vc{background-position:-48px -336px} .flag.flag-ve{background-position:-72px -336px} .flag.flag-vg{background-position:-96px -336px} .flag.flag-vi{background-position:-120px -336px} .flag.flag-vn{background-position:-144px -336px} .flag.flag-vu{background-position:-168px -336px} .flag.flag-wf{background-position:-192px -336px} .flag.flag-ws{background-position:-216px -336px} .flag.flag-ye{background-position:-240px -336px} .flag.flag-yt{background-position:-264px -336px} .flag.flag-za{background-position:-288px -336px} .flag.flag-zm{background-position:-312px -336px} .flag.flag-zw{background-position:-336px -336px} -------------------------------------------------------------------------------- /server.py: -------------------------------------------------------------------------------- 1 | """ 2 | Open Nodes web server 3 | Copyright (c) 2018 Opennodes / Blake Bjorn Anderson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | """ 24 | import datetime 25 | import gzip 26 | import json 27 | import os 28 | import sys 29 | from io import BytesIO 30 | 31 | import pandas as pd 32 | import waitress 33 | from flask import Flask, render_template, request, redirect, flash, Response 34 | from flask_sqlalchemy import SQLAlchemy 35 | from geoip2.errors import AddressNotFoundError 36 | from sqlalchemy import and_ 37 | 38 | from autodoc import Autodoc 39 | from config import load_config, DATABASE_URI 40 | from crawler import COUNTRY, CITY, ASN, connect, update_masternode_list 41 | from models import Node, NodeVisitation 42 | 43 | app = Flask(__name__) 44 | auto = Autodoc(app) 45 | app.config['SQLALCHEMY_DATABASE_URI'] = DATABASE_URI 46 | app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False 47 | db = SQLAlchemy(app) 48 | 49 | CONF = load_config() 50 | 51 | 52 | @app.route('/') 53 | @app.route('/networks/', methods=['GET']) 54 | def network_dashboard(network_name=None): 55 | if not network_name in ("bitcoin", "bitcoin-cash", "litecoin", "dash", "bitcoin-sv", None): 56 | flash("Invalid network") 57 | return redirect("/") 58 | 59 | with open("static/network_summaries.json", 'r') as f: 60 | summaries = json.load(f) 61 | 62 | if network_name: 63 | age_min = summaries[network_name]['age_min'] 64 | age_max = summaries[network_name]['age_max'] 65 | else: 66 | age_min = min((summaries[network]['age_min'] for network in CONF['networks'])) 67 | age_max = max((summaries[network]['age_max'] for network in CONF['networks'])) 68 | 69 | return render_template("network_dashboard.html", 70 | network=network_name, 71 | has_masternodes=True if network_name == "dash" else False, 72 | include_client=False if network_name is not None else False, 73 | include_user_agent=True if network_name is not None else False, 74 | include_network=True if network_name is None else False, 75 | include_version=True if network_name is not None else False, 76 | include_active=True if CONF['export_inactive_nodes'] else False, 77 | age_min=age_min * 1000.0, 78 | age_max=age_max * 1000.0) 79 | 80 | 81 | def gzip_response(input_str, pre_compressed): 82 | response = Response() 83 | if not pre_compressed: 84 | buffer = BytesIO() 85 | gzip_file = gzip.GzipFile(mode='wb', fileobj=buffer) 86 | gzip_file.write(input_str if isinstance(input_str, bytes) else input_str.encode()) 87 | gzip_file.close() 88 | response.data = buffer.getvalue() 89 | else: 90 | response.data = input_str 91 | response.headers['Content-Encoding'] = 'gzip' 92 | response.headers['Vary'] = 'Accept-Encoding' 93 | response.headers['Content-Length'] = len(response.data) 94 | return response 95 | 96 | 97 | @app.route('/api/get_networks', methods=['POST']) 98 | @auto.doc() 99 | def get_networks(): 100 | """ 101 | Returns a list of all available network names 102 | :return: JSON string, ex. "['bitcoin','bitcoin-cash','dash','litecoin']" 103 | """ 104 | return json.dumps([x[0] for x in db.session.query(Node.network).distinct().all()]) 105 | 106 | 107 | @app.route('/api/gzip_file/', methods=['GET']) 108 | @auto.doc() 109 | def gzip_static_file(filename): 110 | """ 111 | Returns a crawl result as a gzipped response 112 | :param filename: file_network.ext - file is 'data' or 'history', ext is either .json, .csv, .txt (data.ext returns data for all crawled networks) 113 | :return: gzip encoded html response 114 | """ 115 | valid_files = ["custom.geo.json"] 116 | for coin in ("", "_bitcoin", "_bitcoin-cash", "_dash", "_litecoin", "_bitcoin-sv"): 117 | for suff in ("", "_unique"): 118 | for ext in (".csv", ".json", ".txt"): 119 | valid_files.append("data" + coin + suff + ext) 120 | valid_files.append("history" + coin + '.json') 121 | if filename not in valid_files: 122 | return redirect("/", code=404) 123 | with open(os.path.join("static", filename), "r") as f: 124 | return gzip_response(f.read(), False) 125 | 126 | 127 | def deconstruct_address_string(inp): 128 | assert isinstance(inp, str) 129 | 130 | resp = {} 131 | aliases = {'btc': 'bitcoin', 132 | 'bch': 'bitcoin-cash', 133 | 'bcc': 'bitcoin-cash', 134 | 'bitcoin-sv': 'bitcoin-cash', 135 | 'bsv': 'bitcoin-cash', 136 | 'ltc': 'litecoin'} 137 | 138 | inp = inp.lower() 139 | network = inp.split(":")[0] 140 | if network: 141 | inp = ":".join(inp.split(":")[1:]) 142 | network = aliases[network] if network in aliases else network 143 | network = network if network in CONF['networks'] else None 144 | if not network: 145 | network = "bitcoin" 146 | resp['warning'] = "Network not recognized, using BTC" 147 | 148 | if ":" in inp: 149 | port = inp.split(":")[-1] 150 | try: 151 | port = int(port) 152 | inp = ":".join(inp.split(":")[:-1]) 153 | except ValueError: 154 | resp['warning'] = "port not recognized, using default" 155 | port = int(CONF['networks'][network]['port']) 156 | else: 157 | port = int(CONF['networks'][network]['port']) 158 | 159 | return network, inp, port, resp 160 | 161 | 162 | @app.route('/api/check_node', methods=['POST']) 163 | @auto.doc() 164 | def check_node(): 165 | """ 166 | Checks the current status of a node. This is a live result, so response times will be longer - to view a saved 167 | result see /api/check_historic_node. 168 | :param node: connection string, e.g. btc:127.0.0.1:8333 - port is optional if it is the network default 169 | :param to_services (integer, optional): outgoing services to broadcast, default=0 170 | :param from_services (integer, optional): outgoing services to broadcast, default=0 171 | :param version (integer, optional): version code to broadcast, default varies by network 172 | :param user_agent (string, optional): user agent to broadcast, default="/open-nodes:0.1/" 173 | :param height (integer, optional): block height to broadcast during handshake. default=network median 174 | :param p2p_nodes (bool, optional): issues a getaddr call and list of connected nodes, default=False 175 | :return: json dict {"result":{"user_agent":"/satoshi:17.0.1/", "version":" .... }, "nodes":[["127.0.0.1:8333, 157532132191], ...]} 176 | """ 177 | 178 | dat = request.form 179 | node = dat.get("node") 180 | network, address, port, resp = deconstruct_address_string(node) 181 | 182 | network_data = CONF['networks'][network] 183 | if dat.get("height"): 184 | network_data['height'] = dat.get("height") 185 | else: 186 | with open("static/network_summaries.json", 'r') as f: 187 | network_data['height'] = int(json.load(f)[network]['med']) 188 | 189 | network_data['protocol_version'] = dat.get("version") or network_data['protocol_version'] 190 | result = connect(network, address, port, 191 | to_services=dat.get("to_services") or network_data['services'], 192 | network_data=network_data, 193 | user_agent=dat.get("user_agent") or None, 194 | p2p_nodes=False, 195 | explicit_p2p=dat.get("p2p_nodes") or False, 196 | from_services=dat.get('from_services') or None, 197 | keepalive=False) 198 | 199 | resp['result'] = result[0] 200 | resp['nodes'] = result[1] 201 | 202 | resp['result'] = geocode(resp['result']) 203 | return to_json(resp) 204 | 205 | 206 | @app.route('/api/check_historic_node', methods=['POST', 'GET']) 207 | @auto.doc() 208 | def check_historic_node(): 209 | """ 210 | Checks the status of a node based on the last crawl 211 | result see /api/check_historical_node 212 | :param node: connection string, e.g. btc:127.0.0.1:8333 - port is optional if it is the network default 213 | :return: json dict {"result":{"user_agent":"/satoshi:17.0.1/", "version":" .... }} 214 | """ 215 | 216 | if request.method == "POST": 217 | dat = request.form 218 | else: 219 | dat = request.args 220 | node = dat.get("node") 221 | 222 | network, address, port, resp = deconstruct_address_string(node) 223 | 224 | if network not in CONF['networks']: 225 | return json.dumps({'error': "network not recognized"}) 226 | 227 | result = db.session.query(Node).get((network, address, port)) 228 | resp['result'] = "None" if result is None else result.to_dict() 229 | 230 | return to_json(resp) 231 | 232 | 233 | @app.route("/about") 234 | def about(): 235 | return render_template("about.html") 236 | 237 | 238 | @app.route("/api_docs") 239 | def api_docs(): 240 | return auto.html() 241 | 242 | 243 | @app.route('/api/get_nodes', methods=['POST']) 244 | @auto.doc() 245 | def get_node_list(): 246 | """ 247 | Gets a list of all nodes visible during the past 30 days 248 | :param network (optional): Filters the result set based on the given network 249 | :return: json array [{"address":"127.0.0.1" ... }, {"address":"0.0.0.0", "port:8333}] 250 | """ 251 | 252 | q = db.session.query(Node.network, Node.address, Node.port, Node.user_agent, Node.version, Node.first_seen, 253 | Node.last_seen, Node.last_checked, Node.country, Node.city, Node.asn, Node.aso).filter( 254 | Node.seen) 255 | if request.args.get("network") is not None: 256 | network = request.args.get("network") 257 | if network not in CONF['networks']: 258 | return {"error": "network must be one of " + ", ".join(CONF['networks'])} 259 | q = q.filter(Node.network == network) 260 | return pd.read_sql(q.statement, q.session.bind).to_json(orient='records') 261 | 262 | 263 | @app.route('/api/get_dash_masternodes', methods=['POST']) 264 | @auto.doc() 265 | def get_dash_masternodes(): 266 | """ 267 | Returns a list of all active dash masternodes - requires running dashd service on target server 268 | :return: json array ["45.76.112.193:9999", "206.189.110.182:9999", ...] 269 | """ 270 | if not os.path.isfile(os.path.join("static", "masternode_list.txt")): 271 | return json.dumps(list(update_masternode_list())) 272 | else: 273 | with open(os.path.join("static", "masternode_list.txt"), "r") as f: 274 | return json.dumps(f.read().splitlines(keepends=False)) 275 | 276 | 277 | @app.route('/api/node_history', methods=['POST']) 278 | @auto.doc() 279 | def get_node_history(): 280 | """ 281 | Returns the data associated with a node, and all crawler visitations on record 282 | :param node: connection string, e.g. btc:127.0.0.1:8333 - port is optional if it is the network default. 283 | :return: json dict {"node":{"user_agent":"/Satoshi/", "last_seen": ... }, "history":{"timestamp":157032190321,"height":56000, "success":1 ...}} 284 | """ 285 | 286 | node = request.form.get("node") 287 | 288 | network, address, port, resp = deconstruct_address_string(node) 289 | 290 | if network not in CONF['networks']: 291 | return json.dumps({'error': "network not recognized"}) 292 | 293 | default_port = int(CONF['networks'][network]['port']) 294 | 295 | resp = {} 296 | 297 | try: 298 | port = int(port) if port is not None else default_port 299 | except ValueError: 300 | resp['warning'] = "port not recognized, using default" 301 | port = default_port 302 | 303 | n = db.session.query(Node.network, Node.address, Node.port, Node.user_agent, Node.version, Node.first_seen, 304 | Node.last_seen, Node.last_checked, Node.country, Node.city, Node.asn, Node.aso) \ 305 | .filter(and_(Node.network == network, Node.address == address, Node.port == port)).one() 306 | 307 | q = db.session.query(NodeVisitation.timestamp, NodeVisitation.height, NodeVisitation.success) \ 308 | .join(Node, and_(Node.network == NodeVisitation.network, Node.address == NodeVisitation.address, 309 | Node.port == NodeVisitation.port)) \ 310 | .filter(and_(Node.network == network, Node.address == address, Node.port == port)) \ 311 | .order_by(NodeVisitation.timestamp.desc()) 312 | 313 | df = pd.read_sql(q.statement, q.session.bind) 314 | df['timestamp'] = df['timestamp'].astype(pd.np.int64) // 10 ** 9 315 | 316 | resp.update({"node": {"network": n.network, 'address': n.address, "port": n.port, "user_agent": n.user_agent, 317 | "version": n.version, 318 | "first_seen": n.first_seen, 319 | "last_seen": n.last_seen, 320 | "last_checked": n.last_checked, 321 | "country": n.country, "city": n.city, "asn": n.asn, "aso": n.aso}, 322 | "history": df.to_dict(orient='records')}) 323 | return to_json(resp) 324 | 325 | 326 | def geocode(result): 327 | if result and result['address'].endswith('.onion'): 328 | aso, asn, country, city = "Anonymous", "Anonymous", "Anonymous", "Anonymous" 329 | elif result: 330 | try: 331 | aso = ASN.asn(result['address']).autonomous_system_organization 332 | asn = ASN.asn(result['address']).autonomous_system_number 333 | except AddressNotFoundError: 334 | aso = None 335 | asn = None 336 | 337 | try: 338 | country = COUNTRY.country(result['address']).country.name 339 | except AddressNotFoundError: 340 | country = None 341 | 342 | try: 343 | city = CITY.city(result['address']).city.name 344 | except AddressNotFoundError: 345 | city = None 346 | else: 347 | return result 348 | 349 | result['aso'] = aso 350 | result['asn'] = asn 351 | result['country'] = country 352 | result['city'] = city 353 | return result 354 | 355 | 356 | def clean_dates(d): 357 | for i in d: 358 | if isinstance(d[i], datetime.datetime): 359 | d[i] = d[i].timestamp() 360 | if isinstance(d[i], dict): 361 | d[i] = clean_dates(d[i]) 362 | return d 363 | 364 | 365 | def to_json(d): 366 | """ 367 | Sanitizes a dictionary - converts datetime.datetime instances to timestamps 368 | :param d: dictionary 369 | :return: json string 370 | """ 371 | d = clean_dates(d) 372 | return json.dumps(d) 373 | 374 | 375 | def main(): 376 | if "--prod" in sys.argv: 377 | waitress.serve(app, host=os.environ.get("SERVER_HOST", "127.0.0.1"), port=os.environ.get("SERVER_PORT", 5000)) 378 | else: 379 | app.run("0.0.0.0", debug=True, port=5000) 380 | 381 | 382 | if __name__ == '__main__': 383 | main() 384 | -------------------------------------------------------------------------------- /static/js/crossfilter.min.js: -------------------------------------------------------------------------------- 1 | !function(r){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=r();else if("function"==typeof define&&define.amd)define([],r);else{var t;t="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:this,t.crossfilter=r()}}(function(){return function(){function r(t,e,n){function o(u,f){if(!e[u]){if(!t[u]){var a="function"==typeof require&&require;if(!f&&a)return a(u,!0);if(i)return i(u,!0);var c=new Error("Cannot find module '"+u+"'");throw c.code="MODULE_NOT_FOUND",c}var s=e[u]={exports:{}};t[u][0].call(s.exports,function(r){var e=t[u][1][r];return o(e||r)},s,s.exports,r,t,e,n)}return e[u].exports}for(var i="function"==typeof require&&require,u=0;ue)return!1;var n=t.length-1;return e==n?t.pop():pt.call(t,e,1),!0}function p(r){var t=this.__data__,e=w(t,r);return 0>e?void 0:t[e][1]}function v(r){return w(this.__data__,r)>-1}function d(r,t){var e=this.__data__,n=w(e,r);return 0>n?e.push([r,t]):e[n][1]=t,this}function y(r){var t=-1,e=r?r.length:0;for(this.clear();++te;)r[e++]=0;return r}function o(r,t){if(t>32)throw new Error("invalid array width!");return r}function i(r){this.length=r,this.subarrays=1,this.width=8,this.masks={0:0},this[0]=u(r)}if("undefined"!=typeof Uint8Array)var u=function(r){return new Uint8Array(r)},f=function(r){return new Uint16Array(r)},a=function(r){return new Uint32Array(r)},c=function(r,t){if(r.length>=t)return r;var e=new r.constructor(t);return e.set(r),e},s=function(r,t){var e;switch(t){case 16:e=f(r.length);break;case 32:e=a(r.length);break;default:throw new Error("invalid array width!")}return e.set(r),e};i.prototype.lengthen=function(r){var t,e;for(t=0,e=this.subarrays;e>t;++t)this[t]=c(this[t],r);this.length=r},i.prototype.add=function(){var r,t,e,n,o;for(n=0,o=this.subarrays;o>n;++n)if(r=this.masks[n],t=this.width-32*n,e=~r&-~r,!(t>=32)||e)return 32>t&&e&1<e;++e)this[e][r]=this[e][t]},i.prototype.truncate=function(r){var t,e;for(t=0,e=this.subarrays;e>t;++t){for(var n=this.length-1;n>=r;n--)this[t][n]=0;this[t].length=r}this.length=r},i.prototype.zero=function(r){var t,e;for(t=0,e=this.subarrays;e>t;++t)if(this[t][r])return!1;return!0},i.prototype.zeroExcept=function(r,t,e){var n,o;for(n=0,o=this.subarrays;o>n;++n)if(n===t?this[n][r]&e:this[n][r])return!1;return!0},i.prototype.zeroExceptMask=function(r,t){var e,n;for(e=0,n=this.subarrays;n>e;++e)if(this[e][r]&t[e])return!1;return!0},i.prototype.only=function(r,t,e){var n,o;for(n=0,o=this.subarrays;o>n;++n)if(this[n][r]!=(n===t?e:0))return!1;return!0},i.prototype.onlyExcept=function(r,t,e,n,o){var i,u,f;for(u=0,f=this.subarrays;f>u;++u)if(i=this[u][r],u===t&&(i&=e),i!=(u===n?o:0))return!1;return!0},t.exports={array8:e,array16:e,array32:e,arrayLengthen:n,arrayWiden:o,bitarray:i}},{}],5:[function(r,t){"use strict";function e(r){function t(t,e,n,o){for(;o>n;){var i=n+o>>>1;r(t[i])n;){var i=n+o>>>1;eu;++u)i(u)?(e.push(u),t[u]=m):t[u]=f++;F.forEach(function(r){r(-1,-1,[],e,!0)}),$.forEach(function(r){r(t)});for(var a=0,c=0;M>a;++a)t[a]!==m&&(a!==c&&(z.copy(c,a),S[c]=S[a]),++c);S.length=M=c,z.truncate(c),A("dataRemoved")}function e(r){var t,e,n,o,i=Array(z.subarrays);for(t=0;te;e++)o=r[e].id(),i[o>>7]&=~(1<<(63&o));return i}function n(r,t){var n=e(t||[]);return z.zeroExceptMask(r,n)}function d(r,t){function e(e,n,u){if(t){pt=0,A=0,ot=[];for(var a=0;ag;++g)tt(Q[g],g)||(z[G][V[g]+n]|=W,t&&(rt[g]=1));else{for(var b=0;v>b;++b)z[G][V[b]+n]|=W,t&&(rt[b]=1);for(var _=d;u>_;++_)z[G][V[_]+n]|=W,t&&(rt[_]=1)}if(!n)return H=Q,K=V,X=Y,Z=rt,lt=v,ht=d,void 0;var x,m=H,w=K,E=Z,O=0;if(a=0,t&&(x=n,n=m.length,u=pt),H=t?new Array(n+u):new Array(M),K=t?new Array(n+u):o(M,M),t&&(Z=o(n+u,1)),t){var k=X.length;X=f.arrayLengthen(X,M);for(var A=0;M>A+k;A++)X[A+k]=Y[A]}for(var j=0;n>a&&u>O;++j)m[a]a;++a,++j)H[j]=m[a],t&&(Z[j]=E[a]),K[j]=w[a];for(;u>O;++O,++j)H[j]=Q[O],t&&(Z[j]=rt[O]),K[j]=V[O]+(t?x:n);p=at(H),lt=p[0],ht=p[1]}function n(r,t,e){ct.forEach(function(r){r(Q,V,t,e)}),Q=V=null}function d(r){if(t){for(var e=0,n=0;ee;e++)r[e]!==m&&(n!==e&&(X[n]=X[e]),n++);X.length=n}for(var o,i=H.length,u=0,f=0;i>u;++u)o=K[u],r[o]!==m&&(u!==f&&(H[f]=H[u]),K[f]=r[o],t&&(Z[f]=Z[u]),++f);for(H.length=f,t&&(Z.length=f);i>f;)K[f++]=0;var a=at(H);lt=a[0],ht=a[1]}function _(r){var e=r[0],n=r[1];if(tt)return tt=null,q(function(r,t){return t>=e&&n>t},0===r[0]&&r[1]===H.length),lt=e,ht=n,it;var o,i,u,f=[],a=[],c=[],s=[];if(lt>e)for(o=e,i=Math.min(lt,n);i>o;++o)f.push(K[o]),c.push(o);else if(e>lt)for(o=lt,i=Math.min(e,ht);i>o;++o)a.push(K[o]),s.push(o);if(n>ht)for(o=Math.max(e,ht),i=n;i>o;++o)f.push(K[o]),c.push(o);else if(ht>n)for(o=Math.max(lt,n),i=ht;i>o;++o)a.push(K[o]),s.push(o);if(t){var l=[],h=[];for(o=0;on;++n)!(z[G][o=K[n]]&W)^!!(i=r(H[n],n))&&(i?u.push(o):f.push(o));if(t)for(n=0;s>n;++n)r(H[n],n)?(u.push(K[n]),a.push(n)):(f.push(K[n]),c.push(n));if(t){var l=[],h=[];for(n=0;n0&&(u=e);--i>=lt&&r>0;)z.zero(n=K[i])&&(u>0?--u:(o.push(S[n]),--r));if(t)for(i=0;i0;i++)z.zero(n=ut[i])&&(u>0?--u:(o.push(S[n]),--r));return o}function D(r,e){var n,o,i=[],u=0;if(e&&e>0&&(u=e),t)for(n=0;n0;n++)z.zero(o=ut[n])&&(u>0?--u:(i.push(S[o]),--r));for(n=lt;ht>n&&r>0;)z.zero(o=K[n])&&(u>0?--u:(i.push(S[o]),--r)),n++;return i}function I(r){function e(e,n,c,l){function h(){return t?(B++,void 0):(++B===T&&(w=f.arrayWiden(w,P<<=1),q=f.arrayWiden(q,P),T=u(P)),void 0)}t&&(L=c,c=H.length-e.length,l=e.length);var p,y,g,b,_,x,m=C,w=t?[]:o(B,T),E=U,O=D,k=I,A=B,j=0,$=0;for(X&&(E=k=s),X&&(O=k=s),C=new Array(B),B=0,q=t?A?q:[]:A>1?f.arrayLengthen(q,M):o(M,T),A&&(g=(y=m[0]).key);l>$&&!((b=r(e[$]))>=b);)++$;for(;l>$;){for(y&&b>=g?(_=y,x=g,w[j]=B,y=m[++j],y&&(g=y.key)):(_={key:b,value:k()},x=b),C[B]=_;x>=b&&(p=n[$]+(t?L:c),t?q[p]?q[p].push(B):q[p]=[B]:q[p]=B,_.value=E(_.value,S[p],!0),z.zeroExcept(p,G,J)||(_.value=O(_.value,S[p],!1)),!(++$>=l));)b=r(e[$]);h()}for(;A>j;)C[w[j]=B]=m[j++],h();if(t)for(var N=0;M>N;N++)q[N]||(q[N]=[]);if(B>j)if(t)for(j=0;L>j;++j)for(N=0;Nj;++j)q[j]=w[q[j]];p=F.indexOf(Q),B>1||t?(Q=i,V=v):(!B&&Y&&(B=1,C=[{key:null,value:k()}]),1===B?(Q=a,V=d):(Q=s,V=s),q=null),F[p]=Q}function n(r){if(B>1||t){var e,n,u,f=B,c=C,l=o(f,f);if(t){for(e=0,u=0;M>e;++e)if(r[e]!==m){for(q[u]=q[e],n=0;ne;++e)r[e]!==m&&(l[q[u]=q[e]]=1,++u);for(C=[],B=0,e=0;f>e;++e)l[e]&&(l[e]=B++,C.push(c[e]));if(B>1||t)if(t)for(e=0;u>e;++e)for(n=0;ne;++e)q[e]=l[q[e]];else q=null;F[F.indexOf(Q)]=B>1||t?(V=v,Q=i):1===B?(V=d,Q=a):V=Q=s}else if(1===B){if(Y)return;for(var h=0;M>h;++h)if(r[h]!==m)return;C=[],B=0,F[F.indexOf(Q)]=Q=V=s}}function i(r,e,n,o,i){if(!(r===W&&e===G||X)){var u,f,a,c,s;if(t){for(u=0,c=n.length;c>u;++u)if(z.zeroExcept(a=n[u],G,J))for(f=0;fu;++u)if(z.onlyExcept(a=o[u],G,J,e,r))for(f=0;fu;++u)z.zeroExcept(a=n[u],G,J)&&(s=C[q[a]],s.value=U(s.value,S[a],!1));for(u=0,c=o.length;c>u;++u)z.onlyExcept(a=o[u],G,J,e,r)&&(s=C[q[a]],s.value=D(s.value,S[a],i))}}}function a(r,t,e,n,o){if(!(r===W&&t===G||X)){var i,u,f,a=C[0];for(i=0,f=e.length;f>i;++i)z.zeroExcept(u=e[i],G,J)&&(a.value=U(a.value,S[u],!1));for(i=0,f=n.length;f>i;++i)z.onlyExcept(u=n[i],G,J,t,r)&&(a.value=D(a.value,S[u],o))}}function v(){var r,e,n;for(r=0;B>r;++r)C[r].value=I();if(t){for(r=0;M>r;++r)for(e=0;er;++r)if(!z.zeroExcept(r,G,J))for(e=0;er;++r)n=C[q[r]],n.value=U(n.value,S[r],!0);for(r=0;M>r;++r)z.zeroExcept(r,G,J)||(n=C[q[r]],n.value=D(n.value,S[r],!1))}}function d(){var r,t=C[0];for(t.value=I(),r=0;M>r;++r)t.value=U(t.value,S[r],!0);for(r=0;M>r;++r)z.zeroExcept(r,G,J)||(t.value=D(t.value,S[r],!1))}function y(){return X&&(V(),X=!1),C}function g(r){var t=N(y(),0,C.length,r);return R.sort(t,0,t.length)}function _(r,t,e){return U=r,D=t,I=e,X=!0,j}function x(){return _(b.reduceIncrement,b.reduceDecrement,l)}function w(r){return _(b.reduceAdd(r),b.reduceSubtract(r),l)}function E(r){function t(t){return r(t.value)}return N=h.by(t),R=p.by(t),j}function O(){return E(c)}function k(){return B}function A(){var r=F.indexOf(Q);return r>=0&&F.splice(r,1),r=ct.indexOf(e),r>=0&&ct.splice(r,1),r=$.indexOf(n),r>=0&&$.splice(r,1),r=st.indexOf(j),r>=0&&st.splice(r,1),j}var j={top:g,all:y,reduce:_,reduceCount:x,reduceSum:w,order:E,orderNatural:O,size:k,dispose:A,remove:A};st.push(j);var C,q,N,R,U,D,I,L,P=8,T=u(P),B=0,Q=s,V=s,X=!0,Y=r===s;return arguments.length<1&&(r=c),F.push(Q),ct.push(e),$.push(n),e(H,K,0,M),x().orderNatural()}function L(){var r=I(s),t=r.all;return delete r.all,delete r.top,delete r.order,delete r.orderNatural,delete r.size,r.value=function(){return t()[0].value},r}function P(){st.forEach(function(r){r.dispose()});var r=C.indexOf(e);return r>=0&&C.splice(r,1),r=C.indexOf(n),r>=0&&C.splice(r,1),r=$.indexOf(d),r>=0&&$.splice(r,1),z.masks[G]&=J,k()}if("string"==typeof r){var T=r;r=function(r){return x(r,T)}}var W,J,G,B,H,K,Q,V,X,Y,Z,rt,tt,et,nt,ot,it={filter:w,filterExact:E,filterRange:O,filterFunction:j,filterAll:k,currentFilter:N,hasCurrentFilter:R,top:U,bottom:D,group:I,groupAll:L,dispose:P,remove:P,accessor:r,id:function(){return B}},ut=[],ft=g.by(function(r){return Q[r]}),at=a.filterAll,ct=[],st=[],lt=0,ht=0,pt=0;C.unshift(e),C.push(n),$.push(d);var vt=z.add();return G=vt.offset,W=vt.one,J=~W,B=G<<7|Math.log(W)/Math.log(2),e(S,0,M),n(S,0,M),it}function _(){function r(r,t){var e;if(!v)for(e=t;M>e;++e)a=c(a,S[e],!0),z.zero(e)||(a=s(a,S[e],!1))}function t(r,t,e,n,o){var i,u,f;if(!v){for(i=0,f=e.length;f>i;++i)z.zero(u=e[i])&&(a=c(a,S[u],o));for(i=0,f=n.length;f>i;++i)z.only(u=n[i],t,r)&&(a=s(a,S[u],o))}}function e(){var r;for(a=h(),r=0;M>r;++r)a=c(a,S[r],!0),z.zero(r)||(a=s(a,S[r],!1))}function n(r,t,e){return c=r,s=t,h=e,v=!0,p}function o(){return n(b.reduceIncrement,b.reduceDecrement,l)}function i(r){return n(b.reduceAdd(r),b.reduceSubtract(r),l)}function u(){return v&&(e(),v=!1),a}function f(){var e=F.indexOf(t);return e>=0&&F.splice(e,1),e=C.indexOf(r),e>=0&&C.splice(e,1),p}var a,c,s,h,p={reduce:n,reduceCount:o,reduceSum:i,value:u,dispose:f,remove:f},v=!0;return F.push(t),C.push(r),r(S,0,M),o()}function w(){return M}function E(){return S}function O(r){var t=[],n=0,o=e(r||[]);for(n=0;M>n;n++)z.zeroExceptMask(n,o)&&t.push(S[n]);return t}function k(r){return"function"!=typeof r?(console.warn("onChange callback parameter must be a function!"),void 0):(q.push(r),function(){q.splice(q.indexOf(r),1)})}function A(r){for(var t=0;tt?f.array8:65537>t?f.array16:f.array32)(r)}function i(r){for(var t=o(r,r),e=-1;++e>>1)+1;--i>0;)n(r,i,o,t);return r}function e(r,t,e){for(var o,i=e-t;--i>0;)o=r[t],r[t]=r[t+i],r[t+i]=o,n(r,1,i,t);return r}function n(t,e,n,o){for(var i,u=t[--o+e],f=r(u);(i=e<<1)<=n&&(n>i&&r(t[o+i])>r(t[o+i+1])&&i++,!(f<=r(t[o+i])));)t[o+e]=t[o+i],e=i;t[o+e]=u}return t.sort=e,t}var n=r("./identity");t.exports=e(n),t.exports.by=e},{"./identity":10}],9:[function(r,t){"use strict";function e(r){function t(t,n,o,i){var u,f,a,c=new Array(i=Math.min(o-n,i));for(f=0;i>f;++f)c[f]=t[n++];if(e(c,0,i),o>n){u=r(c[0]);do r(a=t[n])>u&&(c[0]=a,u=r(e(c,0,i)[0]));while(++no;++o){for(var i=o,u=t[o],f=r(u);i>e&&r(t[i-1])>f;--i)t[i]=t[i-1];t[i]=u}return t}return t}var n=r("./identity");t.exports=e(n),t.exports.by=e},{"./identity":10}],12:[function(r,t){"use strict";function e(){return null}t.exports=e},{}],13:[function(r,t){"use strict";function e(r,t,e){for(var n=0,o=t.length,i=e?JSON.parse(JSON.stringify(r)):new Array(o);o>n;++n)i[n]=r[t[n]];return i}t.exports=e},{}],14:[function(r,t){function e(r){function t(r,t,o){return(i>o-t?n:e)(r,t,o)}function e(e,n,o){var i,u=0|(o-n)/6,f=n+u,a=o-1-u,c=n+o-1>>1,s=c-u,l=c+u,h=e[f],p=r(h),v=e[s],d=r(v),y=e[c],g=r(y),b=e[l],_=r(b),x=e[a],m=r(x);p>d&&(i=h,h=v,v=i,i=p,p=d,d=i),_>m&&(i=b,b=x,x=i,i=_,_=m,m=i),p>g&&(i=h,h=y,y=i,i=p,p=g,g=i),d>g&&(i=v,v=y,y=i,i=d,d=g,g=i),p>_&&(i=h,h=b,b=i,i=p,p=_,_=i),g>_&&(i=y,y=b,b=i,i=g,g=_,_=i),d>m&&(i=v,v=x,x=i,i=d,d=m,m=i),d>g&&(i=v,v=y,y=i,i=d,d=g,g=i),_>m&&(i=b,b=x,x=i,i=_,_=m,m=i);var w=v,E=d,O=b,k=_;e[f]=h,e[s]=e[n],e[c]=y,e[l]=e[o-1],e[a]=x;var A=n+1,z=o-2,j=k>=E&&E>=k;if(j)for(var S=A;z>=S;++S){var M=e[S],F=r(M);if(E>F)S!==A&&(e[S]=e[A],e[A]=M),++A;else if(F>E)for(;;){var C=r(e[z]);{if(!(C>E)){if(E>C){e[S]=e[A],e[A++]=e[z],e[z--]=M;break}e[S]=e[z],e[z--]=M;break}z--}}}else!function(){for(var t=A;z>=t;t++){var n=e[t],o=r(n);if(E>o)t!==A&&(e[t]=e[A],e[A]=n),++A;else if(o>k)for(;;){var i=r(e[z]);{if(!(i>k)){E>i?(e[t]=e[A],e[A++]=e[z],e[z--]=n):(e[t]=e[z],e[z--]=n);break}if(z--,t>z)break}}}}();return e[n]=e[A-1],e[A-1]=w,e[o-1]=e[z+1],e[z+1]=O,t(e,n,A-1),t(e,z+2,o),j?e:(f>A&&z>a&&!function(){for(var t,n;(t=r(e[A]))<=E&&t>=E;)++A;for(;(n=r(e[z]))<=k&&n>=k;)--z;for(var o=A;z>=o;o++){var i=e[o],u=r(i);if(E>=u&&u>=E)o!==A&&(e[o]=e[A],e[A]=i),A++;else if(k>=u&&u>=k)for(;;){n=r(e[z]);{if(!(k>=n&&n>=k)){E>n?(e[o]=e[A],e[A++]=e[z],e[z--]=i):(e[o]=e[z],e[z--]=i);break}if(z--,o>z)break}}}}(),t(e,A,z+1))}var n=o.by(r);return t}var n=r("./identity"),o=r("./insertionsort"),i=32;t.exports=e(n),t.exports.by=e},{"./identity":10,"./insertionsort":11}],15:[function(r,t){"use strict";function e(r){return r+1}function n(r){return r-1}function o(r){return function(t,e){return t+ +r(e)}}function i(r){return function(t,e){return t-r(e)}}t.exports={reduceIncrement:e,reduceDecrement:n,reduceAdd:o,reduceSubtract:i}},{}],16:[function(r,t){"use strict";function e(){return 0}t.exports=e},{}]},{},[1])(1)}); -------------------------------------------------------------------------------- /static/css/jquery-ui.min.css: -------------------------------------------------------------------------------- 1 | /*! jQuery UI - v1.11.1 - 2014-08-13 2 | * http://jqueryui.com 3 | * Includes: core.css, accordion.css, autocomplete.css, button.css, datepicker.css, dialog.css, draggable.css, menu.css, progressbar.css, resizable.css, selectable.css, selectmenu.css, slider.css, sortable.css, spinner.css, tabs.css, tooltip.css, theme.css 4 | * To view and modify this theme, visit http://jqueryui.com/themeroller/?ffDefault=Trebuchet%20MS%2CTahoma%2CVerdana%2CArial%2Csans-serif&fwDefault=bold&fsDefault=1.1em&cornerRadius=4px&bgColorHeader=f6a828&bgTextureHeader=gloss_wave&bgImgOpacityHeader=35&borderColorHeader=e78f08&fcHeader=ffffff&iconColorHeader=ffffff&bgColorContent=eeeeee&bgTextureContent=highlight_soft&bgImgOpacityContent=100&borderColorContent=dddddd&fcContent=333333&iconColorContent=222222&bgColorDefault=f6f6f6&bgTextureDefault=glass&bgImgOpacityDefault=100&borderColorDefault=cccccc&fcDefault=1c94c4&iconColorDefault=ef8c08&bgColorHover=fdf5ce&bgTextureHover=glass&bgImgOpacityHover=100&borderColorHover=fbcb09&fcHover=c77405&iconColorHover=ef8c08&bgColorActive=ffffff&bgTextureActive=glass&bgImgOpacityActive=65&borderColorActive=fbd850&fcActive=eb8f00&iconColorActive=ef8c08&bgColorHighlight=ffe45c&bgTextureHighlight=highlight_soft&bgImgOpacityHighlight=75&borderColorHighlight=fed22f&fcHighlight=363636&iconColorHighlight=228ef1&bgColorError=b81900&bgTextureError=diagonals_thick&bgImgOpacityError=18&borderColorError=cd0a0a&fcError=ffffff&iconColorError=ffd27a&bgColorOverlay=666666&bgTextureOverlay=diagonals_thick&bgImgOpacityOverlay=20&opacityOverlay=50&bgColorShadow=000000&bgTextureShadow=flat&bgImgOpacityShadow=10&opacityShadow=20&thicknessShadow=5px&offsetTopShadow=-5px&offsetLeftShadow=-5px&cornerRadiusShadow=5px 5 | * Copyright 2014 jQuery Foundation and other contributors; Licensed MIT */ 6 | 7 | .ui-helper-hidden{display:none}.ui-helper-hidden-accessible{border:0;clip:rect(0 0 0 0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.ui-helper-reset{margin:0;padding:0;border:0;outline:0;line-height:1.3;text-decoration:none;font-size:100%;list-style:none}.ui-helper-clearfix:before,.ui-helper-clearfix:after{content:"";display:table;border-collapse:collapse}.ui-helper-clearfix:after{clear:both}.ui-helper-clearfix{min-height:0}.ui-helper-zfix{width:100%;height:100%;top:0;left:0;position:absolute;opacity:0;filter:Alpha(Opacity=0)}.ui-front{z-index:100}.ui-state-disabled{cursor:default!important}.ui-icon{display:block;text-indent:-99999px;overflow:hidden;background-repeat:no-repeat}.ui-widget-overlay{position:fixed;top:0;left:0;width:100%;height:100%}.ui-accordion .ui-accordion-header{display:block;cursor:pointer;position:relative;margin:2px 0 0 0;padding:.5em .5em .5em .7em;min-height:0;font-size:100%}.ui-accordion .ui-accordion-icons{padding-left:2.2em}.ui-accordion .ui-accordion-icons .ui-accordion-icons{padding-left:2.2em}.ui-accordion .ui-accordion-header .ui-accordion-header-icon{position:absolute;left:.5em;top:50%;margin-top:-8px}.ui-accordion .ui-accordion-content{padding:1em 2.2em;border-top:0;overflow:auto}.ui-autocomplete{position:absolute;top:0;left:0;cursor:default}.ui-button{display:inline-block;position:relative;padding:0;line-height:normal;margin-right:.1em;cursor:pointer;vertical-align:middle;text-align:center;overflow:visible}.ui-button,.ui-button:link,.ui-button:visited,.ui-button:hover,.ui-button:active{text-decoration:none}.ui-button-icon-only{width:2.2em}button.ui-button-icon-only{width:2.4em}.ui-button-icons-only{width:3.4em}button.ui-button-icons-only{width:3.7em}.ui-button .ui-button-text{display:block;line-height:normal}.ui-button-text-only .ui-button-text{padding:.4em 1em}.ui-button-icon-only .ui-button-text,.ui-button-icons-only .ui-button-text{padding:.4em;text-indent:-9999999px}.ui-button-text-icon-primary .ui-button-text,.ui-button-text-icons .ui-button-text{padding:.4em 1em .4em 2.1em}.ui-button-text-icon-secondary .ui-button-text,.ui-button-text-icons .ui-button-text{padding:.4em 2.1em .4em 1em}.ui-button-text-icons .ui-button-text{padding-left:2.1em;padding-right:2.1em}input.ui-button{padding:.4em 1em}.ui-button-icon-only .ui-icon,.ui-button-text-icon-primary .ui-icon,.ui-button-text-icon-secondary .ui-icon,.ui-button-text-icons .ui-icon,.ui-button-icons-only .ui-icon{position:absolute;top:50%;margin-top:-8px}.ui-button-icon-only .ui-icon{left:50%;margin-left:-8px}.ui-button-text-icon-primary .ui-button-icon-primary,.ui-button-text-icons .ui-button-icon-primary,.ui-button-icons-only .ui-button-icon-primary{left:.5em}.ui-button-text-icon-secondary .ui-button-icon-secondary,.ui-button-text-icons .ui-button-icon-secondary,.ui-button-icons-only .ui-button-icon-secondary{right:.5em}.ui-buttonset{margin-right:7px}.ui-buttonset .ui-button{margin-left:0;margin-right:-.3em}input.ui-button::-moz-focus-inner,button.ui-button::-moz-focus-inner{border:0;padding:0}.ui-datepicker{width:17em;padding:.2em .2em 0;display:none}.ui-datepicker .ui-datepicker-header{position:relative;padding:.2em 0}.ui-datepicker .ui-datepicker-prev,.ui-datepicker .ui-datepicker-next{position:absolute;top:2px;width:1.8em;height:1.8em}.ui-datepicker .ui-datepicker-prev-hover,.ui-datepicker .ui-datepicker-next-hover{top:1px}.ui-datepicker .ui-datepicker-prev{left:2px}.ui-datepicker .ui-datepicker-next{right:2px}.ui-datepicker .ui-datepicker-prev-hover{left:1px}.ui-datepicker .ui-datepicker-next-hover{right:1px}.ui-datepicker .ui-datepicker-prev span,.ui-datepicker .ui-datepicker-next span{display:block;position:absolute;left:50%;margin-left:-8px;top:50%;margin-top:-8px}.ui-datepicker .ui-datepicker-title{margin:0 2.3em;line-height:1.8em;text-align:center}.ui-datepicker .ui-datepicker-title select{font-size:1em;margin:1px 0}.ui-datepicker select.ui-datepicker-month,.ui-datepicker select.ui-datepicker-year{width:45%}.ui-datepicker table{width:100%;font-size:.9em;border-collapse:collapse;margin:0 0 .4em}.ui-datepicker th{padding:.7em .3em;text-align:center;font-weight:bold;border:0}.ui-datepicker td{border:0;padding:1px}.ui-datepicker td span,.ui-datepicker td a{display:block;padding:.2em;text-align:right;text-decoration:none}.ui-datepicker .ui-datepicker-buttonpane{background-image:none;margin:.7em 0 0 0;padding:0 .2em;border-left:0;border-right:0;border-bottom:0}.ui-datepicker .ui-datepicker-buttonpane button{float:right;margin:.5em .2em .4em;cursor:pointer;padding:.2em .6em .3em .6em;width:auto;overflow:visible}.ui-datepicker .ui-datepicker-buttonpane button.ui-datepicker-current{float:left}.ui-datepicker.ui-datepicker-multi{width:auto}.ui-datepicker-multi .ui-datepicker-group{float:left}.ui-datepicker-multi .ui-datepicker-group table{width:95%;margin:0 auto .4em}.ui-datepicker-multi-2 .ui-datepicker-group{width:50%}.ui-datepicker-multi-3 .ui-datepicker-group{width:33.3%}.ui-datepicker-multi-4 .ui-datepicker-group{width:25%}.ui-datepicker-multi .ui-datepicker-group-last .ui-datepicker-header,.ui-datepicker-multi .ui-datepicker-group-middle .ui-datepicker-header{border-left-width:0}.ui-datepicker-multi .ui-datepicker-buttonpane{clear:left}.ui-datepicker-row-break{clear:both;width:100%;font-size:0}.ui-datepicker-rtl{direction:rtl}.ui-datepicker-rtl .ui-datepicker-prev{right:2px;left:auto}.ui-datepicker-rtl .ui-datepicker-next{left:2px;right:auto}.ui-datepicker-rtl .ui-datepicker-prev:hover{right:1px;left:auto}.ui-datepicker-rtl .ui-datepicker-next:hover{left:1px;right:auto}.ui-datepicker-rtl .ui-datepicker-buttonpane{clear:right}.ui-datepicker-rtl .ui-datepicker-buttonpane button{float:left}.ui-datepicker-rtl .ui-datepicker-buttonpane button.ui-datepicker-current,.ui-datepicker-rtl .ui-datepicker-group{float:right}.ui-datepicker-rtl .ui-datepicker-group-last .ui-datepicker-header,.ui-datepicker-rtl .ui-datepicker-group-middle .ui-datepicker-header{border-right-width:0;border-left-width:1px}.ui-dialog{overflow:hidden;position:absolute;top:0;left:0;padding:.2em;outline:0}.ui-dialog .ui-dialog-titlebar{padding:.4em 1em;position:relative}.ui-dialog .ui-dialog-title{float:left;margin:.1em 0;white-space:nowrap;width:90%;overflow:hidden;text-overflow:ellipsis}.ui-dialog .ui-dialog-titlebar-close{position:absolute;right:.3em;top:50%;width:20px;margin:-10px 0 0 0;padding:1px;height:20px}.ui-dialog .ui-dialog-content{position:relative;border:0;padding:.5em 1em;background:none;overflow:auto}.ui-dialog .ui-dialog-buttonpane{text-align:left;border-width:1px 0 0 0;background-image:none;margin-top:.5em;padding:.3em 1em .5em .4em}.ui-dialog .ui-dialog-buttonpane .ui-dialog-buttonset{float:right}.ui-dialog .ui-dialog-buttonpane button{margin:.5em .4em .5em 0;cursor:pointer}.ui-dialog .ui-resizable-se{width:12px;height:12px;right:-5px;bottom:-5px;background-position:16px 16px}.ui-draggable .ui-dialog-titlebar{cursor:move}.ui-draggable-handle{-ms-touch-action:none;touch-action:none}.ui-menu{list-style:none;padding:0;margin:0;display:block;outline:none}.ui-menu .ui-menu{position:absolute}.ui-menu .ui-menu-item{position:relative;margin:0;padding:3px 1em 3px .4em;cursor:pointer;min-height:0;list-style-image:url("data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7")}.ui-menu .ui-menu-divider{margin:5px 0;height:0;font-size:0;line-height:0;border-width:1px 0 0 0}.ui-menu .ui-state-focus,.ui-menu .ui-state-active{margin:-1px}.ui-menu-icons{position:relative}.ui-menu-icons .ui-menu-item{padding-left:2em}.ui-menu .ui-icon{position:absolute;top:0;bottom:0;left:.2em;margin:auto 0}.ui-menu .ui-menu-icon{left:auto;right:0}.ui-progressbar{height:2em;text-align:left;overflow:hidden}.ui-progressbar .ui-progressbar-value{margin:-1px;height:100%}.ui-progressbar .ui-progressbar-overlay{background:url("data:image/gif;base64,R0lGODlhKAAoAIABAAAAAP///yH/C05FVFNDQVBFMi4wAwEAAAAh+QQJAQABACwAAAAAKAAoAAACkYwNqXrdC52DS06a7MFZI+4FHBCKoDeWKXqymPqGqxvJrXZbMx7Ttc+w9XgU2FB3lOyQRWET2IFGiU9m1frDVpxZZc6bfHwv4c1YXP6k1Vdy292Fb6UkuvFtXpvWSzA+HycXJHUXiGYIiMg2R6W459gnWGfHNdjIqDWVqemH2ekpObkpOlppWUqZiqr6edqqWQAAIfkECQEAAQAsAAAAACgAKAAAApSMgZnGfaqcg1E2uuzDmmHUBR8Qil95hiPKqWn3aqtLsS18y7G1SzNeowWBENtQd+T1JktP05nzPTdJZlR6vUxNWWjV+vUWhWNkWFwxl9VpZRedYcflIOLafaa28XdsH/ynlcc1uPVDZxQIR0K25+cICCmoqCe5mGhZOfeYSUh5yJcJyrkZWWpaR8doJ2o4NYq62lAAACH5BAkBAAEALAAAAAAoACgAAAKVDI4Yy22ZnINRNqosw0Bv7i1gyHUkFj7oSaWlu3ovC8GxNso5fluz3qLVhBVeT/Lz7ZTHyxL5dDalQWPVOsQWtRnuwXaFTj9jVVh8pma9JjZ4zYSj5ZOyma7uuolffh+IR5aW97cHuBUXKGKXlKjn+DiHWMcYJah4N0lYCMlJOXipGRr5qdgoSTrqWSq6WFl2ypoaUAAAIfkECQEAAQAsAAAAACgAKAAAApaEb6HLgd/iO7FNWtcFWe+ufODGjRfoiJ2akShbueb0wtI50zm02pbvwfWEMWBQ1zKGlLIhskiEPm9R6vRXxV4ZzWT2yHOGpWMyorblKlNp8HmHEb/lCXjcW7bmtXP8Xt229OVWR1fod2eWqNfHuMjXCPkIGNileOiImVmCOEmoSfn3yXlJWmoHGhqp6ilYuWYpmTqKUgAAIfkECQEAAQAsAAAAACgAKAAAApiEH6kb58biQ3FNWtMFWW3eNVcojuFGfqnZqSebuS06w5V80/X02pKe8zFwP6EFWOT1lDFk8rGERh1TTNOocQ61Hm4Xm2VexUHpzjymViHrFbiELsefVrn6XKfnt2Q9G/+Xdie499XHd2g4h7ioOGhXGJboGAnXSBnoBwKYyfioubZJ2Hn0RuRZaflZOil56Zp6iioKSXpUAAAh+QQJAQABACwAAAAAKAAoAAACkoQRqRvnxuI7kU1a1UU5bd5tnSeOZXhmn5lWK3qNTWvRdQxP8qvaC+/yaYQzXO7BMvaUEmJRd3TsiMAgswmNYrSgZdYrTX6tSHGZO73ezuAw2uxuQ+BbeZfMxsexY35+/Qe4J1inV0g4x3WHuMhIl2jXOKT2Q+VU5fgoSUI52VfZyfkJGkha6jmY+aaYdirq+lQAACH5BAkBAAEALAAAAAAoACgAAAKWBIKpYe0L3YNKToqswUlvznigd4wiR4KhZrKt9Upqip61i9E3vMvxRdHlbEFiEXfk9YARYxOZZD6VQ2pUunBmtRXo1Lf8hMVVcNl8JafV38aM2/Fu5V16Bn63r6xt97j09+MXSFi4BniGFae3hzbH9+hYBzkpuUh5aZmHuanZOZgIuvbGiNeomCnaxxap2upaCZsq+1kAACH5BAkBAAEALAAAAAAoACgAAAKXjI8By5zf4kOxTVrXNVlv1X0d8IGZGKLnNpYtm8Lr9cqVeuOSvfOW79D9aDHizNhDJidFZhNydEahOaDH6nomtJjp1tutKoNWkvA6JqfRVLHU/QUfau9l2x7G54d1fl995xcIGAdXqMfBNadoYrhH+Mg2KBlpVpbluCiXmMnZ2Sh4GBqJ+ckIOqqJ6LmKSllZmsoq6wpQAAAh+QQJAQABACwAAAAAKAAoAAAClYx/oLvoxuJDkU1a1YUZbJ59nSd2ZXhWqbRa2/gF8Gu2DY3iqs7yrq+xBYEkYvFSM8aSSObE+ZgRl1BHFZNr7pRCavZ5BW2142hY3AN/zWtsmf12p9XxxFl2lpLn1rseztfXZjdIWIf2s5dItwjYKBgo9yg5pHgzJXTEeGlZuenpyPmpGQoKOWkYmSpaSnqKileI2FAAACH5BAkBAAEALAAAAAAoACgAAAKVjB+gu+jG4kORTVrVhRlsnn2dJ3ZleFaptFrb+CXmO9OozeL5VfP99HvAWhpiUdcwkpBH3825AwYdU8xTqlLGhtCosArKMpvfa1mMRae9VvWZfeB2XfPkeLmm18lUcBj+p5dnN8jXZ3YIGEhYuOUn45aoCDkp16hl5IjYJvjWKcnoGQpqyPlpOhr3aElaqrq56Bq7VAAAOw==");height:100%;filter:alpha(opacity=25);opacity:0.25}.ui-progressbar-indeterminate .ui-progressbar-value{background-image:none}.ui-resizable{position:relative}.ui-resizable-handle{position:absolute;font-size:0.1px;display:block;-ms-touch-action:none;touch-action:none}.ui-resizable-disabled .ui-resizable-handle,.ui-resizable-autohide .ui-resizable-handle{display:none}.ui-resizable-n{cursor:n-resize;height:7px;width:100%;top:-5px;left:0}.ui-resizable-s{cursor:s-resize;height:7px;width:100%;bottom:-5px;left:0}.ui-resizable-e{cursor:e-resize;width:7px;right:-5px;top:0;height:100%}.ui-resizable-w{cursor:w-resize;width:7px;left:-5px;top:0;height:100%}.ui-resizable-se{cursor:se-resize;width:12px;height:12px;right:1px;bottom:1px}.ui-resizable-sw{cursor:sw-resize;width:9px;height:9px;left:-5px;bottom:-5px}.ui-resizable-nw{cursor:nw-resize;width:9px;height:9px;left:-5px;top:-5px}.ui-resizable-ne{cursor:ne-resize;width:9px;height:9px;right:-5px;top:-5px}.ui-selectable{-ms-touch-action:none;touch-action:none}.ui-selectable-helper{position:absolute;z-index:100;border:1px dotted black}.ui-selectmenu-menu{padding:0;margin:0;position:absolute;top:0;left:0;display:none}.ui-selectmenu-menu .ui-menu{overflow:auto;overflow-x:hidden;padding-bottom:1px}.ui-selectmenu-menu .ui-menu .ui-selectmenu-optgroup{font-size:1em;font-weight:bold;line-height:1.5;padding:2px 0.4em;margin:0.5em 0 0 0;height:auto;border:0}.ui-selectmenu-open{display:block}.ui-selectmenu-button{display:inline-block;overflow:hidden;position:relative;text-decoration:none;cursor:pointer}.ui-selectmenu-button span.ui-icon{right:0.5em;left:auto;margin-top:-8px;position:absolute;top:50%}.ui-selectmenu-button span.ui-selectmenu-text{text-align:left;padding:0.4em 2.1em 0.4em 1em;display:block;line-height:1.4;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.ui-slider{position:relative;text-align:left}.ui-slider .ui-slider-handle{position:absolute;z-index:2;width:1.2em;height:1.2em;cursor:default;-ms-touch-action:none;touch-action:none}.ui-slider .ui-slider-range{position:absolute;z-index:1;font-size:.7em;display:block;border:0;background-position:0 0}.ui-slider.ui-state-disabled .ui-slider-handle,.ui-slider.ui-state-disabled .ui-slider-range{filter:inherit}.ui-slider-horizontal{height:.8em}.ui-slider-horizontal .ui-slider-handle{top:-.3em;margin-left:-.6em}.ui-slider-horizontal .ui-slider-range{top:0;height:100%}.ui-slider-horizontal .ui-slider-range-min{left:0}.ui-slider-horizontal .ui-slider-range-max{right:0}.ui-slider-vertical{width:.8em;height:100px}.ui-slider-vertical .ui-slider-handle{left:-.3em;margin-left:0;margin-bottom:-.6em}.ui-slider-vertical .ui-slider-range{left:0;width:100%}.ui-slider-vertical .ui-slider-range-min{bottom:0}.ui-slider-vertical .ui-slider-range-max{top:0}.ui-sortable-handle{-ms-touch-action:none;touch-action:none}.ui-spinner{position:relative;display:inline-block;overflow:hidden;padding:0;vertical-align:middle}.ui-spinner-input{border:none;background:none;color:inherit;padding:0;margin:.2em 0;vertical-align:middle;margin-left:.4em;margin-right:22px}.ui-spinner-button{width:16px;height:50%;font-size:.5em;padding:0;margin:0;text-align:center;position:absolute;cursor:default;display:block;overflow:hidden;right:0}.ui-spinner a.ui-spinner-button{border-top:none;border-bottom:none;border-right:none}.ui-spinner .ui-icon{position:absolute;margin-top:-8px;top:50%;left:0}.ui-spinner-up{top:0}.ui-spinner-down{bottom:0}.ui-spinner .ui-icon-triangle-1-s{background-position:-65px -16px}.ui-tabs{position:relative;padding:.2em}.ui-tabs .ui-tabs-nav{margin:0;padding:.2em .2em 0}.ui-tabs .ui-tabs-nav li{list-style:none;float:left;position:relative;top:0;margin:1px .2em 0 0;border-bottom-width:0;padding:0;white-space:nowrap}.ui-tabs .ui-tabs-nav .ui-tabs-anchor{float:left;padding:.5em 1em;text-decoration:none}.ui-tabs .ui-tabs-nav li.ui-tabs-active{margin-bottom:-1px;padding-bottom:1px}.ui-tabs .ui-tabs-nav li.ui-tabs-active .ui-tabs-anchor,.ui-tabs .ui-tabs-nav li.ui-state-disabled .ui-tabs-anchor,.ui-tabs .ui-tabs-nav li.ui-tabs-loading .ui-tabs-anchor{cursor:text}.ui-tabs-collapsible .ui-tabs-nav li.ui-tabs-active .ui-tabs-anchor{cursor:pointer}.ui-tabs .ui-tabs-panel{display:block;border-width:0;padding:1em 1.4em;background:none}.ui-tooltip{padding:8px;position:absolute;z-index:9999;max-width:300px;-webkit-box-shadow:0 0 5px #aaa;box-shadow:0 0 5px #aaa}body .ui-tooltip{border-width:2px}.ui-widget{font-family:Trebuchet MS,Tahoma,Verdana,Arial,sans-serif;font-size:1.1em}.ui-widget .ui-widget{font-size:1em}.ui-widget input,.ui-widget select,.ui-widget textarea,.ui-widget button{font-family:Trebuchet MS,Tahoma,Verdana,Arial,sans-serif;font-size:1em}.ui-widget-content{border:1px solid #ddd;background:#eee url("images/ui-bg_highlight-soft_100_eeeeee_1x100.png") 50% top repeat-x;color:#333}.ui-widget-content a{color:#333}.ui-widget-header{border:1px solid #e78f08;background:#f6a828 url("images/ui-bg_gloss-wave_35_f6a828_500x100.png") 50% 50% repeat-x;color:#fff;font-weight:bold}.ui-widget-header a{color:#fff}.ui-state-default,.ui-widget-content .ui-state-default,.ui-widget-header .ui-state-default{border:1px solid #ccc;background:#f6f6f6 url("images/ui-bg_glass_100_f6f6f6_1x400.png") 50% 50% repeat-x;font-weight:bold;color:#1c94c4}.ui-state-default a,.ui-state-default a:link,.ui-state-default a:visited{color:#1c94c4;text-decoration:none}.ui-state-hover,.ui-widget-content .ui-state-hover,.ui-widget-header .ui-state-hover,.ui-state-focus,.ui-widget-content .ui-state-focus,.ui-widget-header .ui-state-focus{border:1px solid #fbcb09;background:#fdf5ce url("images/ui-bg_glass_100_fdf5ce_1x400.png") 50% 50% repeat-x;font-weight:bold;color:#c77405}.ui-state-hover a,.ui-state-hover a:hover,.ui-state-hover a:link,.ui-state-hover a:visited,.ui-state-focus a,.ui-state-focus a:hover,.ui-state-focus a:link,.ui-state-focus a:visited{color:#c77405;text-decoration:none}.ui-state-active,.ui-widget-content .ui-state-active,.ui-widget-header .ui-state-active{border:1px solid #fbd850;background:#fff url("images/ui-bg_glass_65_ffffff_1x400.png") 50% 50% repeat-x;font-weight:bold;color:#eb8f00}.ui-state-active a,.ui-state-active a:link,.ui-state-active a:visited{color:#eb8f00;text-decoration:none}.ui-state-highlight,.ui-widget-content .ui-state-highlight,.ui-widget-header .ui-state-highlight{border:1px solid #fed22f;background:#ffe45c url("images/ui-bg_highlight-soft_75_ffe45c_1x100.png") 50% top repeat-x;color:#363636}.ui-state-highlight a,.ui-widget-content .ui-state-highlight a,.ui-widget-header .ui-state-highlight a{color:#363636}.ui-state-error,.ui-widget-content .ui-state-error,.ui-widget-header .ui-state-error{border:1px solid #cd0a0a;background:#b81900 url("images/ui-bg_diagonals-thick_18_b81900_40x40.png") 50% 50% repeat;color:#fff}.ui-state-error a,.ui-widget-content .ui-state-error a,.ui-widget-header .ui-state-error a{color:#fff}.ui-state-error-text,.ui-widget-content .ui-state-error-text,.ui-widget-header .ui-state-error-text{color:#fff}.ui-priority-primary,.ui-widget-content .ui-priority-primary,.ui-widget-header .ui-priority-primary{font-weight:bold}.ui-priority-secondary,.ui-widget-content .ui-priority-secondary,.ui-widget-header .ui-priority-secondary{opacity:.7;filter:Alpha(Opacity=70);font-weight:normal}.ui-state-disabled,.ui-widget-content .ui-state-disabled,.ui-widget-header .ui-state-disabled{opacity:.35;filter:Alpha(Opacity=35);background-image:none}.ui-state-disabled .ui-icon{filter:Alpha(Opacity=35)}.ui-icon{width:16px;height:16px}.ui-icon,.ui-widget-content .ui-icon{background-image:url("images/ui-icons_222222_256x240.png")}.ui-widget-header .ui-icon{background-image:url("images/ui-icons_ffffff_256x240.png")}.ui-state-default .ui-icon{background-image:url("images/ui-icons_ef8c08_256x240.png")}.ui-state-hover .ui-icon,.ui-state-focus .ui-icon{background-image:url("images/ui-icons_ef8c08_256x240.png")}.ui-state-active .ui-icon{background-image:url("images/ui-icons_ef8c08_256x240.png")}.ui-state-highlight .ui-icon{background-image:url("images/ui-icons_228ef1_256x240.png")}.ui-state-error .ui-icon,.ui-state-error-text .ui-icon{background-image:url("images/ui-icons_ffd27a_256x240.png")}.ui-icon-blank{background-position:16px 16px}.ui-icon-carat-1-n{background-position:0 0}.ui-icon-carat-1-ne{background-position:-16px 0}.ui-icon-carat-1-e{background-position:-32px 0}.ui-icon-carat-1-se{background-position:-48px 0}.ui-icon-carat-1-s{background-position:-64px 0}.ui-icon-carat-1-sw{background-position:-80px 0}.ui-icon-carat-1-w{background-position:-96px 0}.ui-icon-carat-1-nw{background-position:-112px 0}.ui-icon-carat-2-n-s{background-position:-128px 0}.ui-icon-carat-2-e-w{background-position:-144px 0}.ui-icon-triangle-1-n{background-position:0 -16px}.ui-icon-triangle-1-ne{background-position:-16px -16px}.ui-icon-triangle-1-e{background-position:-32px -16px}.ui-icon-triangle-1-se{background-position:-48px -16px}.ui-icon-triangle-1-s{background-position:-64px -16px}.ui-icon-triangle-1-sw{background-position:-80px -16px}.ui-icon-triangle-1-w{background-position:-96px -16px}.ui-icon-triangle-1-nw{background-position:-112px -16px}.ui-icon-triangle-2-n-s{background-position:-128px -16px}.ui-icon-triangle-2-e-w{background-position:-144px -16px}.ui-icon-arrow-1-n{background-position:0 -32px}.ui-icon-arrow-1-ne{background-position:-16px -32px}.ui-icon-arrow-1-e{background-position:-32px -32px}.ui-icon-arrow-1-se{background-position:-48px -32px}.ui-icon-arrow-1-s{background-position:-64px -32px}.ui-icon-arrow-1-sw{background-position:-80px -32px}.ui-icon-arrow-1-w{background-position:-96px -32px}.ui-icon-arrow-1-nw{background-position:-112px -32px}.ui-icon-arrow-2-n-s{background-position:-128px -32px}.ui-icon-arrow-2-ne-sw{background-position:-144px -32px}.ui-icon-arrow-2-e-w{background-position:-160px -32px}.ui-icon-arrow-2-se-nw{background-position:-176px -32px}.ui-icon-arrowstop-1-n{background-position:-192px -32px}.ui-icon-arrowstop-1-e{background-position:-208px -32px}.ui-icon-arrowstop-1-s{background-position:-224px -32px}.ui-icon-arrowstop-1-w{background-position:-240px -32px}.ui-icon-arrowthick-1-n{background-position:0 -48px}.ui-icon-arrowthick-1-ne{background-position:-16px -48px}.ui-icon-arrowthick-1-e{background-position:-32px -48px}.ui-icon-arrowthick-1-se{background-position:-48px -48px}.ui-icon-arrowthick-1-s{background-position:-64px -48px}.ui-icon-arrowthick-1-sw{background-position:-80px -48px}.ui-icon-arrowthick-1-w{background-position:-96px -48px}.ui-icon-arrowthick-1-nw{background-position:-112px -48px}.ui-icon-arrowthick-2-n-s{background-position:-128px -48px}.ui-icon-arrowthick-2-ne-sw{background-position:-144px -48px}.ui-icon-arrowthick-2-e-w{background-position:-160px -48px}.ui-icon-arrowthick-2-se-nw{background-position:-176px -48px}.ui-icon-arrowthickstop-1-n{background-position:-192px -48px}.ui-icon-arrowthickstop-1-e{background-position:-208px -48px}.ui-icon-arrowthickstop-1-s{background-position:-224px -48px}.ui-icon-arrowthickstop-1-w{background-position:-240px -48px}.ui-icon-arrowreturnthick-1-w{background-position:0 -64px}.ui-icon-arrowreturnthick-1-n{background-position:-16px -64px}.ui-icon-arrowreturnthick-1-e{background-position:-32px -64px}.ui-icon-arrowreturnthick-1-s{background-position:-48px -64px}.ui-icon-arrowreturn-1-w{background-position:-64px -64px}.ui-icon-arrowreturn-1-n{background-position:-80px -64px}.ui-icon-arrowreturn-1-e{background-position:-96px -64px}.ui-icon-arrowreturn-1-s{background-position:-112px -64px}.ui-icon-arrowrefresh-1-w{background-position:-128px -64px}.ui-icon-arrowrefresh-1-n{background-position:-144px -64px}.ui-icon-arrowrefresh-1-e{background-position:-160px -64px}.ui-icon-arrowrefresh-1-s{background-position:-176px -64px}.ui-icon-arrow-4{background-position:0 -80px}.ui-icon-arrow-4-diag{background-position:-16px -80px}.ui-icon-extlink{background-position:-32px -80px}.ui-icon-newwin{background-position:-48px -80px}.ui-icon-refresh{background-position:-64px -80px}.ui-icon-shuffle{background-position:-80px -80px}.ui-icon-transfer-e-w{background-position:-96px -80px}.ui-icon-transferthick-e-w{background-position:-112px -80px}.ui-icon-folder-collapsed{background-position:0 -96px}.ui-icon-folder-open{background-position:-16px -96px}.ui-icon-document{background-position:-32px -96px}.ui-icon-document-b{background-position:-48px -96px}.ui-icon-note{background-position:-64px -96px}.ui-icon-mail-closed{background-position:-80px -96px}.ui-icon-mail-open{background-position:-96px -96px}.ui-icon-suitcase{background-position:-112px -96px}.ui-icon-comment{background-position:-128px -96px}.ui-icon-person{background-position:-144px -96px}.ui-icon-print{background-position:-160px -96px}.ui-icon-trash{background-position:-176px -96px}.ui-icon-locked{background-position:-192px -96px}.ui-icon-unlocked{background-position:-208px -96px}.ui-icon-bookmark{background-position:-224px -96px}.ui-icon-tag{background-position:-240px -96px}.ui-icon-home{background-position:0 -112px}.ui-icon-flag{background-position:-16px -112px}.ui-icon-calendar{background-position:-32px -112px}.ui-icon-cart{background-position:-48px -112px}.ui-icon-pencil{background-position:-64px -112px}.ui-icon-clock{background-position:-80px -112px}.ui-icon-disk{background-position:-96px -112px}.ui-icon-calculator{background-position:-112px -112px}.ui-icon-zoomin{background-position:-128px -112px}.ui-icon-zoomout{background-position:-144px -112px}.ui-icon-search{background-position:-160px -112px}.ui-icon-wrench{background-position:-176px -112px}.ui-icon-gear{background-position:-192px -112px}.ui-icon-heart{background-position:-208px -112px}.ui-icon-star{background-position:-224px -112px}.ui-icon-link{background-position:-240px -112px}.ui-icon-cancel{background-position:0 -128px}.ui-icon-plus{background-position:-16px -128px}.ui-icon-plusthick{background-position:-32px -128px}.ui-icon-minus{background-position:-48px -128px}.ui-icon-minusthick{background-position:-64px -128px}.ui-icon-close{background-position:-80px -128px}.ui-icon-closethick{background-position:-96px -128px}.ui-icon-key{background-position:-112px -128px}.ui-icon-lightbulb{background-position:-128px -128px}.ui-icon-scissors{background-position:-144px -128px}.ui-icon-clipboard{background-position:-160px -128px}.ui-icon-copy{background-position:-176px -128px}.ui-icon-contact{background-position:-192px -128px}.ui-icon-image{background-position:-208px -128px}.ui-icon-video{background-position:-224px -128px}.ui-icon-script{background-position:-240px -128px}.ui-icon-alert{background-position:0 -144px}.ui-icon-info{background-position:-16px -144px}.ui-icon-notice{background-position:-32px -144px}.ui-icon-help{background-position:-48px -144px}.ui-icon-check{background-position:-64px -144px}.ui-icon-bullet{background-position:-80px -144px}.ui-icon-radio-on{background-position:-96px -144px}.ui-icon-radio-off{background-position:-112px -144px}.ui-icon-pin-w{background-position:-128px -144px}.ui-icon-pin-s{background-position:-144px -144px}.ui-icon-play{background-position:0 -160px}.ui-icon-pause{background-position:-16px -160px}.ui-icon-seek-next{background-position:-32px -160px}.ui-icon-seek-prev{background-position:-48px -160px}.ui-icon-seek-end{background-position:-64px -160px}.ui-icon-seek-start{background-position:-80px -160px}.ui-icon-seek-first{background-position:-80px -160px}.ui-icon-stop{background-position:-96px -160px}.ui-icon-eject{background-position:-112px -160px}.ui-icon-volume-off{background-position:-128px -160px}.ui-icon-volume-on{background-position:-144px -160px}.ui-icon-power{background-position:0 -176px}.ui-icon-signal-diag{background-position:-16px -176px}.ui-icon-signal{background-position:-32px -176px}.ui-icon-battery-0{background-position:-48px -176px}.ui-icon-battery-1{background-position:-64px -176px}.ui-icon-battery-2{background-position:-80px -176px}.ui-icon-battery-3{background-position:-96px -176px}.ui-icon-circle-plus{background-position:0 -192px}.ui-icon-circle-minus{background-position:-16px -192px}.ui-icon-circle-close{background-position:-32px -192px}.ui-icon-circle-triangle-e{background-position:-48px -192px}.ui-icon-circle-triangle-s{background-position:-64px -192px}.ui-icon-circle-triangle-w{background-position:-80px -192px}.ui-icon-circle-triangle-n{background-position:-96px -192px}.ui-icon-circle-arrow-e{background-position:-112px -192px}.ui-icon-circle-arrow-s{background-position:-128px -192px}.ui-icon-circle-arrow-w{background-position:-144px -192px}.ui-icon-circle-arrow-n{background-position:-160px -192px}.ui-icon-circle-zoomin{background-position:-176px -192px}.ui-icon-circle-zoomout{background-position:-192px -192px}.ui-icon-circle-check{background-position:-208px -192px}.ui-icon-circlesmall-plus{background-position:0 -208px}.ui-icon-circlesmall-minus{background-position:-16px -208px}.ui-icon-circlesmall-close{background-position:-32px -208px}.ui-icon-squaresmall-plus{background-position:-48px -208px}.ui-icon-squaresmall-minus{background-position:-64px -208px}.ui-icon-squaresmall-close{background-position:-80px -208px}.ui-icon-grip-dotted-vertical{background-position:0 -224px}.ui-icon-grip-dotted-horizontal{background-position:-16px -224px}.ui-icon-grip-solid-vertical{background-position:-32px -224px}.ui-icon-grip-solid-horizontal{background-position:-48px -224px}.ui-icon-gripsmall-diagonal-se{background-position:-64px -224px}.ui-icon-grip-diagonal-se{background-position:-80px -224px}.ui-corner-all,.ui-corner-top,.ui-corner-left,.ui-corner-tl{border-top-left-radius:4px}.ui-corner-all,.ui-corner-top,.ui-corner-right,.ui-corner-tr{border-top-right-radius:4px}.ui-corner-all,.ui-corner-bottom,.ui-corner-left,.ui-corner-bl{border-bottom-left-radius:4px}.ui-corner-all,.ui-corner-bottom,.ui-corner-right,.ui-corner-br{border-bottom-right-radius:4px}.ui-widget-overlay{background:#666 url("images/ui-bg_diagonals-thick_20_666666_40x40.png") 50% 50% repeat;opacity:.5;filter:Alpha(Opacity=50)}.ui-widget-shadow{margin:-5px 0 0 -5px;padding:5px;background:#000 url("images/ui-bg_flat_10_000000_40x100.png") 50% 50% repeat-x;opacity:.2;filter:Alpha(Opacity=20);border-radius:5px} -------------------------------------------------------------------------------- /crawler.py: -------------------------------------------------------------------------------- 1 | """ 2 | Open Nodes Crawler 3 | Copyright (c) 2018 Opennodes / Blake Bjorn Anderson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | """ 24 | 25 | import datetime 26 | import json 27 | import logging 28 | import os 29 | import re 30 | import socket 31 | import sys 32 | import time 33 | from concurrent.futures import ThreadPoolExecutor 34 | 35 | import numpy as np 36 | import pandas as pd 37 | import requests 38 | from geoip2.database import Reader 39 | from geoip2.errors import AddressNotFoundError 40 | from sqlalchemy import and_, or_, func, not_, case 41 | 42 | from config import load_config 43 | from models import Node, NodeVisitation, CrawlSummary, UserAgent, session 44 | from protocol import ProtocolError, Connection, ConnectionError, Keepalive 45 | 46 | logging.basicConfig(level=logging.INFO) 47 | 48 | CONF = load_config() 49 | ASN = Reader("geoip/GeoLite2-ASN.mmdb") 50 | COUNTRY = Reader("geoip/GeoLite2-Country.mmdb") 51 | CITY = Reader("geoip/GeoLite2-City.mmdb") 52 | RENAMED_COUNTRIES = {"South Korea": "Republic of Korea"} 53 | USER_AGENTS = {} 54 | 55 | 56 | def get_user_agent_id(user_agent): 57 | user_agent = str(user_agent) 58 | if len(user_agent) > 60: 59 | user_agent = user_agent[:60] 60 | if user_agent not in USER_AGENTS: 61 | u = session.query(UserAgent).filter(UserAgent.user_agent == user_agent).first() 62 | if u is None: 63 | u = UserAgent(user_agent=user_agent) 64 | session.add(u) 65 | session.flush() 66 | logging.info(f"New User Agent > {u.id} {u.user_agent}") 67 | USER_AGENTS[str(user_agent)] = int(u.id) 68 | return USER_AGENTS[user_agent] 69 | 70 | 71 | def connect(network, address, port, to_services, network_data, user_agent=None, explicit_p2p=False, p2p_nodes=True, 72 | from_services=None, keepalive=False, attempt=1): 73 | results = {'network': network, 'address': address, 'port': port, 74 | 'timestamp': datetime.datetime.utcnow(), 'seen': 0, 'attempt': attempt} 75 | 76 | try: 77 | handshake_msgs = [] 78 | new_addrs = [] 79 | 80 | proxy = CONF['tor_proxy'] if address.endswith(".onion") else None 81 | 82 | conn = Connection((address, port), 83 | (CONF['source_address'], 0), 84 | magic_number=network_data['magic_number'], 85 | socket_timeout=CONF['socket_timeout'], 86 | proxy=proxy, 87 | protocol_version=int(network_data['protocol_version']), 88 | min_protocol_version=network_data['min_protocol_version'], 89 | to_services=int(to_services), 90 | from_services=int(from_services or network_data['services']), 91 | user_agent=user_agent or CONF['user_agent'], 92 | height=int(network_data['height']), 93 | relay=CONF['relay']) 94 | 95 | try: 96 | conn.open() 97 | except (ProtocolError, ConnectionError, socket.error) as err: 98 | results['error'] = str(err) 99 | logging.debug("connection failed %s %s", type(err), err) 100 | else: 101 | try: 102 | handshake_msgs = conn.handshake() 103 | assert handshake_msgs 104 | results['seen'] = 1 105 | results['height'] = int(handshake_msgs[0]['height']) 106 | results['version'] = int(handshake_msgs[0]['version']) 107 | results['user_agent'] = handshake_msgs[0]['user_agent'].decode() 108 | results['services'] = int(handshake_msgs[0]['services']) 109 | except (ProtocolError, ConnectionError, socket.error, AssertionError) as err: 110 | results['error'] = str(err) 111 | logging.debug("handshake failed %s", err) 112 | 113 | msgs = [] 114 | if len(handshake_msgs) > 0 and (p2p_nodes or explicit_p2p): 115 | getaddr = True 116 | chance = CONF['getaddr_prop'] 117 | if chance < 1.0 and p2p_nodes and not explicit_p2p and "--seed" not in sys.argv: 118 | if np.random.rand() > chance: 119 | getaddr = False 120 | 121 | if getaddr: 122 | try: 123 | conn.getaddr(block=False) 124 | msgs = msgs + conn.get_messages(commands=[b"addr"]) 125 | time.sleep(5) 126 | msgs = msgs + conn.get_messages(commands=[b"addr"]) 127 | except (ProtocolError, ConnectionError, socket.error) as err: 128 | logging.debug("getaddr failed %s", err) 129 | if keepalive: 130 | Keepalive(conn, 10).keepalive(addr=True if p2p_nodes else False) 131 | for msg in msgs: 132 | if msg['count'] > 1: 133 | ts = results['timestamp'].timestamp() 134 | for addr in msg['addr_list']: 135 | if ts - addr['timestamp'] < 12 * 60 * 60: # within 6 hours 136 | new_addrs.append(addr) 137 | conn.close() 138 | return results, new_addrs 139 | except Exception as err: 140 | logging.warning("unspecified connection error: %s", err) 141 | return {}, [] 142 | 143 | 144 | def get_seeds(port, dns_seeds, address_seeds, default_services=0): 145 | """ 146 | Initializes a list of reachable nodes from DNS seeders and hardcoded nodes to bootstrap the crawler. 147 | """ 148 | export_list = [] 149 | for seeder in dns_seeds: 150 | nodes = [] 151 | 152 | try: 153 | ipv4_nodes = socket.getaddrinfo(seeder, None, socket.AF_INET) 154 | except socket.gaierror: 155 | if CONF['ipv6']: 156 | try: 157 | ipv6_nodes = socket.getaddrinfo(seeder, None, socket.AF_INET6) 158 | except socket.gaierror as err: 159 | logging.warning("%s %s", seeder, err) 160 | else: 161 | nodes.extend(ipv6_nodes) 162 | else: 163 | nodes.extend(ipv4_nodes) 164 | 165 | for node in nodes: 166 | address = node[-1][0] 167 | export_list.append((address, port, default_services)) 168 | 169 | for address in address_seeds: 170 | export_list.append((address, port, default_services)) 171 | 172 | return export_list 173 | 174 | 175 | def init_crawler(networks): 176 | # Populates list of all known node addresses and block heights 177 | db_networks = [x[0] for x in session.query(Node.network).distinct().all()] 178 | node_addresses = {} 179 | recent_heights = {} 180 | for network in set(db_networks + networks): 181 | node_addresses[network] = {f"{y.address};{y.port}" for y in 182 | session.query(Node.address, Node.port).filter(Node.network == network).all()} 183 | count = session.query(Node).filter(and_(Node.network == network, Node.last_height != None)).count() 184 | if count > 0: 185 | median = session.query(Node.last_height).filter(and_(Node.network == network, Node.last_height != None)) \ 186 | .order_by(Node.last_height).limit(1).offset(count // 2).one()[0] 187 | if median: 188 | recent_heights[network] = [median] 189 | if network not in recent_heights: 190 | recent_heights[network] = [500000] 191 | return node_addresses, recent_heights 192 | 193 | 194 | def check_dns(network_data, node_addresses): 195 | nodes = [] 196 | for network in network_data: 197 | nc = network_data[network] 198 | dns_node_addrs = get_seeds(nc['port'], nc['dns_seeds'], nc['address_seeds'], default_services=nc['services']) 199 | for nodeAddr in dns_node_addrs: 200 | if nodeAddr[0] and nodeAddr[1]: 201 | if not f"{nodeAddr[0]};{nodeAddr[1]}" in node_addresses[network]: 202 | node_addresses[network].add(f"{nodeAddr[0]};{nodeAddr[1]}") 203 | new_node = Node(network=network, address=nodeAddr[0], 204 | port=int(nodeAddr[1]), services=int(nodeAddr[2])) 205 | nodes.append(new_node) 206 | return nodes 207 | 208 | 209 | def prune_nodes(): 210 | # prune old nodes that can't be reached 211 | pruned = session.query(Node) \ 212 | .filter( 213 | and_( 214 | Node.last_seen == None, Node.first_checked != None, 215 | Node.first_checked <= datetime.datetime.utcnow() - datetime.timedelta(days=CONF['min_pruning_age']) 216 | )).delete() 217 | 218 | if pruned > 0: 219 | logging.info(f"Pruned {pruned} nodes") 220 | 221 | # prune visitations that no longer have a parent node 222 | if CONF['prune_visitations']: 223 | deleted = session.query(NodeVisitation) \ 224 | .outerjoin(Node, Node.id == NodeVisitation.parent_id) \ 225 | .filter(Node.address == None).delete(synchronize_session=False) 226 | logging.info(f"{deleted} Visitations deleted") 227 | 228 | session.commit() 229 | 230 | 231 | def calculate_pending_nodes(start_time): 232 | now = datetime.datetime.utcnow() 233 | # Get a list of all never checked nodes, and nodes that have been checked recently: 234 | q = session.query(Node) 235 | q = q.filter(or_( 236 | Node.first_checked == None, 237 | Node.last_checked == None, 238 | # Assume 30m interval 239 | # If it hasn't been seen before, check every 6h 240 | and_(Node.last_seen == None, Node.last_checked != None, 241 | Node.last_checked < now - datetime.timedelta(minutes=CONF['crawl_interval'] * 12)), 242 | # If it has been seen in the last 6 hours, check it every 30 minutes 243 | and_(Node.last_seen != None, Node.last_seen > now - datetime.timedelta(hours=6), 244 | Node.last_checked < now - datetime.timedelta(minutes=CONF['crawl_interval'])), 245 | # If it has been seen in the last 2 weeks, check it every 12 hours 246 | and_(Node.last_seen != None, Node.last_seen > now - datetime.timedelta(hours=24 * 14), 247 | Node.last_checked < now - datetime.timedelta(minutes=CONF['crawl_interval'] * 24)), 248 | # Otherwise every day 249 | and_(Node.last_seen != None, 250 | Node.last_checked < now - datetime.timedelta(minutes=CONF['crawl_interval'] * 48)) 251 | )).filter(not_(and_(Node.last_checked != None, Node.last_checked > start_time))) 252 | 253 | if CONF['crawl_order']: 254 | case_order = [] 255 | for i in range(len(CONF['crawl_order'])): 256 | case_order.append((Node.network == CONF['crawl_order'][i], str(i))) 257 | q = q.order_by(case(case_order, else_=Node.network), Node.seen.desc(), Node.last_checked) 258 | else: 259 | q = q.order_by(Node.seen.desc(), Node.last_checked) 260 | 261 | if CONF['max_queue'] > 0: 262 | count = q.count() 263 | q = q.limit(CONF['max_queue']) 264 | else: 265 | count = q.count() 266 | if count > CONF['max_queue']: 267 | logging.info(f"{count} nodes pending") 268 | 269 | if CONF['database_concurrency']: 270 | nodes = q.with_for_update().all() 271 | session.bulk_update_mappings(Node, [ 272 | {'id': x.id, 'last_checked': now} for x in nodes]) 273 | session.commit() 274 | return nodes 275 | 276 | return q.all() 277 | 278 | 279 | def process_pending_nodes(node_addresses, node_processing_queue, recent_heights, thread_pool, mnodes=None): 280 | futures_dict = {} 281 | 282 | checked_nodes = 0 283 | seen_nodes = 0 284 | pending_nodes = 0 285 | skipped_nodes = 0 286 | retried_nodes = 0 287 | found_on_retry = 0 288 | new_nodes_to_add = [] 289 | 290 | for net in recent_heights: 291 | CONF['networks'][net]['height'] = max(set(recent_heights[net]), 292 | key=recent_heights[net].count) 293 | recent_heights[net] = [CONF['networks'][net]['height']] 294 | 295 | # Get list of seen IPs and Ports so we don't send a bitcoin magic number to a bitcoin-cash node 296 | q = session.query(Node.network, Node.address, Node.port, Node.last_seen) \ 297 | .filter(Node.last_seen > datetime.datetime.utcnow() - datetime.timedelta(days=3)) 298 | 299 | active_ips = {} 300 | for x in q.all(): 301 | key = x.address + "|" + str(x.port) 302 | if key not in active_ips: 303 | active_ips[key] = (x.network, x.last_seen) 304 | else: 305 | # Prioritize bitcoin cash nodes, as its the only client that bans when the wrong magic number is sent 306 | if x.network == "bitcoin-cash": 307 | active_ips[key] = (x.network, x.last_seen) 308 | elif x.last_seen > active_ips[key][1] and active_ips[key][0] != "bitcoin-cash": 309 | active_ips[key] = (x.network, x.last_seen) 310 | 311 | while node_processing_queue: 312 | node = node_processing_queue.pop(0) 313 | if f"{node.address}|{node.port}" in active_ips and \ 314 | active_ips[f"{node.address}|{node.port}"][0] != node.network: 315 | node.last_checked = datetime.datetime.utcnow() 316 | session.add(node) 317 | skipped_nodes += 1 318 | continue 319 | future = thread_pool.submit(connect, node.network, node.address, node.port, node.services, 320 | CONF['networks'][node.network]) 321 | futures_dict[f"{node.network}|{node.address}|{node.port}"] = node, future 322 | time.sleep(0.001) 323 | 324 | total_to_complete = len(futures_dict) 325 | 326 | while len(futures_dict) > 0: 327 | time.sleep(1) 328 | for i in list(futures_dict.keys()): 329 | if not futures_dict[i][1].done(): 330 | continue 331 | 332 | checked_nodes += 1 333 | if checked_nodes % 1000 == 0: 334 | logging.info(f" {round(checked_nodes / total_to_complete * 100.0, 1)}%") 335 | 336 | node, future = futures_dict.pop(i) 337 | result, new_addrs = future.result() 338 | if not result: 339 | continue 340 | 341 | if not result['seen']: 342 | if CONF['retry_threshold'] and CONF['retry_threshold'] > 0 and (not node.seen or ( 343 | node.last_seen and node.last_seen < datetime.datetime.utcnow() - 344 | datetime.timedelta(hours=CONF['retry_threshold']))): 345 | pass 346 | elif result['attempt'] < CONF['retries'] + 1: 347 | future = thread_pool.submit(connect, node.network, node.address, node.port, node.services, 348 | CONF['networks'][node.network], attempt=result['attempt'] + 1) 349 | futures_dict[f"{node.network}|{node.address}|{node.port}"] = node, future 350 | total_to_complete += 1 351 | retried_nodes += 1 352 | continue 353 | elif result['seen'] and result['attempt'] > 1: 354 | found_on_retry += 1 355 | 356 | x = result['timestamp'] 357 | timestamp = datetime.datetime(x.year, x.month, x.day, x.hour, x.minute, x.second, x.microsecond) 358 | 359 | node.last_checked = timestamp 360 | if node.first_checked is None: 361 | node.first_checked = timestamp 362 | if result["seen"] and not any((x.match(result['user_agent']) for x in CONF['excluded_user_agents'])): 363 | node.version = result['version'] 364 | node.last_seen = timestamp 365 | node.services = result['services'] 366 | node.user_agent = result['user_agent'] 367 | node.last_height = result['height'] 368 | if node.first_seen is None: 369 | node.first_seen = timestamp 370 | node.country, node.city, node.aso, node.asn = geocode_ip(node.address) 371 | node.seen = True 372 | 373 | seen_nodes += 1 374 | recent_heights[result['network']].append(result['height']) 375 | 376 | session.add(node) 377 | 378 | if node.seen: 379 | if not node.id: 380 | session.commit() 381 | vis = NodeVisitation(parent_id=node.id, 382 | user_agent_id=get_user_agent_id(result['user_agent']) 383 | if 'user_agent' in result else None, 384 | success=result["seen"], 385 | timestamp=timestamp, 386 | height=result['height'] if result["seen"] else None) 387 | 388 | if mnodes and node.network == "dash" and f"{node.address}:{node.port}" in mnodes: 389 | vis.is_masternode = True 390 | 391 | session.add(vis) 392 | 393 | if new_addrs: 394 | for n in new_addrs: 395 | addr = n['ipv4'] or n['ipv6'] or n['onion'] 396 | if not f"{addr};{n['port']}" in node_addresses[result['network']]: 397 | pending_nodes += 1 398 | node_addresses[result['network']].add(f"{addr};{n['port']}") 399 | new_node = Node(network=str(result['network']), address=addr, port=int(n['port']), 400 | services=int(n['services'])) 401 | if CONF['database_concurrency']: 402 | new_nodes_to_add.append(new_node) 403 | else: 404 | session.add(new_node) 405 | 406 | if CONF['database_concurrency']: 407 | # Get all unchecked nodes and nodes first seen in the past hour, 408 | # don't insert any new nodes that have already been inserted 409 | nn = session.query(Node.network, Node.address, Node.port) \ 410 | .filter(or_(Node.first_checked == None, 411 | Node.first_checked > datetime.datetime.utcnow() - datetime.timedelta(hours=1))) \ 412 | .with_for_update().all() 413 | new_set = {f"{n.network};{n.address};{n.port}" for n in nn} 414 | for i in reversed(range(len(new_nodes_to_add))): 415 | ni = f"{new_nodes_to_add[i].network};{new_nodes_to_add[i].address};{new_nodes_to_add[i].port}" 416 | if ni in new_set: 417 | del new_nodes_to_add[i] 418 | 419 | session.commit() 420 | logging.info(f"Checked {checked_nodes - retried_nodes} Nodes, {seen_nodes} Seen, {pending_nodes} More queued up. " 421 | f"({found_on_retry}/{retried_nodes} retry successes, {skipped_nodes} skipped x-network nodes)") 422 | return node_processing_queue, node_addresses 423 | 424 | 425 | def update_masternode_list(): 426 | if os.environ.get("DASH_RPC_URI"): 427 | resp = requests.post(os.environ.get("DASH_RPC_URI"), 428 | json={"jsonrpc": "2.0", "id": "jsonrpc", "method": "masternode", "params": ["list"]}, 429 | auth=(os.environ.get("DASH_RPC_USER"), os.environ.get("DASH_RPC_PASS"))) 430 | masternodes = resp.json()['result'] 431 | else: 432 | comm = "dash-cli" 433 | if os.path.isdir(CONF['dash_cli_path']): 434 | comm = os.path.join(CONF['dash_cli_path'], "dash-cli") 435 | masternodes = os.popen(f"{comm} masternode list full").read().strip() 436 | masternodes = json.loads(masternodes) 437 | 438 | m_nodes = set() 439 | if masternodes: 440 | for i, vals in masternodes.items(): 441 | if isinstance(vals, dict): 442 | address = vals['address'] 443 | else: 444 | address = vals[-1] 445 | m_nodes.add(address.strip()) 446 | 447 | if not masternodes and CONF['dash_masternodes_api']: 448 | try: 449 | m_nodes = set(requests.post(CONF['dash_masternodes_api']).json()) 450 | except: 451 | pass 452 | 453 | if m_nodes: 454 | with open("static/masternode_list.txt", 'w') as f: 455 | f.write("\n".join(m_nodes)) 456 | elif os.path.isfile("static/masternode_list.txt"): 457 | with open("static/masternode_list.txt", "r") as f: 458 | m_nodes = set(f.read().splitlines(keepends=False)) 459 | return m_nodes 460 | 461 | 462 | def set_master_nodes(m_nodes): 463 | if not m_nodes: 464 | return 465 | window_idx = 0 466 | window_size = 10000 467 | q = session.query(Node).filter(Node.seen == True) 468 | while True: 469 | start, stop = window_size * window_idx, window_size * (window_idx + 1) 470 | nodes = q.slice(start, stop).all() 471 | if nodes is None: 472 | break 473 | for n in nodes: 474 | if n.address + ":" + str(n.port) in m_nodes: 475 | n.is_masternode = True 476 | session.add(n) 477 | elif n.is_masternode: 478 | n.is_masternode = False 479 | session.add(n) 480 | session.commit() 481 | window_idx += 1 482 | if len(nodes) < window_size: 483 | break 484 | 485 | 486 | def code_ip_type(inp): 487 | if ".onion" in inp: 488 | return "Onion" 489 | elif "." in inp: 490 | return "IPv4" 491 | elif ":" in inp: 492 | return "IPv6" 493 | else: 494 | return "Unknown" 495 | 496 | 497 | def geocode_ip(address): 498 | aso = None 499 | asn = None 500 | country = None 501 | city = None 502 | if not address.endswith(".onion"): 503 | try: 504 | aso = ASN.asn(address).autonomous_system_organization 505 | asn = ASN.asn(address).autonomous_system_number 506 | except AddressNotFoundError: 507 | pass 508 | try: 509 | country = COUNTRY.country(address).country.name 510 | country = RENAMED_COUNTRIES.get(country, country) 511 | city = CITY.city(address).city.name 512 | except AddressNotFoundError: 513 | pass 514 | return country, city, aso, asn 515 | 516 | 517 | def check_active(height, deviation_config): 518 | return (deviation_config[1] - deviation_config[0]) <= height <= (deviation_config[1] + deviation_config[0]) 519 | 520 | 521 | def dump_summary(): 522 | # Set updated countries 523 | for n in session.query(Node).all(): 524 | n.country, n.city, n.aso, n.asn = geocode_ip(n.address, ) 525 | 526 | # Get and set dash masternodes 527 | if CONF['get_dash_masternodes']: 528 | mnodes = update_masternode_list() 529 | set_master_nodes(mnodes) 530 | logging.info("masternodes updated") 531 | 532 | q = session.query(Node.id, Node.network, Node.address, Node.port, Node.user_agent, Node.version, Node.asn, Node.aso, 533 | Node.country, Node.city, Node.last_seen, Node.last_height, Node.is_masternode) \ 534 | .filter(Node.seen == True) \ 535 | .filter(Node.last_seen >= datetime.datetime.utcnow() - datetime.timedelta(days=7)) 536 | 537 | nodes = pd.read_sql(q.statement, q.session.bind) 538 | nodes[['port', 'version', 'last_height']] = nodes[['port', 'version', 'last_height']].fillna(0) 539 | nodes = nodes.fillna("") 540 | 541 | if nodes.empty: 542 | logging.warning("Nodes table is empty, no results to dump") 543 | return 544 | 545 | # Exclude user agents 546 | if CONF['excluded_user_agents']: 547 | for agent_re in CONF['excluded_user_agents']: 548 | agent_re = re.compile(agent_re) 549 | nodes = nodes[~nodes['user_agent'].str.match(agent_re)].copy() 550 | 551 | now = datetime.datetime.utcnow() 552 | labels = [] 553 | for age, label in [(2, "2h"), (8, "8h"), (24, "24h"), (24 * 7, "7d"), (24 * 30, "30d")]: 554 | stt = time.time() 555 | q = session.query(Node.id, 556 | func.sum(case([(NodeVisitation.success, 1)], else_=0)).label("success"), 557 | func.count(NodeVisitation.parent_id).label("total")) \ 558 | .join(NodeVisitation, Node.id == NodeVisitation.parent_id) \ 559 | .group_by(Node.id) \ 560 | .filter(Node.last_seen > now - datetime.timedelta(hours=age)) \ 561 | .filter(NodeVisitation.timestamp >= now - datetime.timedelta(hours=age)) 562 | df = pd.read_sql(q.statement, q.session.bind) 563 | df[label] = (df['success'] / df['total']).fillna(0.0) 564 | nodes = nodes.merge(df[['id', label]], how="left") 565 | labels.append(label) 566 | logging.info(f"done {label} in {round(time.time() - stt, 3)}s") 567 | nodes = nodes.drop(['id'], 1) 568 | nodes[labels] = nodes[labels].fillna(0.0).round(3) 569 | nodes[['network', 'address']] = nodes[['network', 'address']].fillna("") 570 | nodes['address_type'] = nodes['address'].apply(code_ip_type) 571 | 572 | nodes['network'] = nodes[['network', 'user_agent']].apply( 573 | lambda x: "bitcoin-sv" if x['network'] == 'bitcoin-cash' and ' SV' in x['user_agent'] else x['network'], axis=1) 574 | 575 | networks = nodes['network'].unique() 576 | 577 | # Calculate summaries 578 | summaries = {} 579 | for network in networks: 580 | summary_df = nodes[(nodes['network'] == network) & 581 | (nodes['last_seen'] > datetime.datetime.utcnow() - datetime.timedelta( 582 | hours=8))] 583 | if summary_df.empty: 584 | continue 585 | 586 | summaries[network] = { 587 | "min": int(summary_df['last_height'].fillna(np.inf).min()), 588 | "max": int(summary_df['last_height'].fillna(0.0).max()), 589 | "mean": float(summary_df['last_height'].mean()), 590 | "stdev": float(summary_df['last_height'].std()), 591 | "med": float(summary_df['last_height'].median()), 592 | "1q": float(np.percentile(summary_df['last_height'], 25)), 593 | "3q": float(np.percentile(summary_df['last_height'], 75)), 594 | "2.5pct": float(np.percentile(summary_df['last_height'], 1)), 595 | "97.5pct": float(np.percentile(summary_df['last_height'], 99)), 596 | "age_min": nodes[nodes['network'] == network]['last_seen'].min().timestamp(), 597 | "age_max": summary_df['last_seen'].max().timestamp() 598 | } 599 | summaries[network]['iqr'] = summaries[network]['3q'] - summaries[network]['1q'] 600 | summaries[network]['95_range'] = summaries[network]['97.5pct'] - summaries[network]['2.5pct'] 601 | 602 | summaries["_timestamp"] = datetime.datetime.utcnow().isoformat() 603 | with open("static/network_summaries.json", 'w') as f: 604 | json.dump(summaries, f) 605 | 606 | if CONF['inactive_use_iqr']: 607 | deviations = {network: summaries[network]['iqr'] * ( 608 | CONF['inactive_threshold'][network] if network in CONF['inactive_threshold'] else 609 | CONF['inactive_threshold']['default']) for network in networks} 610 | else: 611 | deviations = {net: CONF['inactive_threshold'][net] if net in CONF['inactive_threshold'] else \ 612 | CONF['inactive_threshold']['default'] for net in networks} 613 | 614 | for i in deviations: 615 | deviations[i] = (deviations[i], summaries[i]['3q']) 616 | 617 | nodes['is_active'] = nodes[['network', 'last_height']] \ 618 | .apply(lambda x: check_active(x['last_height'], deviations[x['network']]), axis=1) 619 | 620 | if not CONF['export_inactive_nodes']: 621 | nodes = nodes[nodes['is_active']].copy() 622 | 623 | nodes['last_seen'] = nodes['last_seen'].values.astype(np.int64) // 10 ** 9 624 | nodes.to_csv("static/data.csv", index=False) 625 | 626 | with open("static/data.txt", "w") as f: 627 | f.write(space_sep_df(nodes)) 628 | 629 | for network in nodes['network'].unique(): 630 | net_df = nodes[nodes['network'] == network].copy() 631 | net_df = net_df.drop(['network'], 1) 632 | 633 | net_df.to_csv(f"static/data_{network}.csv", index=False) 634 | with open(os.path.join("static", f"data_{network}.json"), "w") as f: 635 | json.dump({'data': net_df.to_dict(orient="records")}, f) 636 | with open(os.path.join("static", f"data_{network}.txt"), "w") as f: 637 | f.write(space_sep_df(net_df)) 638 | 639 | nodes = nodes.drop(['user_agent', 'version', 'last_height'], 1) 640 | with open(os.path.join("static", "data.json"), "w") as f: 641 | json.dump({'data': nodes.to_dict(orient="records")}, f) 642 | 643 | # Write unique addresses only 644 | def group_nets(x): 645 | return ", ".join(sorted(set(x))) 646 | 647 | nodes = nodes.groupby(by=['address', 'asn', 'aso', 'country', 'city', 'address_type'], as_index=False).agg( 648 | {"network": group_nets, "2h": "mean", "8h": "mean", "24h": "mean", "7d": "mean", "30d": "mean"}) 649 | nodes.to_csv("static/data_unique.csv", index=False) 650 | 651 | with open(os.path.join("static", "data_unique.json"), "w") as f: 652 | json.dump({'data': nodes.to_dict(orient="records")}, f) 653 | with open(os.path.join("static", "data_unique.txt"), "w") as f: 654 | f.write(space_sep_df(nodes)) 655 | 656 | for network in networks: 657 | net_df = nodes[nodes['network'].str.contains(network)] 658 | net_df = net_df.drop(['network'], 1) 659 | net_df.to_csv(os.path.join("static", f"data_{network}_unique.csv"), index=False) 660 | with open(os.path.join("static", f"data_{network}_unique.json"), "w") as f: 661 | json.dump({'data': net_df.to_dict(orient="records")}, f) 662 | with open(os.path.join("static", f"data_{network}_unique.txt"), "w") as f: 663 | f.write(space_sep_df(net_df)) 664 | 665 | 666 | def space_sep_df(df, spacing=3): 667 | df = df.copy() 668 | df = pd.DataFrame([df.columns], columns=df.columns).append(df) 669 | for col in df.columns: 670 | df[col] = df[col].astype(str) 671 | max_len = df[col].str.len().max() + spacing 672 | df[col] = df[col].str.pad(max_len, side="right") 673 | out_str = "\n".join(("".join((str(row[x + 1]) for x in range(len(df.columns)))) for row in df.itertuples())) 674 | return out_str 675 | 676 | 677 | def main(seed=False): 678 | start_time = datetime.datetime.utcnow() 679 | thread_pool = ThreadPoolExecutor(max_workers=CONF['threads']) 680 | networks = list(CONF['networks'].keys()) 681 | prune_nodes() 682 | node_addresses, recent_heights = init_crawler(networks) 683 | 684 | if CONF['get_dash_masternodes']: 685 | mnodes = update_masternode_list() 686 | else: 687 | mnodes = None 688 | 689 | if seed: 690 | seed_nodes = check_dns(CONF['networks'], node_addresses) 691 | if seed_nodes: 692 | for n in seed_nodes: 693 | session.add(n) 694 | session.commit() 695 | 696 | node_processing_queue = calculate_pending_nodes(start_time) 697 | while node_processing_queue: 698 | node_processing_queue, node_addresses = process_pending_nodes(node_addresses, node_processing_queue, 699 | recent_heights, thread_pool, mnodes) 700 | node_processing_queue = calculate_pending_nodes(start_time) 701 | logging.info(f"Crawling complete in {round((datetime.datetime.utcnow() - start_time).seconds, 1)} seconds") 702 | 703 | 704 | def dump(): 705 | start_time = datetime.datetime.utcnow() 706 | dump_summary() 707 | generate_historic_data() 708 | logging.info(f"Results saved in {round((datetime.datetime.utcnow() - start_time).seconds, 1)} seconds") 709 | 710 | 711 | def generate_historic_data(): 712 | networks = [x[0] for x in session.query(Node.network).distinct()] 713 | sd = session.query(func.min(Node.first_seen)).one()[0] 714 | start_date = datetime.datetime(sd.year, sd.month, sd.day, 715 | sd.hour // CONF['historic_interval'] * CONF['historic_interval'], 0, 0) 716 | end_date = session.query(func.max(Node.last_seen)).one()[0] 717 | 718 | historic_interval = datetime.timedelta(hours=CONF['historic_interval']) 719 | 720 | last_date = start_date 721 | while last_date < end_date: 722 | last_date += historic_interval 723 | 724 | interval_end = start_date + historic_interval 725 | session.query(CrawlSummary).filter( 726 | CrawlSummary.timestamp >= (last_date - datetime.timedelta(hours=CONF['historic_interval'] * 1.5))).delete() 727 | session.commit() 728 | while interval_end < end_date: 729 | if session.query(CrawlSummary).filter(CrawlSummary.timestamp == interval_end).count() >= 1: 730 | interval_end += historic_interval 731 | continue 732 | logging.info(f"Summarizing period starting with {interval_end - historic_interval}") 733 | 734 | sv_sq = session.query(UserAgent.id).filter(UserAgent.user_agent.ilike("% SV%")).subquery() 735 | 736 | case_stmt = case([(sv_sq.c.id != None, 'bitcoin-sv')], else_=Node.network) 737 | 738 | q = session.query(NodeVisitation.parent_id.label("id"), 739 | case_stmt.label("network"), 740 | func.max(NodeVisitation.height).label("height"), 741 | func.max(case([(NodeVisitation.is_masternode, 1)], else_=0)).label("is_masternode")) \ 742 | .join(sv_sq, NodeVisitation.user_agent_id == sv_sq.c.id) \ 743 | .join(Node, Node.id == NodeVisitation.parent_id) \ 744 | .filter(NodeVisitation.timestamp >= interval_end - historic_interval) \ 745 | .filter(NodeVisitation.timestamp <= interval_end) \ 746 | .filter(NodeVisitation.success == True) \ 747 | .filter(Node.first_seen <= interval_end) \ 748 | .filter(Node.last_seen >= interval_end - historic_interval) \ 749 | .group_by(NodeVisitation.parent_id, case_stmt) 750 | df = pd.read_sql(q.statement, q.session.bind) 751 | 752 | df['height'] = df['height'].astype(int) 753 | if not df.empty: 754 | networks = df['network'].unique() 755 | 756 | medians = df.groupby(by=['network']).agg({"height": "median"}) 757 | deviations = {network: CONF['inactive_threshold'][network] if network in CONF['inactive_threshold'] else \ 758 | CONF['inactive_threshold']['default'] for network in networks} 759 | 760 | for i in list(deviations.keys()): 761 | if i in medians.index: 762 | deviations[i] = (deviations[i], medians.loc[i]['height']) 763 | else: 764 | deviations.pop(i) 765 | 766 | df['active'] = df[['network', 'height']].apply( 767 | lambda x: check_active(x['height'], deviations[x['network']]), axis=1) 768 | df = df[df['active']].drop(['active'], 1) 769 | 770 | for network in networks: 771 | net_df = df[df['network'] == network] 772 | cs = CrawlSummary(timestamp=interval_end, 773 | network=network, 774 | node_count=len(net_df), 775 | masternode_count=sum(net_df['is_masternode']), 776 | lookback_hours=CONF['historic_interval']) 777 | 778 | session.add(cs) 779 | session.commit() 780 | 781 | interval_end += datetime.timedelta(hours=CONF['historic_interval']) 782 | 783 | q = session.query(CrawlSummary).order_by(CrawlSummary.timestamp) 784 | df = pd.read_sql(q.statement, q.session.bind) 785 | df['timestamp'] = df['timestamp'].values.astype(np.int64) // 10 ** 9 786 | 787 | for network in networks: 788 | df[df['network'] == network][['timestamp', 'node_count', 'masternode_count']] \ 789 | .to_json(os.path.join("static", f"history_{network}.json"), orient='records') 790 | 791 | 792 | def prune_database(): 793 | if not os.path.isdir("db_cache"): 794 | os.mkdir("db_cache") 795 | 796 | q = session.query(Node) 797 | nodes = pd.read_sql(q.statement, q.session.bind) 798 | 799 | fv = session.query(func.min(NodeVisitation.timestamp)).first()[0] 800 | end_date = datetime.datetime.utcnow() - datetime.timedelta(hours=24 * CONF['max_pruning_age']) 801 | end_date = datetime.datetime(end_date.year, end_date.month, end_date.day, 0, 0, 0) 802 | 803 | current_date = datetime.datetime(fv.year, fv.month, fv.day, 0, 0, 0) 804 | current_end = current_date + datetime.timedelta(days=1) 805 | 806 | while current_end < end_date: 807 | vq = session.query(NodeVisitation) \ 808 | .filter(NodeVisitation.timestamp >= current_date) \ 809 | .filter(NodeVisitation.timestamp < current_end) 810 | 811 | f_name = f"visitations_{current_date.strftime('%Y-%m-%d')}.gz" 812 | f_name = os.path.join("db_cache", f_name) 813 | f_name_alt = f"nodes_{current_date.strftime('%Y-%m-%d')}.gz" 814 | f_name_alt = os.path.join("db_cache", f_name_alt) 815 | 816 | df = pd.read_sql(vq.statement, vq.session.bind) 817 | an = nodes.merge(df[['parent_id']].drop_duplicates(), left_on="id", right_on="parent_id") 818 | an = an[[x for x in an.columns if x != "parent_id"]] 819 | 820 | df['timestamp'] = pd.to_datetime(df['timestamp']) 821 | df['timestamp'] = df['timestamp'].astype(np.int64) / 1000000000 822 | df['height'] = df['height'].fillna(-1).apply(lambda x: int(x) if x > 0 else "") 823 | df['success'] = df['success'].fillna(-1).apply(lambda x: int(x) if x > 0 else "") 824 | df['user_agent_id'] = df['user_agent_id'].fillna(-1).apply(lambda x: int(x) if x > 0 else "") 825 | df['is_masternode'] = df['is_masternode'].fillna(-1).apply(lambda x: int(x) if x > 0 else "") 826 | df.to_csv(f_name, compression="gzip", index=False) 827 | 828 | for col in ("first_seen", "last_seen", "first_checked", "last_checked"): 829 | an[col] = pd.to_datetime(an[col]) 830 | an[col] = an[col].astype(np.int64) / 1000000000 831 | for col in ("seen", "last_height", "version", "services", "is_masternode"): 832 | an[col] = an[col].apply(lambda x: int(x) if x else "") 833 | an.to_csv(f_name_alt, compression="gzip", index=False) 834 | 835 | deleted = vq.delete() 836 | session.commit() 837 | 838 | current_date = current_date + datetime.timedelta(days=1) 839 | current_end = current_date + datetime.timedelta(days=1) 840 | logging.info(f"pruned up to {current_end} // {deleted} visitations removed") 841 | 842 | 843 | if __name__ == "__main__": 844 | 845 | if "--crawl" in sys.argv: 846 | main(seed=True if "--seed" in sys.argv else False) 847 | 848 | if "--dump" in sys.argv: 849 | dump() 850 | 851 | if "--prune" in sys.argv: 852 | prune_database() 853 | 854 | if "--daemon" in sys.argv: 855 | conf = CONF.get('daemon', {}) 856 | crawl_interval = int(conf.get('crawl_interval', 15)) 857 | dump_interval = int(conf.get('dump_interval', 60)) 858 | prune_interval = conf.get('prune_interval', None) 859 | current = int(time.time()) 860 | last_minutes = -1 861 | 862 | main(seed=True) 863 | dump() 864 | if prune_interval is not None: 865 | prune_database() 866 | while True: 867 | minutes = int(current / 60) 868 | if minutes != last_minutes: 869 | last_minutes = minutes 870 | if minutes % crawl_interval == 0: 871 | main(seed=False) 872 | if minutes % dump_interval == 0: 873 | dump() 874 | if prune_interval is not None and minutes % int(prune_interval) == 0: 875 | prune_database() 876 | 877 | current += 1 878 | while current > time.time(): 879 | time.sleep(0.1) 880 | -------------------------------------------------------------------------------- /protocol.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # protocol.py - Bitcoin protocol access for Bitnodes. 5 | # 6 | # Copyright (c) Addy Yeow Chin Heng 7 | # 8 | # Modified by open-nodes project for python3 compatibility 9 | # 10 | # Permission is hereby granted, free of charge, to any person obtaining 11 | # a copy of this software and associated documentation files (the 12 | # "Software"), to deal in the Software without restriction, including 13 | # without limitation the rights to use, copy, modify, merge, publish, 14 | # distribute, sublicense, and/or sell copies of the Software, and to 15 | # permit persons to whom the Software is furnished to do so, subject to 16 | # the following conditions: 17 | # 18 | # The above copyright notice and this permission notice shall be 19 | # included in all copies or substantial portions of the Software. 20 | # 21 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 22 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 23 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 24 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 25 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 26 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 27 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 28 | 29 | """ 30 | Bitcoin protocol access for Bitnodes. 31 | Reference: https://en.bitcoin.it/wiki/Protocol_specification 32 | 33 | ------------------------------------------------------------------------------- 34 | PACKET STRUCTURE FOR BITCOIN PROTOCOL 35 | protocol version >= 70001 36 | ------------------------------------------------------------------------------- 37 | [---MESSAGE---] 38 | [ 4] MAGIC_NUMBER (\xF9\xBE\xB4\xD9) uint32_t 39 | [12] COMMAND char[12] 40 | [ 4] LENGTH H uint16_t 54 | [26] ADDR_FROM 55 | [ 8] SERVICES H uint16_t 60 | [ 8] NONCE = 70001) bool 64 | 65 | [---ADDR_PAYLOAD---] 66 | [..] COUNT variable integer 67 | [..] ADDR_LIST multiple of COUNT (max 1000) 68 | [ 4] TIMESTAMP H uint16_t 74 | 75 | [---PING_PAYLOAD---] 76 | [ 8] NONCE LE (little-endian) 520 | msg['prev_block_hash'] = hexlify(data.read(32)[::-1]) 521 | 522 | # BE -> LE 523 | msg['merkle_root'] = hexlify(data.read(32)[::-1]) 524 | 525 | msg['timestamp'] = struct.unpack(" BE 544 | ] 545 | return b''.join(payload) 546 | 547 | def serialize_block_headers_payload(self, headers): 548 | payload = [ 549 | self.serialize_int(len(headers)), 550 | ] 551 | payload.extend( 552 | [self.serialize_block_header(header) for header in headers]) 553 | return b''.join(payload) 554 | 555 | def deserialize_block_headers_payload(self, data): 556 | msg = {} 557 | data = BytesIO(data) 558 | 559 | msg['count'] = self.deserialize_int(data) 560 | msg['headers'] = [] 561 | for _ in range(msg['count']): 562 | header = self.deserialize_block_header(data) 563 | msg['headers'].append(header) 564 | 565 | return msg 566 | 567 | def serialize_network_address(self, addr): 568 | network_address = [] 569 | if len(addr) == 4: 570 | (timestamp, services, ip_address, port) = addr 571 | network_address.append(struct.pack("H", port)) 589 | return b''.join(network_address) 590 | 591 | def deserialize_network_address(self, data, has_timestamp=False): 592 | timestamp = None 593 | if has_timestamp: 594 | timestamp = unpack("H", data.read(2)) 601 | _ipv6 += _ipv4 602 | 603 | ipv4 = "" 604 | ipv6 = "" 605 | onion = "" 606 | 607 | if _ipv6[:6] == ONION_PREFIX: 608 | onion = b32encode(_ipv6[6:]).lower().decode("utf8") + ".onion" # use .onion 609 | else: 610 | ipv6 = socket.inet_ntop(socket.AF_INET6, _ipv6) 611 | ipv4 = socket.inet_ntop(socket.AF_INET, _ipv4) 612 | if ipv4 in ipv6: 613 | ipv6 = "" # use ipv4 614 | else: 615 | ipv4 = "" # use ipv6 616 | 617 | return { 618 | 'timestamp': timestamp, 619 | 'services': services, 620 | 'ipv4': ipv4, 621 | 'ipv6': ipv6, 622 | 'onion': onion, 623 | 'port': port, 624 | } 625 | 626 | def serialize_inventory(self, item): 627 | (inv_type, inv_hash) = item 628 | payload = [ 629 | struct.pack(" BE 631 | ] 632 | return b''.join(payload) 633 | 634 | def deserialize_inventory(self, data): 635 | inv_type = unpack(" LE 637 | return { 638 | 'type': inv_type, 639 | 'hash': hexlify(inv_hash), 640 | } 641 | 642 | def serialize_tx_in(self, tx_in): 643 | payload = [ 644 | unhexlify(tx_in['prev_out_hash'])[::-1], # LE -> BE 645 | struct.pack(" LE 655 | prev_out_index = struct.unpack(" BE 689 | unhexlify(header['merkle_root'])[::-1], # LE -> BE 690 | struct.pack(" LE 700 | header = BytesIO(header) 701 | version = struct.unpack(" LE 703 | merkle_root = header.read(32)[::-1] # BE -> LE 704 | timestamp = struct.unpack(" 0: 806 | chunks = [] 807 | while length > 0: 808 | chunk = self.socket.recv(SOCKET_BUFSIZE) 809 | if not chunk: 810 | raise RemoteHostClosedConnection("{} closed connection".format(self.to_addr)) 811 | chunks.append(chunk) 812 | length -= len(chunk) 813 | data = b''.join(chunks) 814 | else: 815 | data = self.socket.recv(SOCKET_BUFSIZE) 816 | if not data: 817 | raise RemoteHostClosedConnection("{} closed connection".format(self.to_addr)) 818 | if len(data) > SOCKET_BUFSIZE: 819 | end_t = time.time() 820 | self.bps.append((len(data) * 8) / (end_t - start_t)) 821 | return data 822 | 823 | def get_messages(self, length=0, commands=None): 824 | msgs = [] 825 | data = self.recv(length=length) 826 | while len(data) > 0: 827 | time.sleep(0.0001) 828 | try: 829 | (msg, data) = self.serializer.deserialize_msg(data) 830 | except PayloadTooShortError: 831 | data += self.recv( 832 | length=self.serializer.required_len - len(data)) 833 | (msg, data) = self.serializer.deserialize_msg(data) 834 | if msg.get('command') == b"ping": 835 | self.pong(msg['nonce']) # respond to ping immediately 836 | elif msg.get('command') == b"version": 837 | self.verack() # respond to version immediately 838 | msgs.append(msg) 839 | if len(msgs) > 0 and commands: 840 | msgs[:] = [m for m in msgs if m.get('command') in commands] 841 | return msgs 842 | 843 | def set_min_version(self, version): 844 | self.serializer.protocol_version = min( 845 | self.serializer.protocol_version, 846 | version.get(b'version', self.serializer.protocol_version)) 847 | 848 | def handshake(self): 849 | # [version] >>> 850 | msg = self.serializer.serialize_msg( 851 | command=b"version", to_addr=self.to_addr, from_addr=self.from_addr) 852 | self.send(msg) 853 | 854 | # <<< [version 124 bytes] [verack 24 bytes] 855 | time.sleep(1) 856 | msgs = self.get_messages(length=148, commands=[b"version", b"verack"]) 857 | if len(msgs) > 0: 858 | msgs[:] = sorted(msgs, key=itemgetter('command'), reverse=True) 859 | self.set_min_version(msgs[0]) 860 | return msgs 861 | 862 | def verack(self): 863 | # [verack] >>> 864 | msg = self.serializer.serialize_msg(command=b"verack") 865 | self.send(msg) 866 | 867 | def getaddr(self, block=True): 868 | # [getaddr] >>> 869 | msg = self.serializer.serialize_msg(command=b"getaddr") 870 | self.send(msg) 871 | 872 | # Caller should call get_messages separately. 873 | if not block: 874 | return None 875 | 876 | # <<< [addr].. 877 | time.sleep(3) 878 | msgs = self.get_messages(commands=[b"addr"]) 879 | return msgs 880 | 881 | def getpeerinfo(self, block=True): 882 | # [getaddr] >>> 883 | msg = self.serializer.serialize_msg(command=b"getpeerinfo") 884 | self.send(msg) 885 | 886 | # Caller should call get_messages separately. 887 | if not block: 888 | return None 889 | 890 | # <<< [addr].. 891 | msgs = self.get_messages(commands=[b"getpeerinfo"]) 892 | return msgs 893 | 894 | def addr(self, addr_list): 895 | # addr_list = [(TIMESTAMP, SERVICES, "IP_ADDRESS", PORT),] 896 | # [addr] >>> 897 | msg = self.serializer.serialize_msg( 898 | command=b"addr", addr_list=addr_list) 899 | self.send(msg) 900 | 901 | def ping(self, nonce=None): 902 | if nonce is None: 903 | nonce = random.getrandbits(64) 904 | 905 | # [ping] >>> 906 | msg = self.serializer.serialize_msg(command=b"ping", nonce=nonce) 907 | self.send(msg) 908 | 909 | def pong(self, nonce): 910 | # [pong] >>> 911 | msg = self.serializer.serialize_msg(command=b"pong", nonce=nonce) 912 | self.send(msg) 913 | 914 | def inv(self, inventory): 915 | # inventory = [(INV_TYPE, "INV_HASH"),] 916 | # [inv] >>> 917 | msg = self.serializer.serialize_msg( 918 | command=b"inv", inventory=inventory) 919 | self.send(msg) 920 | 921 | def getdata(self, inventory): 922 | # inventory = [(INV_TYPE, "INV_HASH"),] 923 | # [getdata] >>> 924 | msg = self.serializer.serialize_msg( 925 | command=b"getdata", inventory=inventory) 926 | self.send(msg) 927 | 928 | # <<< [tx] [block].. 929 | time.sleep(1) 930 | msgs = self.get_messages(commands=[b"tx", b"block"]) 931 | return msgs 932 | 933 | def getblocks(self, block_hashes, last_block_hash=None): 934 | if last_block_hash is None: 935 | last_block_hash = "0" * 64 936 | 937 | # block_hashes = ["BLOCK_HASH",] 938 | # [getblocks] >>> 939 | msg = self.serializer.serialize_msg(command=b"getblocks", 940 | block_hashes=block_hashes, 941 | last_block_hash=last_block_hash) 942 | self.send(msg) 943 | 944 | # <<< [inv].. 945 | time.sleep(1) 946 | msgs = self.get_messages(commands=[b"inv"]) 947 | return msgs 948 | 949 | def getheaders(self, block_hashes, last_block_hash=None): 950 | if last_block_hash is None: 951 | last_block_hash = "0" * 64 952 | 953 | # block_hashes = ["BLOCK_HASH",] 954 | # [getheaders] >>> 955 | msg = self.serializer.serialize_msg(command=b"getheaders", 956 | block_hashes=block_hashes, 957 | last_block_hash=last_block_hash) 958 | self.send(msg) 959 | 960 | # <<< [headers].. 961 | time.sleep(1) 962 | msgs = self.get_messages(commands=[b"headers"]) 963 | return msgs 964 | 965 | def headers(self, headers): 966 | # headers = [{ 967 | # 'version': VERSION, 968 | # 'prev_block_hash': PREV_BLOCK_HASH, 969 | # 'merkle_root': MERKLE_ROOT, 970 | # 'timestamp': TIMESTAMP, 971 | # 'bits': BITS, 972 | # 'nonce': NONCE 973 | # },] 974 | # [headers] >>> 975 | msg = self.serializer.serialize_msg(command=b"headers", headers=headers) 976 | self.send(msg) 977 | 978 | class Keepalive(object): 979 | """ 980 | Implements keepalive mechanic to keep the specified connection with a node. 981 | """ 982 | def __init__(self, conn, keepalive_time): 983 | self.conn = conn 984 | self.keepalive_time = keepalive_time 985 | 986 | def keepalive(self, addr=False): 987 | st = time.time() 988 | last_ping = time.time() - 10 989 | addrs = [] 990 | while time.time() - st < self.keepalive_time: 991 | if time.time() - last_ping > 9: 992 | try: 993 | self.ping() 994 | last_ping = time.time() 995 | except socket.error as err: 996 | logging.debug("keepalive failed %s", err) 997 | break 998 | time.sleep(0.3) 999 | try: 1000 | if addr: 1001 | new = self.conn.get_messages(commands=[b'addr']) 1002 | addrs += new 1003 | else: 1004 | self.conn.get_messages() 1005 | except socket.timeout: 1006 | pass 1007 | except (ProtocolError, ConnectionError, socket.error) as err: 1008 | logging.debug("getmsg failed %s", err) 1009 | break 1010 | return addrs 1011 | 1012 | def ping(self): 1013 | """ 1014 | Sends a ping message. Ping time is stored in Redis for round-trip time 1015 | (RTT) calculation. 1016 | """ 1017 | nonce = random.getrandbits(64) 1018 | try: 1019 | self.conn.ping(nonce=nonce) 1020 | except socket.error: 1021 | raise 1022 | self.last_ping = time.time() 1023 | --------------------------------------------------------------------------------