├── LICENSE ├── README.md ├── bin ├── app.py └── prefix2as │ ├── __init__.py │ ├── db │ ├── __init__.py │ ├── orm │ │ └── route4.py │ └── sql │ │ └── route4.sql │ ├── models.py │ └── views.py ├── prefix2as.conf.sample ├── requirements.txt └── tools └── updateRouting ├── lib ├── __init__.py ├── db.py ├── fetch.py └── mrt.py ├── updateRouting.conf.sample └── updateRouting.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Shuto Imai. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # prefix2as 2 | インターネット上に流れているフルルートを用いて、 IP アドレスと AS 番号を相互変換する Web API システムです。 3 | 自宅 Threat Intelligence プロジェクト「斥候」の一機能として、Threat Intelligence に活用するために作りました。 4 | 5 | ## 特徴 6 | * RADBなど IRR を使わず実際に流れているフルルートを用いるので、より正確な実態を掴めます。 7 | * Route Views Archive Project が提供するフルルートをインポートする仕組みを備えているので、自前でフルルートを用意できない人でも使えます。 8 | * フルルートを自前で用意できる人のためにも、 quagga で 出力した MRT ファイルのインポートに対応しています。(strftime format 対応) 9 | * 外部 API を使わないので外部に調査対象情報が漏れず、BAN もされず、費用もかかりません。 10 | 11 | ## Demo 12 | * IP アドレス指定時 13 | ``` 14 | http://localhost:8001/prefix2as/search?ip4=203.178.137.58 15 | 16 | { 17 | "header": { 18 | "errorCode": 0, 19 | "status": "success" 20 | }, 21 | "response": [ 22 | { 23 | "as": 2500, 24 | "date": 1526299202, 25 | "prefix": "203.178.128.0/17" 26 | } 27 | ] 28 | } 29 | ``` 30 | 31 | * AS 番号指定時 32 | ``` 33 | http://localhost:8001/prefix2as/search?as=2500 34 | 35 | { 36 | "header": { 37 | "errorCode": 0, 38 | "status": "success" 39 | }, 40 | "response": [ 41 | { 42 | "as": 2500, 43 | "date": 1526299201, 44 | "prefix": "133.4.128.0/18" 45 | }, 46 | { 47 | "as": 2500, 48 | "date": 1526299201, 49 | "prefix": "133.138.0.0/16" 50 | }, 51 | { 52 | "as": 2500, 53 | "date": 1526299201, 54 | "prefix": "150.52.0.0/16" 55 | }, 56 | { 57 | "as": 2500, 58 | "date": 1526299201, 59 | "prefix": "163.221.0.0/16" 60 | }, 61 | { 62 | "as": 2500, 63 | "date": 1526299202, 64 | "prefix": "202.0.73.0/24" 65 | }, 66 | { 67 | "as": 2500, 68 | "date": 1526299202, 69 | "prefix": "202.244.32.0/21" 70 | }, 71 | { 72 | "as": 2500, 73 | "date": 1526299202, 74 | "prefix": "202.249.0.0/18" 75 | }, 76 | { 77 | "as": 2500, 78 | "date": 1526299202, 79 | "prefix": "203.178.128.0/17" 80 | } 81 | ] 82 | } 83 | ``` 84 | 85 | # Install 86 | 87 | ## 推奨環境 88 | * Python 3.6 以上 89 | * MySQL 5.6 以上 90 | 91 | 92 | ## libbgpdump のインストール 93 | prefix2asのフルルートインポートには処理速度の関係上、libbgpdump の bgpdump コマンドを使っています。 94 | お使いのディストリビューションに合わせて libbgpdump をインストールしてください。 95 | 96 | ## ライブラリのインストール 97 | ``` 98 | pip install -r ./requirements.txt 99 | ``` 100 | 101 | ## データベースの作成 102 | * 好きな名前で作ってください。「sekkoh」とか。 103 | 104 | ## フルルートインポートツール「updateRouting」の設定 105 | ``` 106 | cd tools/updateRouting 107 | cp updateRouting.conf.sample updateRouting.conf 108 | vim updateRouting.conf 109 | ``` 110 | * dburl の値を利用するデータベースのURLに設定してください。 111 | ``` 112 | dburl = mysql+pymysql://user:password@127.0.0.1:3306/sekkoh?charset=utf8 113 | ``` 114 | 115 | ※他にもインポートする世代数や、ローカル上の MRT ファイルを指定可能です。 116 | ※詳細は updateRouting.conf を参照してください。 117 | 118 | ## フルルートのインポート 119 | updateRouting を使って Route Views Archive Project からフルルートをダウンロードし、データベースにインポートします。 120 | ``` 121 | ./updateRouting.py -c ./updateRouting.conf 122 | ``` 123 | 124 | Route Views Archive Project のフルルートは2時間ごとに更新されるので、 125 | 定期的に更新したい方は cron で 2時間おきに実行するようにすると良いでしょう。 126 | 127 | ## prefix2as の設定 128 | ``` 129 | cp prefix2as.conf.sample prefix2as.conf 130 | vim prefix2as.conf 131 | ``` 132 | * updateRouting と同じように、DBURL の値を利用するデータベースのURLに設定してください。 133 | ``` 134 | DBURL = 'mysql+pymysql://user:password@127.0.0.1:3306/sekkoh?charset=utf8' 135 | ``` 136 | 137 | # Startup 138 | ``` 139 | cd bin 140 | ./app.py 141 | ``` 142 | 143 | 起動後はデモのように色々試してみてください。 144 | 145 | # Route Views Project について 146 | Route Views Project (http://www.routeviews.org/) は University of Oregon Foundation を通じて寄付を募っています。 147 | Route Views Archive Project から定期的にフルルートを取得する方は寄付をご検討ください。 148 | 149 | http://www.routeviews.org/routeviews/index.php/about/ 150 | 151 | # License 152 | MIT License 153 | 154 | -------------------------------------------------------------------------------- /bin/app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from flask import Flask, Blueprint 4 | from prefix2as.views import prefix2as 5 | 6 | def create_app(): 7 | 8 | app = Flask(__name__) 9 | try: 10 | app.config.from_pyfile('../prefix2as.conf') 11 | except FileNotFoundError as exc: 12 | app.logger.critical("'../prefix2as.conf' is not found.") 13 | raise FileNotFoundError(exc) 14 | 15 | try: 16 | dburl = app.config['DBURL'] 17 | app.config['SQLALCHEMY_DATABASE_URI'] = dburl 18 | app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True 19 | except KeyError as exc: 20 | app.logger.critical("DBURL is not set. please set dburl at prefix2as.conf!") 21 | raise KeyError(exc) 22 | 23 | 24 | app.register_blueprint(prefix2as) 25 | 26 | return app 27 | 28 | if __name__ == '__main__': 29 | app = create_app() 30 | app.run(debug=True, host=app.config['LISTEN'], port=app.config['PORT']) 31 | -------------------------------------------------------------------------------- /bin/prefix2as/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shutingrz/prefix2as/d8f7664df43b772c00323da69f280f493ad5495d/bin/prefix2as/__init__.py -------------------------------------------------------------------------------- /bin/prefix2as/db/__init__.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import create_engine, MetaData, Table 2 | from sqlalchemy.orm import scoped_session, mapper 3 | from sqlalchemy.orm.session import sessionmaker 4 | from flask import Flask, current_app 5 | 6 | database_uri = current_app.config['SQLALCHEMY_DATABASE_URI'] 7 | engine = create_engine(database_uri) 8 | 9 | metadata = MetaData(bind=engine) 10 | session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine)) 11 | 12 | class Model(object): 13 | query = session.query_property() 14 | 15 | 16 | -------------------------------------------------------------------------------- /bin/prefix2as/db/orm/route4.py: -------------------------------------------------------------------------------- 1 | from prefix2as import db 2 | 3 | class Route4(db.Model): 4 | def __init__(self, id, history_id, date, asnum, prefix, start_ip, end_ip, size): 5 | self.id = id 6 | self.history_id = history_id 7 | self.asnum = asnum 8 | self.prefix = prefix 9 | self.start_ip = start_ip 10 | self.end_ip = end_ip 11 | self.size = size 12 | 13 | db.mapper(Route4, db.Table('route4', db.metadata, autoload=True)) 14 | 15 | -------------------------------------------------------------------------------- /bin/prefix2as/db/sql/route4.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE route4( 2 | id bigint auto_increment primary key, 3 | history_id integer, 4 | date bigint unsigned, 5 | asnum bigint unsigned, 6 | prefix varchar(18), 7 | start_ip bigint unsigned, 8 | end_ip bigint unsigned, 9 | size integer unsigned 10 | ); 11 | -------------------------------------------------------------------------------- /bin/prefix2as/models.py: -------------------------------------------------------------------------------- 1 | import ipaddress 2 | import sqlalchemy 3 | from sqlalchemy import asc, desc 4 | from flask import current_app 5 | 6 | class Prefix2asModel(object): 7 | 8 | def __init__(self): 9 | pass 10 | 11 | def _initDB(self): 12 | try: 13 | from prefix2as import db 14 | from prefix2as.db.orm.route4 import Route4 15 | except sqlalchemy.exc.OperationalError as exc: 16 | current_app.logger.critical("Database connection error: %s" % exc) 17 | return None, None 18 | except sqlalchemy.exc.NoSuchTableError as exc: 19 | current_app.logger.critical("route4 table is not exist. please exec updateRouting.py") 20 | return None, None 21 | except Exception as exc: 22 | current_app.logger.critical("Unknown OR/M error: %s" % exc) 23 | raise Exception(exc) 24 | 25 | return db, Route4 26 | 27 | def getRoutes_fromPrefix(self, prefix, date=None): 28 | db, Route4 = self._initDB() 29 | 30 | if db is Route4 is None: 31 | return None, 100 32 | 33 | 34 | #work around: select all routes without get min size. (if use "order by , very slowly.) 35 | ip_int = int(ipaddress.ip_address(prefix)) 36 | data = db.session.query(Route4.asnum, Route4.prefix, Route4.date, Route4.size).\ 37 | filter(Route4.start_ip <= ip_int, ip_int <= Route4.end_ip).\ 38 | all() 39 | 40 | #if route is not found. 41 | if len(data) < 1: 42 | return [], 0 43 | 44 | #get best path (min size) 45 | size_tmparray = [] 46 | for i in data: 47 | size_tmparray.append(i[3]) 48 | 49 | min_size = min(size_tmparray) 50 | 51 | routes = [] 52 | for item in data: 53 | if item[3] == min_size: 54 | routes.append({'as': item[0], 'prefix': item[1], 'date': item[2]}) 55 | 56 | return routes, 0 57 | 58 | def getRoutes_fromASnum(self, asnum, date=None): 59 | db, Route4 = self._initDB() 60 | 61 | if db is Route4 is None: 62 | return None, 100 63 | 64 | data = db.session.query(Route4.asnum, Route4.prefix, Route4.date, Route4.size).\ 65 | filter(Route4.asnum == asnum).\ 66 | all() 67 | 68 | routes = [] 69 | for item in data: 70 | routes.append({'as': item[0], 'prefix': item[1], 'date': item[2]}) 71 | 72 | return routes, 0 73 | 74 | -------------------------------------------------------------------------------- /bin/prefix2as/views.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, jsonify, url_for, current_app, request 2 | from prefix2as.models import Prefix2asModel 3 | import ipaddress 4 | 5 | prefix2as = Blueprint('prefix2as', __name__, url_prefix='/prefix2as') 6 | 7 | @prefix2as.route('/') 8 | def index(): 9 | return jsonify( 10 | search = url_for('.search'), 11 | ) 12 | 13 | def _makeErrorMessage(code): 14 | data = {'header': {'status': 'error', 'errorCode': code}, 'response': {}} 15 | return data 16 | 17 | def _makeResponseMessage(response): 18 | data = {'header': {'status': 'success', 'errorCode': 0}, 'response': response} 19 | return data 20 | 21 | @prefix2as.route('/search', methods=['GET']) 22 | def search(): 23 | ip4 = request.args.get('ip4', None) 24 | asnum = request.args.get('as', None) 25 | 26 | if ip4 is not None: 27 | try: 28 | ipaddress.ip_address(ip4) 29 | except Exception as exp: 30 | return jsonify(_makeErrorMessage(10)) 31 | model = Prefix2asModel() 32 | routes, code = model.getRoutes_fromPrefix(ip4) 33 | elif asnum is not None: 34 | if asnum.isdigit(): 35 | if int(asnum) < 2**32: 36 | model = Prefix2asModel() 37 | routes, code = model.getRoutes_fromASnum(asnum) 38 | else: 39 | return jsonify(_makeErrorMessage(20)) 40 | else: 41 | return jsonify(_makeErrorMessage(30)) 42 | else: 43 | return jsonify(_makeErrorMessage(40)) 44 | 45 | if routes is None: 46 | return jsonify(_makeErrorMessage(code)) 47 | else: 48 | return jsonify(_makeResponseMessage(routes)) 49 | 50 | 51 | -------------------------------------------------------------------------------- /prefix2as.conf.sample: -------------------------------------------------------------------------------- 1 | #[Server] 2 | LISTEN = '0.0.0.0' 3 | PORT = '8001' 4 | 5 | #[DB] 6 | #DBURL = 'mysql+pymysql://{username}:{password}@{host}:{port}/{dbname}?charset=utf8' 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Flask 2 | Flask-MySQL 3 | Flask-SQLAlchemy 4 | mysql-connector-python 5 | -------------------------------------------------------------------------------- /tools/updateRouting/lib/__init__.py: -------------------------------------------------------------------------------- 1 | from . import mrt 2 | from . import db 3 | from . import fetch 4 | -------------------------------------------------------------------------------- /tools/updateRouting/lib/db.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from sqlalchemy import create_engine 3 | from sqlalchemy.orm import sessionmaker 4 | from sqlalchemy import Column, String, DateTime, text, DATETIME, BIGINT, INTEGER 5 | #from sqlalchemy.dialects.mysql import INTEGER 6 | from sqlalchemy.ext.declarative import declarative_base 7 | from logging import getLogger 8 | from datetime import datetime 9 | 10 | Base = declarative_base() 11 | 12 | logger = getLogger(__name__) 13 | 14 | class DBController(): 15 | def __init__(self, dburl): 16 | self.dburl = dburl 17 | self.engine = None 18 | self.Session = None 19 | 20 | def getSession(self): 21 | return self._createSession(self.dburl) 22 | 23 | def _createSession(self, dburl): 24 | engine = create_engine(dburl) 25 | Base.metadata.create_all(bind=engine) 26 | Session = sessionmaker(bind=engine) 27 | return Session() 28 | 29 | 30 | class Route4DBController(DBController): 31 | def __init__(self, dburl): 32 | super(Route4DBController, self).__init__(dburl) 33 | 34 | def update(self, mrt, max_history): 35 | logger.info("Database updating...") 36 | session = self.getSession() 37 | 38 | self._deleteHistory(session, max_history) 39 | 40 | histid = self._createHistory(session) 41 | 42 | logger.debug("insert statement create start.") 43 | route4 = [dict(history_id=histid, 44 | date=route["date"], 45 | asnum=route["asnum"], 46 | prefix=route["prefix"], 47 | start_ip=route["start_ip"], 48 | end_ip=route["end_ip"], 49 | size=route["size"] 50 | ) for route in mrt] 51 | session.execute(Route4.__table__.insert(), route4) 52 | logger.debug("insert statement create end.") 53 | logger.debug("commit start.") 54 | session.commit() 55 | logger.debug("commit end.") 56 | session.close() 57 | 58 | logger.info("...end") 59 | 60 | def _deleteHistory(self, session, max_history): 61 | if max_history == 0: # no delete. 62 | return False 63 | 64 | res = session.query(Route4History.id).order_by(Route4History.id.desc()).first() 65 | 66 | #for initial commit. 67 | if res is None: 68 | logger.debug("_deleteHistory: did not delete (history is none.)") 69 | return False 70 | 71 | deletable_maxid = int(res.id) - max_history + 1 72 | 73 | logger.debug("max_history:%s , deletable_maxid:%s" % (max_history, deletable_maxid)) 74 | 75 | if deletable_maxid < 1: 76 | logger.debug("_deleteHistory: did not delete (deletable_maxid < 1)") 77 | return False 78 | 79 | logger.debug("Route4 delete: id <= %s start." % deletable_maxid) 80 | session.query(Route4).filter(Route4.history_id <= deletable_maxid).delete() 81 | session.commit() 82 | logger.debug("Route4 deleted.") 83 | return True 84 | 85 | def _createHistory(self, session): 86 | history = Route4History() 87 | session.add(history) 88 | session.commit() 89 | 90 | res = session.query(Route4History.id).order_by(Route4History.id.desc()).first() 91 | return res.id 92 | 93 | 94 | class Route4(Base): 95 | __tablename__ = "route4" 96 | 97 | id = Column(BIGINT, primary_key=True, autoincrement=True) 98 | history_id = Column(INTEGER) 99 | date = Column(BIGINT) 100 | asnum = Column(BIGINT) 101 | prefix = Column(String(18)) # max: 255.255.255.255/32 => 18 102 | start_ip = Column(BIGINT) 103 | end_ip = Column(BIGINT) 104 | size = Column(INTEGER) 105 | 106 | class Route4History(Base): 107 | __tablename__ = "route4history" 108 | 109 | id = Column(BIGINT, primary_key=True, autoincrement=True) 110 | created_at = Column(DATETIME, default=datetime.now) 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /tools/updateRouting/lib/fetch.py: -------------------------------------------------------------------------------- 1 | import os, re, urllib.request, bz2 2 | from urllib.parse import urlparse 3 | from datetime import datetime, timezone, timedelta 4 | from logging import getLogger 5 | 6 | logger = getLogger(__name__) 7 | 8 | class Fetch(): 9 | URL = 1 10 | 11 | SUPPORTED_URLSCHEME = ["http", "https"] 12 | 13 | def __init__(self): 14 | pass 15 | 16 | def fetch(self, url): 17 | tmppath = None 18 | tmppath = self._fetchMRTFile(url) 19 | 20 | if tmppath is None: 21 | raise NameError("tmppath is None.") 22 | else: 23 | return tmppath 24 | 25 | 26 | def _fetchMRTFile(self, url): 27 | if not self._isExpectedURLFormat(url): 28 | raise SyntaxError("url must URL scheme.") 29 | 30 | logger.debug("URL: %s" % url) 31 | 32 | if self._isURLResourceValid(url): 33 | logger.debug("fetch start.") 34 | opener = urllib.request.build_opener() 35 | tmpFilepath, headers = urllib.request.urlretrieve(url) 36 | logger.debug("fetch end.") 37 | return tmpFilepath 38 | else: 39 | return None 40 | 41 | def _isURLResourceValid(self, url): 42 | opener = urllib.request.build_opener() 43 | 44 | req = urllib.request.Request(url, method="HEAD") 45 | resp = opener.open(req) 46 | return True 47 | 48 | def _isExpectedURLFormat(self, url): 49 | parse = urlparse(url) 50 | if parse.scheme in Fetch.SUPPORTED_URLSCHEME: 51 | return True 52 | else: 53 | return False 54 | 55 | def _expandTimeFormat(self, path, time): 56 | tmppath = path 57 | while True: 58 | match = re.search(r"\${(?P