├── README.md ├── aircraft.py ├── avosint.py ├── geoloc.py ├── investigation_authorities.py ├── libs ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-36.pyc │ ├── display.cpython-36.pyc │ ├── geoloc.cpython-36.pyc │ └── planes.cpython-36.pyc ├── display.py ├── flight_data.py ├── icao_converter.py └── planes.py ├── monitor.py ├── parsr.conf ├── registers.json ├── registers.py ├── requirements.txt ├── tail_to_register.py └── wiki_api.py /README.md: -------------------------------------------------------------------------------- 1 | # AVOSINT 2 | A tool to search Aviation-related intelligence from public sources. 3 | 4 | ## Usage 5 | ### Launch parsr docker image (for pdf-file stored registers) 6 | `docker run -p 3001:3001 axarev/parsr` 7 | 8 | ### Launch avosint 9 | `./avosint.py [--action ACTION] [--tail-number TAIL-NUMBER] [--icao ICAO]` 10 | 11 | With ACTION being either `ICAO`, `tail`, `convert`, `monitor` 12 | 13 | `tail` - Gather infos starting from tail number. Option `--tail-number` is required. 14 | 15 | `convert` - Convert USA hex to ICAO. Option `--icao` is required. 16 | 17 | `monitor` - Gathers positionnal information from osint sources and detects hovering patterns. Requires `--icao` number 18 | 19 | Returns the following informations when possible: 20 | * Owner of the aircraft 21 | * User of the aircraft 22 | * Aircraft transponder id 23 | * Aircraft manufacturer serial number 24 | * Aircraft model 25 | * Aircraft picture links 26 | * Aircraft incident history 27 | 28 | The following display is then presented: 29 | ```========================================== 30 | Current Status: [Done] 31 | Last action: tail 32 | Current tail: {tail_n} 33 | ========================================== 34 | ✈️ Aircraft infos: 35 | 36 | Manufacturer: {} 37 | Manufacturer Serial Number: {} 38 | Tail Number: {} 39 | Call Sign: {} 40 | Last known position: {} 41 | Last known altitude: {} 42 | 43 | 🧍 Owner infos 44 | 45 | Name: {} 46 | Street: {} 47 | City: {} 48 | ZIP: {} 49 | Country: {} 50 | 51 | New Action [ICAO, tail, convert, monitor, exit, quit] (None): 52 | ``` 53 | 54 | ## Dependencies 55 | ### Install python requirements 56 | `pip install -r requirements.txt` 57 | 58 | This tool also uses the OpenSkyApi available at https://github.com/openskynetwork/opensky-api. Install it using: 59 | ```bash 60 | git clone https://github.com/openskynetwork/opensky-api 61 | pip install -e /path/to/repository/python 62 | ``` 63 | ### Install Parsr docker image 64 | `docker run -p 3001:3001 axarev/parsr` 65 | ### Parsr 66 | As some registers are in the form of a pdf file, AVOSINT uses parsr (https://github.com/axa-group/Parsr) 67 | Due to a bug in the current version of the parsr library (https://github.com/axa-group/Parsr/issues/565#issue-1111665010) it is necessary to apply the following fix in the `parsr-client` python library: 68 | 69 | 70 | ```diff 71 | return { 72 | - 'file': file, 73 | - 'config': config, 74 | + 'file': file_path, 75 | + 'config': config_path, 76 | 'status_code': r.status_code, 77 | 'server_response': r.text 78 | } 79 | ``` 80 | ### Donations 81 | If you wish to offer me a coffee, you can donate at 82 | [![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/K3K4BO91O) 83 | -------------------------------------------------------------------------------- /aircraft.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import requests 3 | import json 4 | from bs4 import BeautifulSoup 5 | from geoloc import Coordinates 6 | 7 | # Decoders for countries here 8 | 9 | class Aircraft: 10 | def __init__(self, tail_n, msn=None, call=None, \ 11 | latitude=None, longitude=None, craft_type=None, \ 12 | origin=None, destination=None, altitude=None, \ 13 | manufacturer=None, icao=None,notes=None): 14 | 15 | self.tail_n = tail_n 16 | self.msn = msn 17 | self.call = call 18 | self.origin = origin 19 | self.manufacturer = manufacturer 20 | self.destination = destination 21 | self.altitude = altitude 22 | self.latitude = latitude 23 | self.longitude = longitude 24 | self.icao = icao 25 | self.notes = notes 26 | 27 | 28 | def __str__(self): 29 | return self.__repr__() 30 | 31 | def __repr__(self): 32 | return """ 33 | Manufacturer/Type: %s 34 | Manufacturer Serial Number: %s 35 | Transponder code (ICAO24): %s 36 | Tail Number: %s 37 | Call Sign: %s 38 | Last known position: %s 39 | Last known altitude: %s 40 | 41 | Notes: 42 | %s 43 | """ % (self.manufacturer, self.msn, 44 | self.icao, self.tail_n, 45 | self.call, (self.latitude, self.longitude), 46 | self.altitude, self.notes) 47 | -------------------------------------------------------------------------------- /avosint.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # coding: utf8 4 | # Python script to lookup plane owner's in a particular geographic area using public data from planefinder.net and the federal aviation agency. 5 | # If a particular owner is found, the plane infos are shown 6 | 7 | 8 | # TODO 9 | # Implement ADS-B 10 | # Implement news source, location API, and search based on location name 11 | import os 12 | import sys 13 | import requests 14 | import random as rand 15 | import json 16 | import logging 17 | import argparse 18 | import time 19 | import socket 20 | import csv 21 | 22 | from registers import * 23 | from tail_to_register import * 24 | from investigation_authorities import * 25 | from monitor import monitor 26 | from wiki_api import search_wiki_commons 27 | from opensky_api import OpenSkyApi 28 | from bs4 import BeautifulSoup 29 | from threading import Thread 30 | 31 | # Data sources 32 | flightradar = 'http://data.flightradar24.com/zones/fcgi/feed.js?bounds=' 33 | planefinder = 'https://planefinder.net/endpoints/update.php'\ 34 | '?callback=planeDataCallback&faa=1&routetype=iata&cfCache=true'\ 35 | '&bounds=37%2C-80%2C40%2C-74&_=1452535140' 36 | flight_data_src = 'http://data-live.flightradar24.com/clickhandler/?version=1.5&flight=' 37 | 38 | 39 | # News source 40 | AP = 'Associated Press' 41 | AFP = 'Agence France Presse' 42 | AP_KEY = 'API KEY HERE' 43 | 44 | 45 | # Docker 46 | DOCKER_HOST = '127.0.0.1' 47 | DOCKER_PORT = 3001 48 | 49 | 50 | # GLOBAL VARIABLES 51 | verbose = False 52 | 53 | # Text colors using ANSI escaping. Surely theres a better way to do this 54 | class bcolors: 55 | ERRO = '\033[31m' 56 | WARN = '\033[93m' 57 | OKAY = '\033[32m' 58 | STOP = '\033[0m' 59 | 60 | class NoIntelException(Exception): 61 | """ Raised when no information has been found in registers""" 62 | pass 63 | 64 | def printok(str): 65 | return print(bcolors.OKAY+'[OK]'+bcolors.STOP+' {}'.format(str)) 66 | 67 | def printko(str): 68 | return print(bcolors.ERRO+'[KO]'+bcolors.STOP+' {}'.format(str)) 69 | 70 | def printwarn(str): 71 | return print(bcolors.WARN+'[WRN]'+bcolors.STOP+' {}'.format(str)) 72 | 73 | def printverbose(str): 74 | if verbose: 75 | print(str) 76 | else: 77 | pass 78 | def quit(): 79 | print('bye then !\nIf you wish, you can buy me a coffee at https://ko-fi.com/arctos') 80 | return 0 81 | def check_config(): 82 | check_config_file_coherence = False 83 | check_docker_connectivity = False 84 | 85 | print("[*] Checking config") 86 | 87 | # Tests connectivity to the docker container 88 | timeout_seconds=1 89 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 90 | sock.settimeout(timeout_seconds) 91 | result = sock.connect_ex((DOCKER_HOST, DOCKER_PORT)) 92 | sock.close() 93 | 94 | docker_result = result 95 | 96 | if result == 0: 97 | printok("Parsr docker container is reachable") 98 | return True 99 | else: 100 | printwarn("Could not contact docker container. PDF registery lookup will not work.") 101 | return False 102 | 103 | 104 | 105 | def getInterestingPlaces(): 106 | # Get news feed about things that could require a plane flyover 107 | # Wildfire, traffic accidents, police intervention, natural disasters, etc. 108 | return None 109 | 110 | def intel_from_ICAO(icao): 111 | """ 112 | Gather intel starting with icao number 113 | """ 114 | return None 115 | 116 | def opensky(tail_n): 117 | print("[*] Gathering infos from opensky network database. This can take some time") 118 | headers = { 119 | 'User-Agent': 'AVOSINT - CLI tool to gather aviation OSINT.'\ 120 | 'Infos and contact: https://github.com/n0skill/AVOSINT' 121 | } 122 | 123 | if os.path.exists('/tmp/opensky.cache') \ 124 | and os.stat("/tmp/opensky.cache").st_size != 0: 125 | print('[*] File exists. Do not download again') 126 | 127 | else: 128 | r = requests.get( 129 | 'https://opensky-network.org/datasets/metadata/aircraftDatabase.csv', 130 | stream=True, 131 | headers=headers) 132 | 133 | if r.status_code == 200: 134 | with open('/tmp/opensky.cache', 'wb') as f: 135 | total_l = int(r.headers.get('content-length')) 136 | dl = 0 137 | for data in r.iter_content(chunk_size=8192*4): 138 | dl += len(data) 139 | f.write(data) 140 | print('\r[*] Downloading {:2f}'.format((dl/total_l)*100), end='') 141 | print('\r[*] Done loading !') 142 | else: 143 | printwarn(r.status_code) 144 | 145 | with open('/tmp/opensky.cache', 'r') as f: 146 | parsed_content = csv.reader(f) 147 | for line in parsed_content: 148 | if tail_n in line: 149 | # Aircraft infos 150 | icao = line[0] 151 | manufacturer = line[3] 152 | msn = line[6] 153 | # Owner infos 154 | owner = line[13] 155 | return Aircraft(tail_n, 156 | icao=icao, 157 | manufacturer=manufacturer, 158 | msn=msn), \ 159 | Owner(owner) 160 | 161 | def intel_from_tail_n(tail_number): 162 | """ 163 | Gather intel from tail number 164 | 1) Owner information 165 | 2) Last changes of ownership 166 | 3) Last known position 167 | """ 168 | 169 | wiki_infos = None 170 | owner_infos = None 171 | aircraft_infos = None 172 | 173 | print("[*] Getting intel for tail number {}".format(tail_number)) 174 | 175 | # Step 1 - Gather ownership information 176 | # Cleaning up tail for register lookup according to config file 177 | tail_number = tail_number.upper() 178 | if '-' in tail_number: 179 | tail_prefix = tail_number.split('-')[0]+'-' 180 | else: 181 | tail_prefix = tail_number[0] 182 | 183 | # Gather all information together 184 | 185 | # First, from official registers 186 | try: 187 | owner_infos, aircraft_infos = tail_to_register_function[tail_prefix](tail_number) 188 | except Exception as e: 189 | printverbose("[!] Exception while calling tail_to_register: {}".format(e)) 190 | 191 | # Opensky network 192 | try: 193 | os_aircraft, os_owner = opensky(tail_number) 194 | except Exception as e: 195 | print("[!] Exception while calling opensky: {}".format(e)) 196 | # Wikipedia infos 197 | try: 198 | wiki_infos = search_wiki_commons(tail_number) 199 | except Exception as e: 200 | printwarn(e) 201 | # Last changes of ownership 202 | 203 | # Last known position 204 | icao = os_aircraft.icao.lower() 205 | api = OpenSkyApi() 206 | s = api.get_states(icao24=icao) 207 | 208 | try: 209 | if s is not None and len(s.states) > 0: 210 | last_lat = (s.states)[0].latitude 211 | last_lon = (s.states)[0].longitude 212 | os_aircraft.latitude = last_lat 213 | os_aircraft.longitude = last_lon 214 | except Exception as e: 215 | printko(e) 216 | 217 | # Detailled info (pictures etc) 218 | # TODO 219 | 220 | # Merge infos and return them 221 | try: 222 | if aircraft_infos != None: 223 | for attr, value in aircraft_infos.__dict__.items(): 224 | if value == None and getattr(os_aircraft, attr) is not None: 225 | setattr(aircraft_infos, attr, getattr(os_aircraft, attr)) 226 | else: 227 | aircraft_infos = os_aircraft 228 | 229 | except Exception as e: 230 | printko(e) 231 | finally: 232 | return owner_infos, aircraft_infos, wiki_infos 233 | 234 | def main(): 235 | # 1 - Check OSINT from ICAO 236 | # 2 - Check OSINT from tail number 237 | # 3 - Tool to convert icao to tail number 238 | parser = argparse.ArgumentParser() 239 | parser.add_argument("--action", 240 | help="Action to perform ('ICAO', 'tail', 'convert')", 241 | type=str, required=True) 242 | parser.add_argument('--tail-number', 243 | help='Tail number to lookup') 244 | # Optional arguments 245 | parser.add_argument("--icao", 246 | help="ICAO code to retrieve OSINT for", required=False) 247 | parser.add_argument("--config", 248 | help="Config file", type=str) 249 | parser.add_argument("--proxy", 250 | help="Use proxy address", type=str) 251 | parser.add_argument("--interactive", 252 | action="store_true") 253 | parser.add_argument("--verbose", 254 | action="store_true") 255 | 256 | require_group = parser.add_mutually_exclusive_group(required=False) 257 | require_group.add_argument("--country", help="country code", type=str) 258 | require_group.add_argument( 259 | "--coords", 260 | help="longitude coord in decimal format", 261 | nargs=4, 262 | type=float) 263 | 264 | args = parser.parse_args() 265 | 266 | # For storing intel recieved 267 | owner_infos = None 268 | aircraft_infos = None 269 | incident_reports = None 270 | wiki_infos = None 271 | status = None 272 | if not args.action: 273 | print("[*] No action was specified. Quit.") 274 | return 275 | else: 276 | if check_config() == False: 277 | printwarn("Not all check passed. Usage may be degraded") 278 | else: 279 | printok("All checks passed") 280 | 281 | action = args.action 282 | tail_number = args.tail_number 283 | icao = args.icao 284 | verbose = args.verbose 285 | 286 | while action != 'quit': 287 | if action == "ICAO": 288 | try: 289 | intel_from_ICAO(icao) 290 | except Exception as e: 291 | status = 'ActionICAOException' 292 | elif action == "tail": 293 | try: 294 | if tail_number == None: 295 | tail_number = input("Enter tail number to lookup: ") 296 | 297 | owner_infos, aircraft_infos, wiki_infos = intel_from_tail_n(tail_number) 298 | incident_reports = search_incidents(tail_number, args.verbose) 299 | except Exception as e: 300 | status = 'IncidentSearchException' 301 | 302 | status = 'Done' 303 | elif action == "convert": 304 | convert_US_ICAO_to_tail() 305 | status = 'Done' 306 | elif action == "monitor": 307 | if args.verbose == False: 308 | os.system('clear') 309 | print("[*] Monitor aircraft mode") 310 | if icao is None: 311 | icao = input("Enter icao number: ") 312 | monitor(icao) 313 | 314 | 315 | 316 | # Exits context (deselection of tail_numer or ICAO etc) 317 | elif action == 'exit': 318 | tail_number = None 319 | owner_infos = None 320 | aircraft_infos = None 321 | status = 'Waiting for action' 322 | else: 323 | print("[!] Unknown action. Try again") 324 | action = input("Enter valid action [ICAO, tail, convert, monitor, exit, quit]") 325 | 326 | # Print retrieved intel 327 | if args.verbose == False: 328 | os.system('clear') 329 | print("==========================================") 330 | print("Current Status: "+bcolors.OKAY+"[{}]".format(status)+bcolors.STOP) 331 | print("Last action: {}".format(action)) 332 | print("Current tail: {}".format(tail_number)) 333 | print("==========================================") 334 | print("✈️ Aircraft infos:") 335 | print(aircraft_infos) 336 | print("🧍 Owner infos") 337 | print(owner_infos) 338 | 339 | if incident_reports is not None: 340 | print("💥 Incident reports") 341 | print("\t{}".format(incident_reports)) 342 | 343 | if wiki_infos: 344 | print("📖 Wikipedia informations") 345 | print("\t{}".format(wiki_infos)) 346 | 347 | tail_number = None 348 | owner_infos = None 349 | aircraft_infos = None 350 | action = input('New Action [ICAO, tail,'\ 351 | 'convert, monitor, exit, quit] ({}):'\ 352 | .format(tail_number)) 353 | 354 | quit() 355 | 356 | if __name__ == "__main__": 357 | try: 358 | main() 359 | except KeyboardInterrupt as e: 360 | quit() 361 | -------------------------------------------------------------------------------- /geoloc.py: -------------------------------------------------------------------------------- 1 | class Coordinates: 2 | def __init__(self, latitude, longitude): 3 | self.latitude = latitude 4 | self.longitude = longitude 5 | def __repr__(self): 6 | return "%s;%s" % (self.latitude, self.longitude) 7 | def __add__(self, other): 8 | return Coordinates(self.latitude + other.latitude, 9 | self.longitude + other.longitude) 10 | 11 | 12 | class Area: 13 | def __init__(self, coord1, coord2): 14 | self.corner_1 = coord1 15 | self.corner_2 = coord2 16 | 17 | def __repr__(self): 18 | return "%s TO %s" % (self.corner_1, self.corner_2) 19 | 20 | # Size of area is one side times the other 21 | def __size__(self): 22 | return corner_1.latitude - corner_2.latitude * corner_1.longitude - corner_2.longitude 23 | 24 | 25 | class Path(list): 26 | def __init__(self): 27 | return self.list() 28 | 29 | def __repr__(self): 30 | # Calculate scale for cli representation 31 | pass 32 | -------------------------------------------------------------------------------- /investigation_authorities.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import time 3 | from bs4 import BeautifulSoup 4 | 5 | def search_incidents(tail_n, is_verbose): 6 | r = requests.get('https://aviation-safety.net/database/registration/regsearch.php?regi={}'.format(tail_n)) 7 | 8 | if r.status_code == 200: 9 | soup= BeautifulSoup(r.content, 'html.parser') 10 | td = soup.find('span', {'class': 'nobr'}) 11 | if td: 12 | r = requests.get('https://aviation-safety.net'+td.find('a')['href']) 13 | return r.url 14 | if r.status_code == 403: 15 | return 'HTTP 403 while retriving incidents' 16 | return None 17 | -------------------------------------------------------------------------------- /libs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/n0skill/AVOSINT/0636ef6caa73140c7cac1e5762bbaca7d14086b8/libs/__init__.py -------------------------------------------------------------------------------- /libs/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/n0skill/AVOSINT/0636ef6caa73140c7cac1e5762bbaca7d14086b8/libs/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /libs/__pycache__/display.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/n0skill/AVOSINT/0636ef6caa73140c7cac1e5762bbaca7d14086b8/libs/__pycache__/display.cpython-36.pyc -------------------------------------------------------------------------------- /libs/__pycache__/geoloc.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/n0skill/AVOSINT/0636ef6caa73140c7cac1e5762bbaca7d14086b8/libs/__pycache__/geoloc.cpython-36.pyc -------------------------------------------------------------------------------- /libs/__pycache__/planes.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/n0skill/AVOSINT/0636ef6caa73140c7cac1e5762bbaca7d14086b8/libs/__pycache__/planes.cpython-36.pyc -------------------------------------------------------------------------------- /libs/display.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | 5 | class Display: 6 | header = "Interactive interface for AVOSINT" 7 | titles = "Tail #\t Position\t Heading\t Owner\t Is_hovering\t " 8 | last_res = [] 9 | def __init__(self): 10 | self.selected_index = 0 11 | self.i = 0 12 | 13 | os.system('clear') 14 | print(Display.header) 15 | print('Initializing display') 16 | # TODO: start a thread for interaction and non blocking inputs 17 | return 18 | 19 | def loading(self): 20 | os.system('clear') 21 | print(Display.header + '\t loading planes from area...') 22 | print(Display.titles) 23 | for ind, plane in enumerate(Display.last_res): 24 | selector = '' 25 | if ind == self.selected_index: 26 | selector = '<--' 27 | print(plane) 28 | 29 | def update(self, plane_list): 30 | upd_chr = '' 31 | net_chr = '' 32 | os.system('clear') 33 | print(Display.header) 34 | print(Display.titles) 35 | for ind, plane in enumerate(plane_list): 36 | selector = '' 37 | if ind == self.selected_index: 38 | selector = '<--' 39 | print(plane) 40 | Display.last_res = plane_list 41 | time.sleep(1) 42 | -------------------------------------------------------------------------------- /libs/flight_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/n0skill/AVOSINT/0636ef6caa73140c7cac1e5762bbaca7d14086b8/libs/flight_data.py -------------------------------------------------------------------------------- /libs/icao_converter.py: -------------------------------------------------------------------------------- 1 | base9 = '123456789' # The first digit (after the "N") is always one of these. 2 | base10 = '0123456789' # The possible second and third digits are one of these. 3 | # Note that "I" and "O" are never used as letters, to prevent confusion with "1" and "0" 4 | base34 = 'ABCDEFGHJKLMNPQRSTUVWXYZ0123456789' 5 | icaooffset = 0xA00001 # The lowest possible number, N1, is this. 6 | b1 = 101711 # basis between N1... and N2... 7 | b2 = 10111 # basis between N10.... and N11.... 8 | 9 | 10 | 11 | def suffix(rem): 12 | """ Produces the alpha(numeric) suffix from a number 0 - 950 """ 13 | if rem == 0: 14 | suf = '' 15 | else: 16 | if rem <= 600: #Class A suffix -- only letters. 17 | rem = rem - 1 18 | suf = base34[rem // 25] 19 | if rem % 25 > 0: 20 | suf = suf + base34[rem % 25 - 1]# second class A letter, if present. 21 | else: #rems > 600 : First digit of suffix is a number. Second digit may be blank, letter, or number. 22 | rem = rem - 601 23 | suf = base10[rem // 35] 24 | if rem % 35 > 0: 25 | suf = suf + base34[rem % 35 - 1] 26 | return suf 27 | 28 | def enc_suffix(suf): 29 | """ Produces a remainder from a 0 - 2 digit suffix. 30 | No error checking. Using illegal strings will have strange results.""" 31 | if len(suf) == 0: 32 | return 0 33 | r0 = base34.find(suf[0]) 34 | if len(suf) == 1: 35 | r1 = 0 36 | else: 37 | r1 = base34.find(suf[1]) + 1 38 | if r0 < 24: # first char is a letter, use base 25 39 | return r0 * 25 + r1 + 1 40 | else: # first is a number -- base 35. 41 | return r0 * 35 + r1 - 239 42 | 43 | def icao_to_tail(icao): 44 | if (icao < 0) or (icao > 0xadf7c7): 45 | return "Undefined" 46 | icao = icao - icaooffset 47 | d1 = icao // b1 48 | nnum = 'N' + base9[d1] 49 | r1 = icao % b1 50 | if r1 < 601: 51 | nnum = nnum + suffix(r1) # of the form N1ZZ 52 | else: 53 | d2 = (r1 - 601) // b2 # find second digit. 54 | nnum = nnum + base10[d2] 55 | r2 = (r1 - 601) % b2 # and residue after that 56 | if r2 < 601: # No third digit. (form N12ZZ) 57 | nnum = nnum + suffix(r2) 58 | else: 59 | d3 = (r2 - 601) // 951 # Three-digits have extended suffix. 60 | r3 = (r2 - 601) % 951 61 | nnum = nnum + base10[d3] + suffix(r3) 62 | return nnum 63 | 64 | def tail_to_icao(tail): 65 | if tail[0] != 'N': 66 | return -1 67 | icao = icaooffset 68 | icao = icao + base9.find(tail[1]) * b1 69 | if len(tail) == 2: # simple 'N3' etc. 70 | return icao 71 | d2 = base10.find(tail[2]) 72 | if d2 == -1: # Form N1A 73 | icao = icao + enc_suffix(tail[2:4]) 74 | return icao 75 | else: # Form N11... or N111.. 76 | icao = icao + d2 * b2 + 601 77 | d3 = base10.find(tail[3]) 78 | if d3 > -1: #Form N111 Suffix is base 35. 79 | icao = icao + d3 * 951 + 601 80 | icao = icao + enc_suffix(tail[4:6]) 81 | return icao 82 | else: #Form N11A 83 | icao = icao + enc_suffix(tail[3:5]) 84 | return icao 85 | -------------------------------------------------------------------------------- /libs/planes.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from .geoloc import * 3 | import requests 4 | from bs4 import BeautifulSoup 5 | import json 6 | 7 | # Decoders for countries here 8 | from .registers import * 9 | 10 | class Owner: 11 | def __init__(self, name, street, city, zip_code, country): 12 | self.name = name 13 | self.street = street 14 | self.city = city 15 | self.zip_code = zip_code 16 | self.country = country 17 | 18 | def __repr__(self): 19 | return "%s %s %s %s %s" % (self.name, self.street, self.city, self.zip_code, self.country) 20 | 21 | def __str__(self): 22 | return self.__repr__() 23 | 24 | # TODO: move location stuff to another class "realtime plane" 25 | class Craft: 26 | def __init__(self, webi, numb, call, latitude, longitude, craft_type=None, origin=None, destination=None, altitude=None): 27 | self.coords = Coordinates(latitude, longitude) 28 | self.webi = webi 29 | self.numb = numb 30 | self.call = call 31 | self.origin = origin 32 | self.destination = destination 33 | self.altitude = altitude 34 | self.owner = self.get_owner() 35 | 36 | 37 | def __str__(self): 38 | return self.__repr__() 39 | 40 | def __repr__(self): 41 | return "%s\t%s\t%s\t%s\t%s" % (self.numb, self.call, self.coords, self.altitude, str(self.owner)) 42 | 43 | def get_owner(self): 44 | if self.numb.startswith('HB-'): 45 | return CH(self.numb) 46 | elif self.numb.startswith('F-'): 47 | return FR(self.numb) 48 | elif self.numb.startswith('TF'): 49 | return IS(self.numb) 50 | elif self.numb.startswith('N'): 51 | return US(self.numb) 52 | elif self.numb.startswith('OO-'): 53 | return BE(self.numb) 54 | elif self.numb.startswith('OE-'): 55 | return AT(self.numb) 56 | elif self.numb.startswith('SE-'): 57 | return SW(self.numb) 58 | elif self.numb.startswith('OK-'): 59 | return CZ(self.numb) 60 | elif self.numb.startswith('G-'): 61 | return UK(self.numb) 62 | elif self.numb.startswith('EI-'): 63 | return IE(self.numb) 64 | elif self.numb.startswith('M-'): 65 | return IM(self.numb) 66 | elif self.numb.startswith('I-'): 67 | return IT(self.numb) 68 | elif self.numb.startswith('C-'): 69 | return CA(self.numb) 70 | elif self.numb.startswith('YR-'): 71 | return RO(self.numb) 72 | elif self.numb.startswith('VH-'): 73 | return AU(self.numb) 74 | elif self.numb.startswith('9A-'): 75 | return HR(self.numb) 76 | elif self.numb.startswith('9V-'): 77 | return SG(self.numb) 78 | elif self.numb.startswith('ZK-'): 79 | return NZ(self.numb) 80 | elif self.numb.startswith('PP-') or self.numb.startswith('PR-') or self.numb.startswith('PS-') or self.numb.startswith('PT-') or self.numb.startswith('PU-'): 81 | return BR(self.numb) 82 | elif self.numb.startswith('D-'): 83 | return DE(self.numb) 84 | elif self.numb.startswith('UR-'): 85 | return UA(self.numb) 86 | elif self.numb.startswith('HS-') or self.numb.startswith('U-'): 87 | return TH(self.numb) 88 | else: 89 | if self.numb != '': 90 | raise NotImplementedError(f'Agency not implemented for {self.numb}') 91 | raise Exception('No tail number found') 92 | 93 | 94 | 95 | def get_path(self): 96 | return [] 97 | #j = getjson(flight_data_src+self.name) 98 | 99 | class RT_craft(Craft): 100 | pass 101 | -------------------------------------------------------------------------------- /monitor.py: -------------------------------------------------------------------------------- 1 | from opensky_api import OpenSkyApi 2 | import time 3 | import math 4 | import numpy as np 5 | import os 6 | def monitor(icao): 7 | positions = [] 8 | api = OpenSkyApi() 9 | accumulated_angle = 0.0 10 | old_alpha = 0 11 | url = 'https://globe.adsbexchange.com/?icao={}'.format(icao) 12 | hovering = False 13 | icao = icao.lower() 14 | 15 | while True: 16 | s = api.get_states(icao24=icao) 17 | if s is not None and len(s.states) > 0: 18 | positions.append( 19 | np.array([ 20 | (s.states)[0].latitude, 21 | (s.states)[0].longitude 22 | ]) 23 | ) 24 | # Computes angle between two vectors: 25 | # v1 between last and penultimate position 26 | # v2 between penultimate and antepenultimate position 27 | if len(positions) >= 3: 28 | v1 = positions[-1] - positions[-2] 29 | v2 = positions[-2] - positions[-3] 30 | angle = np.degrees(np.math.atan2(np.linalg.det([v1,v2]),np.dot(v1,v2))) 31 | if not math.isnan(angle): 32 | accumulated_angle += angle 33 | if math.fabs(accumulated_angle) > 360: 34 | hovering = True 35 | else: 36 | hovering = False 37 | os.system('clear') 38 | print("==========================================") 39 | print("AVOSINT - Monitoring 👀") 40 | print("Monitoring URL {}".format(url)) 41 | print("==========================================") 42 | if len(positions) > 0: 43 | print("🌎 Last known position:\t{}".format(positions[-1])) 44 | print("🕴️ Is it hovering:\t{}".format(hovering)) 45 | print("🧭 Accumulated angle:\t{}".format(accumulated_angle)) 46 | time.sleep(2) 47 | 48 | 49 | -------------------------------------------------------------------------------- /parsr.conf: -------------------------------------------------------------------------------- 1 | { 2 | "version": 0.9, 3 | "extractor": { 4 | "pdf": "pdfjs", 5 | "ocr": "tesseract", 6 | "language": ["eng", "fra"] 7 | }, 8 | "cleaner": [ 9 | "out-of-page-removal", 10 | "whitespace-removal", 11 | "redundancy-detection", 12 | "table-detection", 13 | ["header-footer-detection", { "maxMarginPercentage": 15 }], 14 | ["reading-order-detection", { "minColumnWidthInPagePercent": 15 }], 15 | "page-number-detection" 16 | ], 17 | "output": { 18 | "granularity": "word", 19 | "includeMarginals": false, 20 | "includeDrawings": false, 21 | "formats": { 22 | "json": true, 23 | "text": false, 24 | "csv": false, 25 | "markdown": false, 26 | "pdf": false 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /registers.json: -------------------------------------------------------------------------------- 1 | { 2 | "CH": { 3 | "name": "BAZL", 4 | "website": "bazl.admin.ch", 5 | "data_url": "https://app02.bazl.admin.ch/web/bazl-backend/lfr", 6 | "data_type": "json", 7 | "request_type": "POST", 8 | "headers": { 9 | "Pragma": "no-cache", 10 | "Origin": "https://app02.bazl.admin.ch/web/bazl/fr/", 11 | "Content-Type": "application/json", 12 | "Referer": "https://app02.bazl.admin.ch/web/bazl/fr/" 13 | }, 14 | "data": "{\"page_result_limit\":10,\n\"current_page_number\":1,\n\"sort_list\":\"registration\",\n\"language\":\"fr\",\n\"queryProperties\":{\"registration\":\"{{TAILN}}\",\n\"aircraftStatus\":[\"Registered\",\"Reserved\",\"Reservation Expired\",\"Registration in Progress\"]\n }}", 15 | "tail_index_for_search": 3 16 | }, 17 | "DK": { 18 | "name": "Denmark Registry", 19 | "website": "http://www.oy-reg.dk/register/", 20 | "data_url": "http://www.oy-reg.dk/register/?reg=", 21 | "data_type": "html", 22 | "request_type": "GET", 23 | "tail_index_for_search": 3, 24 | "infos": ["pictures"] 25 | }, 26 | "TH": { 27 | "name": "Thailand register", 28 | "website": "TODO", 29 | "data_url": "TODO", 30 | "data_type": "xlsx", 31 | "request_type": "GET", 32 | "tail_index_for_search": 3, 33 | "infos": ["pictures"] 34 | 35 | }, 36 | "UK": { 37 | "name": "UK CAA", 38 | "website": "caa.co.uk", 39 | "data_url": "" 40 | }, 41 | "LV": { 42 | "name": "Latvian CAA", 43 | "website": "https://www.caa.gov.lv/", 44 | "data_url": "https://www.caa.gov.lv/lv/media/1546/download/", 45 | "data_type": "xlsx", 46 | "request_type": "GET", 47 | "tail_index_for_search": 0 48 | }, 49 | "BA": { 50 | "name": "Bosinan CAA", 51 | "website": "http://bhdca.gov.ba", 52 | "data_url": "http://bhdca.gov.ba/english/dokumenti/airworthiness/BiH%20Aircraft%20Register_eng.pdf", 53 | "data_type": "pdf", 54 | "request_type": "GET", 55 | "tail_index_for_search": 0 56 | }, 57 | "HR": { 58 | "name": "HRVATSKI REGISTAR CIVILNIH ZRAKOPLOVA", 59 | "website": "ccaa.hr", 60 | "data_url": "https://www.ccaa.hr/file/144803bdf4c520b5de211e8ae74a33d14f10", 61 | "request_type": "GET", 62 | "data_type": "pdf", 63 | "tail_index_for_search": 3 64 | }, 65 | "CY": { 66 | "name": "Department of Civil Aviation", 67 | "website": "mcw.gov.cy", 68 | "data_url": "http://www.mcw.gov.cy/mcw/dca/dca.nsf/All/E23B9BBB43F36720C2258714002C62FD/$file/AIRCRAFT%20REGISTER%20%2030%20JUN%202021%20public%20REVISED.pdf", 69 | "request_type": "GET", 70 | "data_type": "pdf", 71 | "tail_index_for_search": 0 72 | }, 73 | "MV": { 74 | "name": "", 75 | "website": "http://www.aviainfo.gov.mv/", 76 | "data_url": "http://www.aviainfo.gov.mv/downloads/airworthiness/registers/civil_aircraft_register.pdf", 77 | "request_type": "GET", 78 | "data_type": "pdf", 79 | "tail_index_for_search": 0 80 | }, 81 | "CA": { 82 | "name": "Canadian CAA", 83 | "website": "https://gc.ca", 84 | "data_url": "http://wwwapps.tc.gc.ca/saf-sec-sur/2/ccarcs-riacc/RchSimpRes.aspx?cn=%7c%7c&mn=%7c%7c&sn=%7c%7c&on=%7c%7c&m=%7c{{TAILN}}%7c&rfr=RchSimp.aspx", 85 | "request_type": "GET", 86 | "data_type": "html", 87 | "tail_index_for_search": 2 88 | }, 89 | "GG": { 90 | "name": "Guernsey CAA", 91 | "website": "https://www.2-reg.com/", 92 | "data_url": "https://www.2-reg.com/wp-content/uploads/2022/03/Register_20220301.pdf", 93 | "request_type": "GET", 94 | "data_type": "pdf", 95 | "tail_index_for_search": 0 96 | }, 97 | "MT": { 98 | "name": "Malta CAA", 99 | "website": "https://www.transport.gov.mt/", 100 | "data_url": "https://www.transport.gov.mt/Query-Registration-24-Feb-2022.pdf-f7397", 101 | "request_type": "GET", 102 | "data_type": "pdf", 103 | "tail_index_for_search": 0 104 | }, 105 | "CAY": { 106 | "name": "Cayman islands", 107 | "website": "https://www.caacayman.com", 108 | "data_url": "https://www.caacayman.com/wp-content/uploads/pdf/Active%20Aircraft%20Register.pdf", 109 | "request_type": "GET", 110 | "data_type": "pdf", 111 | "tail_index_for_search": 0 112 | }, 113 | "MD": { 114 | "name": "Moldova", 115 | "website": "https://www.caa.md/", 116 | "data_url": "https://www.caa.md/storage/upload/cms/pages//tmp/phpoPGSra/Registrul%20Aerian%20al%20RM%2028.01.2022.pdf", 117 | "request_type": "GET", 118 | "data_type": "pdf", 119 | "tail_index_for_search": 0, 120 | "infos": [""] 121 | }, 122 | "CZ": { 123 | "name": "Letecký rejstřík", 124 | "website": "https://lr.caa.cz/api/avreg/filtered?start=0&length=6000&search=&order=", 125 | "data_url": "https://lr.caa.cz/api/avreg/filtered?start=0&length=6000&search=&order=", 126 | "request_type": "GET", 127 | "data_type": "json", 128 | "tail_index_for_search": 0, 129 | "infos": [""] 130 | }, 131 | "BZ": { 132 | "name": "BELIZE DEPARTMENT OF CIVIL AVIATION", 133 | "website": "https://www.civilaviation.gov.bz/", 134 | "data_url": "https://www.civilaviation.gov.bz/images/downloads/BDCA-Aircraft-Register.pdf", 135 | "request_type": "GET", 136 | "data_type": "pdf", 137 | "tail_index_for_search": 0, 138 | "infos": [""] 139 | }, 140 | "VE": { 141 | "name": "Bellingcat's venezuelian aircraft ", 142 | "website": "1Kgu0uoXLGhoCHUyMgDsdqtW_XH3fvIglkVx5EdJhRnU", 143 | "data_url": "https://docs.google.com/spreadsheets/d/1Kgu0uoXLGhoCHUyMgDsdqtW_XH3fvIglkVx5EdJhRnU/export?format=xlsx", 144 | "request_type": "GET", 145 | "data_type": "xlsx", 146 | "tail_index_for_search": 0, 147 | "infos": [""] 148 | }, 149 | "SG": { 150 | "name": "Singapore Civil Aircraft Authority", 151 | "website": "https://www.caas.gov.sg/", 152 | "data_url": "https://www.caas.gov.sg/docs/default-source/docs---srg/fs/approval-listings/singapore-registered-aircraft-engine-nos---feb-2022.xlsx", 153 | "request_type": "GET", 154 | "data_type": "xlsx", 155 | "tail_index_for_search": 0, 156 | "infos": [""] 157 | }, 158 | "SC": { 159 | "name": "Seychelles CAA", 160 | "website": "https://www.scaa.sc/", 161 | "data_url": "https://www.scaa.sc/index.php/regulatory/e-registers/aircraft-civil-register", 162 | "request_type": "GET", 163 | "data_type": "html", 164 | "tail_index_for_search": 0, 165 | "infos": [""] 166 | }, 167 | "ES": { 168 | "name": "Ministerio de transportes, movilidad y agenda urbana", 169 | "website": "seguridadaerea.gob.es", 170 | "data_url": "https://www.seguridadaerea.gob.es/sites/default/files/aeronaves_inscritas.pdf", 171 | "request_type": "GET", 172 | "data_type": "pdf", 173 | "tail_index_for_search": 0, 174 | "infos": [""] 175 | }, 176 | "ME": { 177 | "name": "Agencija za civilno vazduhoplovstvo", 178 | "website": "https://caa.me/", 179 | "data_url": "https://www.caa.me/en/{{tailn}}", 180 | "request_type": "GET", 181 | "data_type": "html", 182 | "tail_index_for_search": 0, 183 | "infos": [""] 184 | }, 185 | "EE": { 186 | "name": "Estonian CAA", 187 | "website": "https://transpordiamet.ee/", 188 | "data_url": "https://transpordiamet.ee/maa-vee-ohusoiduk/lennundustehnika/ohusoidukite-register", 189 | "request_type": "GET", 190 | "data_type": { 191 | "html": { 192 | "table": 193 | { 194 | "class":"table table-bordered table-bordered-vp" 195 | } 196 | } 197 | }, 198 | "tail_index_for_search": 0, 199 | "infos": [""] 200 | }, 201 | "BM": { 202 | "name": "Bermuda Aircraft Registry", 203 | "website": "https://www.bcaa.bm/", 204 | "data_url": "https://www.bcaa.bm/sites/default/files/Web%20Docs/Notices_BACs_OTARs/Bermuda%20Aircraft%20Registry%20-%20Russian%20Air%20Operators.pdf", 205 | "request_type": "GET", 206 | "data_type": "pdf", 207 | "tail_index_for_search": 0, 208 | "infos": [""] 209 | }, 210 | "KZ": { 211 | "name": "Bellingcats' KZ spreadsheet", 212 | "website": "", 213 | "data_url": "https://docs.google.com/spreadsheets/d/1oXU0OPT5XYEDA1C1juHwsZ_xuBBGFQ4LC8g-bq9tH9k/export?format=xlsx", 214 | "request_type": "GET", 215 | "data_type": "xlsx", 216 | "tail_index_for_search": 0, 217 | "infos": [""] 218 | }, 219 | "BY": { 220 | "name": "Aeroflight Aviation Enthusiasts Cloud - Belarus", 221 | "website": "", 222 | "data_url": "https://docs.google.com/spreadsheets/d/1iIW69IaIGE-7Un6kKpVV8ERo1SFuRd2okUp2289wKlA/export?format=xlsx", 223 | "request_type": "GET", 224 | "data_type": "xlsx", 225 | "tail_index_for_search": 0, 226 | "infos": [""] 227 | }, 228 | "BG": { 229 | "name": "Bulgaria", 230 | "website": "https://www.caa.bg", 231 | "data_url": "https://www.caa.bg/sites/default/files/upload/documents/2021-11/Aircraft_Register%2026_11_2021.xlsx", 232 | "request_type": "GET", 233 | "data_type": "xlsx", 234 | "tail_index_for_search": 0, 235 | "infos": [""] 236 | } 237 | 238 | } 239 | -------------------------------------------------------------------------------- /registers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import json 4 | import requests 5 | import ssl 6 | import calendar 7 | import urllib 8 | 9 | from bs4 import BeautifulSoup 10 | from requests.adapters import HTTPAdapter 11 | from requests.packages.urllib3.poolmanager import PoolManager 12 | from requests.packages.urllib3.util.ssl_ import create_urllib3_context 13 | from urllib3.exceptions import NewConnectionError 14 | from openpyxl import load_workbook 15 | from parsr_client import ParsrClient as client 16 | from pprint import pprint 17 | from aircraft import Aircraft 18 | 19 | from urllib3.exceptions import InsecureRequestWarning 20 | # Suppress only the single warning from urllib3 needed. 21 | requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) 22 | 23 | class TLSv1Adapter(HTTPAdapter): 24 | """"Transport adapter" that allows us to use TLSv1.""" 25 | 26 | def init_poolmanager(self, connections, maxsize, block=False): 27 | self.poolmanager = PoolManager( 28 | num_pools=connections, 29 | maxsize=maxsize, 30 | block=block, 31 | ssl_version=ssl.PROTOCOL_TLSv1) 32 | 33 | 34 | class NODHAdapter(HTTPAdapter): 35 | def init_poolmanager(self, *args, **kwargs): 36 | context = create_urllib3_context( 37 | ciphers='ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:!DH') 38 | kwargs['ssl_context'] = context 39 | return super(NODHAdapter, self).init_poolmanager(*args, **kwargs) 40 | 41 | class NoIntelException(Exception): 42 | """ Raised when no info has been found """ 43 | pass 44 | 45 | ################################################# 46 | # Registers.py 47 | # Goal: Gather data from various agencies 48 | # depending on country code of tail number 49 | # 50 | # Returns tuple: (owner infos, aircraft infos) 51 | # 52 | 53 | 54 | class Owner: 55 | def __init__(self): 56 | self.name = 'TBD' 57 | self.country= 'TBD' 58 | 59 | def __init__(self, name, street='Unknown', city='Unknown', zip_code='Unknown', country='Unknown'): 60 | self.name = name 61 | self.street = street 62 | self.city = city 63 | self.zip_code = zip_code 64 | self.country = country 65 | 66 | def __repr__(self): 67 | return u""" 68 | Name: %s 69 | Street: %s 70 | City: %s 71 | ZIP: %s 72 | Country: %s 73 | """ % (self.name, self.street, self.city, self.zip_code, self.country) 74 | 75 | def __str__(self): 76 | return self.__repr__() 77 | 78 | class DataSource: 79 | def __init__(self, url, src_type, request_type, is_secure, http_data='', headers=''): 80 | self.url = url 81 | self.src_type = src_type 82 | self.request_type = request_type 83 | self.is_secure = is_secure 84 | self.data = http_data 85 | self.headers = headers 86 | 87 | if self.headers == '': 88 | self.headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15'} 89 | 90 | def gather(self, tail_n): 91 | """ 92 | Gathers data from the configured DataSource for a register 93 | Returns a parsable object: 94 | - workbook for xlsx sources 95 | - json object for json sources 96 | - json object for pdf sources (using parsr) 97 | - raw request response content when content type could not be determined 98 | - BeautifulSoup soup for html sources 99 | """ 100 | 101 | 102 | # Match and replace tail number where appropriate 103 | if self.data and '{{TAILN}}' in self.data: 104 | print('[*] Replacing {{TAILN}} from data with acutal tail number') 105 | self.data = self.data.replace('{{TAILN}}', tail_n) 106 | 107 | if '{{TAILN}}' in self.url: 108 | print('[*] Replacing {{TAILN}} from url with actual tail number') 109 | self.url = self.url.replace('{{TAILN}}', tail_n) 110 | 111 | if '{{tailn}}' in self.url: 112 | print('[*] Replacing {{tailn}} from url with actual tail number') 113 | self.url = self.url.replace('{{tailn}}', tail_n.lower()) 114 | 115 | print('[*] Gathering info from url', self.url) 116 | 117 | # For ressources that need to be fetched 118 | # using an HTTP GET request 119 | okay = False 120 | 121 | if self.request_type == 'API': 122 | from google_auth_oauthlib.flow import InstalledAppFlow 123 | from googleapiclient.discovery import build 124 | from googleapiclient.errors import HttpError 125 | 126 | # If modifying these scopes, delete the file token.json. 127 | SCOPES = ['https://www.googleapis.com/auth/spreadsheets.readonly'] 128 | # The ID and range of a sample spreadsheet. 129 | SAMPLE_SPREADSHEET_ID = '1Kgu0uoXLGhoCHUyMgDsdqtW_XH3fvIglkVx5EdJhRnU' 130 | SAMPLE_RANGE_NAME = 'Class Data!A2:E' 131 | # Call the Sheets API 132 | service = build('sheets', 'v4', credentials=creds) 133 | sheet = service.spreadsheets() 134 | result = sheet.values().get(spreadsheetId=SAMPLE_SPREADSHEET_ID, 135 | range=SAMPLE_RANGE_NAME).execute() 136 | values = result.get('values', []) 137 | 138 | if not values: 139 | print('No data found.') 140 | return 141 | 142 | print('Name, Major:') 143 | for row in values: 144 | # Print columns A and E, which correspond to indices 0 and 4. 145 | print('%s, %s' % (row[0], row[4])) 146 | return 0 147 | 148 | if self.request_type == 'GET': 149 | r = requests.get(self.url) 150 | elif self.request_type == 'POST': 151 | r = requests.post(self.url, data=self.data, headers=self.headers) 152 | if r.status_code == 200: 153 | okay = True 154 | # Online html 155 | if self.src_type == 'html' or 'html' in self.src_type: 156 | soup = BeautifulSoup(r.content, 'html.parser') 157 | # if html subtypes are present 158 | if 'html' in self.src_type and type(self.src_type) is dict: 159 | soup = BeautifulSoup(r.content, 'html.parser') 160 | elem = self.src_type.get('html') 161 | f = None 162 | for key, value in elem.items(): 163 | f = soup.find(key, value) 164 | return f 165 | # Else, return the whole soup 166 | else: 167 | return soup 168 | 169 | # Online jso 170 | if self.src_type == 'json': 171 | j = json.loads(r.content) 172 | return j 173 | 174 | # Online xls 175 | elif self.src_type == 'xlsx': 176 | with open('/tmp/book.xlsx', 'wb') as f: 177 | f.write(r.content) 178 | book = load_workbook( 179 | '/tmp/book.xlsx') 180 | return book 181 | # PDF file 182 | elif self.src_type == 'pdf': 183 | parsr = client('localhost:3001') 184 | with open('/tmp/avosint.pdf', 'wb') as f: 185 | f.write(r.content) 186 | 187 | job = parsr.send_document( 188 | file_path='/tmp/avosint.pdf', 189 | config_path='./parsr.conf', 190 | document_name='Sample File2', 191 | wait_till_finished=True, 192 | save_request_id=True, 193 | silent=False) 194 | j = parsr.get_json() 195 | return j 196 | else: 197 | print("[WRN]", r.status_code) 198 | else: 199 | print("[!] Error {} when retrieving from\n[!] URL: {}"\ 200 | .format(r.status_code, self.url)) 201 | print("[!] DataSource error while gathering information.") 202 | print("[!] Try to gather using session.") 203 | s = requests.session() 204 | r = s.get(self.url, headers=self.headers) 205 | if self.src_type == 'html' or 'html' in self.src_type: 206 | r = requests.get(self.url) 207 | if r.status_code == 200: 208 | soup = BeautifulSoup(r.content, 'html.parser') 209 | # if html subtypes are present 210 | if 'html' in self.src_type and type(self.src_type) is dict: 211 | soup = BeautifulSoup(r.content, 'html.parser') 212 | elem = self.src_type.get('html') 213 | f = None 214 | for key, value in elem.items(): 215 | f = soup.find(key, value) 216 | return f 217 | # Else, return the whole soup 218 | else: 219 | return soup 220 | else: 221 | print('[WRN]', r.url) 222 | 223 | if self.src_type == 'json': 224 | if self.request_type=="GET": 225 | r = requests.get(self.url+tail_n,headers=self.headers) 226 | if self.request_type=="POST": 227 | r = requests.post(self.url, data=self.data, headers=self.headers) 228 | if r.status_code == 200: 229 | j = json.loads(r.content) 230 | return j 231 | 232 | # Online xls 233 | elif self.src_type == 'xlsx': 234 | r = requests.get(self.url) 235 | with open('/tmp/book.xlsx', 'wb') as f: 236 | f.write(r.content) 237 | book = load_workbook( 238 | '/tmp/book.xlsx') 239 | return book 240 | 241 | 242 | class Registry: 243 | def __init__(self, 244 | name, 245 | url, 246 | data_source_url, 247 | data_source_type, 248 | data_request_type, 249 | post_data='', 250 | headers='', 251 | is_secure=True, 252 | tail_start=0): 253 | 254 | self.name = name 255 | self.url = url 256 | self.data_source = DataSource( 257 | data_source_url, 258 | data_source_type, 259 | data_request_type, 260 | is_secure, 261 | post_data, 262 | headers) 263 | self.results = [] 264 | self.tail_start = tail_start 265 | 266 | def request_infos(self, tail_n): 267 | # Format tail number according to regex 268 | tail_n = tail_n[self.tail_start:] 269 | res = self.data_source.gather(tail_n) 270 | return res 271 | 272 | 273 | def register_from_config(key): 274 | f = open('./registers.json') 275 | j = json.load(f) 276 | 277 | # Getting config elements for country key 278 | config = j[key] 279 | generic_registry = Registry( 280 | config['name'], 281 | config['website'], 282 | config['data_url'], 283 | config['data_type'], 284 | config['request_type'], 285 | config['data'] if 'data' in config else None, 286 | headers=config['headers'] if 'headers' in config else None, 287 | tail_start=config['tail_index_for_search']) 288 | return generic_registry 289 | 290 | 291 | 292 | def NL(tail_n): 293 | return 294 | 295 | def CH(tail_n): 296 | """ 297 | Get information on aircraft from tail number 298 | """ 299 | SwissRegister = register_from_config("CH") 300 | jsonobj = SwissRegister.request_infos(tail_n) 301 | if len(jsonobj) == 0: 302 | print("[!][CH][{}] Error when retrieving from registry".format(tail_n)) 303 | raise NoIntelException 304 | infoarray = jsonobj[0] 305 | own_ops = infoarray.get('ownerOperators') 306 | leng = len(own_ops) 307 | name = own_ops[leng-1].get('ownerOperator') 308 | addr = own_ops[leng-1].get('address') 309 | street = addr.get('street') 310 | street_n = addr.get('streetNo') 311 | zipcode = addr.get('zipCode') 312 | city = addr.get('city') 313 | owner = Owner(name, street + ' ' + street_n, 314 | city, zipcode, "Switzerland") 315 | 316 | return owner, Aircraft(tail_n) 317 | 318 | 319 | def FR(tail_n): 320 | s = requests.session() 321 | headers = { 322 | 'Origin': 'https://immat.aviation-civile.gouv.fr', 323 | 'Referer': 'https://immat.aviation-civile.gouv.fr/immat/servlet/aeronef_liste.html' 324 | } 325 | 326 | data = [ 327 | ('FORM_DTO_ID', 'DTO_RECHERCHE_AER'), 328 | ('FORM_ACTION', 'SEARCH'), 329 | ('FORM_CHECK', 'true'), 330 | ('FORM_ROW', ''), 331 | ('CTRL_ID', '1010'), 332 | ('SUBFORM_DTC_ID', 'DTC_1'), 333 | ('$DTO_RECHERCHE_AER$PER_ID', ''), 334 | ('$DTO_RECHERCHE_AER$PER_VERSION', ''), 335 | ('$DTO_RECHERCHE_AER$PERSONNE', ''), 336 | ('$DTO_RECHERCHE_AER$MRQ_CD', tail_n), 337 | ('$DTO_RECHERCHE_AER$CNT_LIBELLE', ''), 338 | ('$DTO_RECHERCHE_AER$SI_RADIE', '1'), 339 | ('$DTO_RECHERCHE_AER$SI_RADIEcheckbox', '1'), 340 | ('$DTO_RECHERCHE_AER$SI_INSCRIT', '1'), 341 | ('$DTO_RECHERCHE_AER$SI_INSCRITcheckbox', '1'), 342 | ('$DTO_RECHERCHE_AER$CRE_NUM_SERIE', ''), 343 | ] 344 | #s.mount("https://", TLSv1Adapter()) # Add TSLV1 adapter (outdated) 345 | 346 | response = s.post('https://immat.aviation-civile.gouv.fr/immat/servlet/aeronef_liste.html', 347 | headers=headers, 348 | data=data) 349 | 350 | if response.status_code == 200: 351 | soup = BeautifulSoup(response.text, 'html.parser') 352 | bls = soup.find('a', string=tail_n) 353 | 354 | if bls is None: 355 | print('[!][FR] Could not find supplied tail number in agency registers') 356 | else: 357 | response = s.get( 358 | 'https://immat.aviation-civile.gouv.fr/immat/servlet/' + bls['href'], 359 | headers=headers) 360 | soup = BeautifulSoup(response.text, 'html.parser') 361 | bls = soup.find('a', string="Données juridiques") 362 | r = s.get('https://immat.aviation-civile.gouv.fr/immat/servlet/' + bls['href']) 363 | if r.status_code == 200: 364 | soup = BeautifulSoup(r.text, 'html.parser') 365 | bls = soup.find_all('td', {'class': "tdLigneListe"}) 366 | if len(bls) > 0: 367 | name = bls[0].text 368 | addr = bls[1].text 369 | city = bls[2].text 370 | return Owner(name, addr, city, '', 'France'), Aircraft(tail_n) 371 | else: 372 | print('[!][FR] Error while retrieving info for {}'.format(tail_n)) 373 | else: 374 | print('[!][FR] Error while retrieving info for {}'.format(tail_n)) 375 | 376 | def US(tail_n): 377 | name = '' 378 | addr = '' 379 | city = '' 380 | resp = requests.get( 381 | "https://registry.faa.gov/AircraftInquiry/Search/NNumberResult?nNumberTxt="+tail_n) 382 | if resp.status_code == 200: 383 | soup = BeautifulSoup(resp.content, 'html.parser') 384 | 385 | # Get table regarding owner information 386 | tables = soup.find_all( 387 | 'table', {'class', 'devkit-table'}) 388 | for table in tables: 389 | caption = table.find('caption', {'class', 'devkit-table-title'}) 390 | 391 | # This is the table we are interested in 392 | if caption.text == 'Registered Owner': 393 | rows = table.find_all('tr') 394 | for row in rows: 395 | cols = row.find_all('td') 396 | for col in cols: 397 | if col['data-label'] == 'Name': 398 | name = col.text 399 | elif col['data-label'] == 'Street': 400 | addr = addr + col.text 401 | elif col['data-label'] == 'City': 402 | city = col.text 403 | elif col['data-label'] == 'State': 404 | city = city + ', ' + col.text 405 | elif col['data-label'] == 'Zip Code': 406 | zip_code = col.text 407 | return Owner(name, addr, city, zip_code, 'USA'), Aircraft(tail_n) 408 | else: 409 | print("[!][{}] HTTP status code from {}"\ 410 | .format(resp.status_code, resp.url)) 411 | print('[!] Error while retrieving from US') 412 | return None, None 413 | 414 | 415 | def IS(tail_n): 416 | name = '' 417 | street = '' 418 | city = '' 419 | s = requests.session() 420 | # s.mount("https://", TLSv1Adapter()) # Outdated TSLv1 -> Needs a specific adapter 421 | req = s.get( 422 | 'https://www.icetra.is/aviation/aircraft/register?aq='+tail_n) 423 | if req.status_code == 200: 424 | soup = BeautifulSoup(req.text, 'html.parser') 425 | own = soup.find('li', {'class': 'owner'}) 426 | if own is not None: 427 | won = own.stripped_strings 428 | for i, j in enumerate(won): 429 | if i == 1: 430 | name = j 431 | elif i == 2: 432 | street = j 433 | elif i == 3: 434 | city = j 435 | if own is not None: 436 | return Owner(name, street, city, '', 'Iceland'), Aircraft(tail_n) 437 | 438 | 439 | 440 | 441 | def SW(tail_n): 442 | data = { 443 | 'selection': 'regno', 444 | 'regno': tail_n, 445 | 'part': '', 446 | 'owner': '', 447 | 'item': '', 448 | 'X-Requested-With': 'XMLHttpRequest' 449 | } 450 | s = requests.session() 451 | rep = s.post( 452 | 'https://sle-p.transportstyrelsen.se/extweb/en-gb/sokluftfartyg', data=data) 453 | if rep.status_code == 200: 454 | soup = BeautifulSoup( 455 | rep.content, features="html.parser") 456 | results_element = soup.find_all( 457 | 'label', {'class': 'owner-headline'})[0] 458 | name = results_element.parent.find( 459 | 'a', {'class': 'open-owner-page'}).text 460 | street = results_element.parent.text.split('\r\n')[ 461 | 1].strip() 462 | city = results_element.parent.text.split('\r\n')[ 463 | 2].strip() 464 | country = results_element.parent.text.split('\r\n')[ 465 | 3].strip() 466 | return Owner(name, street, city, '', country), Aircraft(tail_n) 467 | else: 468 | raise Exception('Error retrieving from SW') 469 | 470 | 471 | def AT(tail_n): 472 | tail_n = tail_n.lstrip('OE-') 473 | rep = requests.get( 474 | 'https://www.austrocontrol.at/'\ 475 | 'lfa-publish-service/v2/oenfl/'\ 476 | 'luftfahrzeuge?kennzeichen='+tail_n 477 | ) 478 | print(rep.url) 479 | 480 | if rep.status_code == 200: 481 | j = json.loads(rep.content) 482 | owner_infos = [x for x in j if x['kennzeichen'] == tail_n] 483 | name, loc_info, country = owner_infos[0]['halter'].split('\r\n') 484 | city, street = loc_info.split(', ') 485 | npa, city = city.split(' ') 486 | return Owner(name, street, city, npa, 'Austria'), Aircraft(tail_n) 487 | 488 | def AU(tail_n): 489 | """ 490 | Australia Registry 491 | 492 | """ 493 | r = requests.get( 494 | 'https://www.casa.gov.au/search-centre/aircraft-register/'+tail_n[3:]) 495 | if r.status_code == 200: 496 | soup = BeautifulSoup( 497 | r.content, features="html.parser") 498 | 499 | name = soup.find('div', 500 | {'class':'field--name-field-registration-holder'} 501 | ).text.strip('Registration holder:\n') 502 | addr = soup.find('div', 503 | {'class':'field--name-field-reg-hold-address-1'} 504 | ).text.strip('Address 1:\n') 505 | city = soup.find('div', 506 | {'class':'field--name-field-tx-reg-hold-suburb'} 507 | ).text.strip('Suburb / City:\n') 508 | return Owner(name, addr, city, '', 'Australia'), Aircraft(tail_n) 509 | raise Exception( 510 | "Could not get info from AU register") 511 | def BA(tail_n): 512 | register = register_from_config("BA") 513 | infos = register.request_infos(tail_n) 514 | for page in infos['pages']: 515 | for element in page['elements']: 516 | if element['type'] == 'table': 517 | for line in element['content']: 518 | tail_n_from_file = line['content'][1]['content'][0]['content'] 519 | if tail_n == tail_n_from_file: 520 | owner_line = line['content'][5]['content'] 521 | # Owner info 522 | owner_name = ' '.join( 523 | [owner_line[i]['content'] for i in range(0, len(owner_line))] 524 | ) 525 | return Owner(owner_name, '', '', '', '') 526 | def BE(tail_n): 527 | rep = requests.get( 528 | 'https://es.mobilit.fgov.be/aircraft-registry/'\ 529 | 'rest/aircrafts?aircraftStates=REGISTERED®istrationMark=' + 530 | tail_n + '&page=0&pageSize=10&sort=REGISTRATIONMARK&sortDirection=ascending') 531 | 532 | if rep.status_code == 200: 533 | j = json.loads(rep.text) 534 | if len(j) > 0: 535 | if j[0].get('id') != '': 536 | rep = requests.get( 537 | 'https://es.mobilit.fgov.be/aircraft-registry/rest/aircrafts/'+str(j[0].get('id'))) 538 | if rep.status_code == 200: 539 | # print(rep.text) 540 | j = json.loads(rep.text) 541 | name = j.get('stakeHolderRoleList')[0].get('name') 542 | street = j.get('stakeHolderRoleList')[ 543 | 0].get('addresses').get('street') 544 | city = j.get('stakeHolderRoleList')[ 545 | 0].get('addresses').get('city') 546 | return Owner(name, street, city, '', 'Belgium'), Aircraft(tail_n) 547 | else: 548 | return Exception("Error retrieving from BE") 549 | def BG(tail_n): 550 | try: 551 | register = register_from_config("BG") 552 | book = register.request_infos(tail_n) 553 | infos_sheet = book["Registrations LZ"] 554 | for row in infos_sheet.values: 555 | if tail_n in row: 556 | msn = row[3] 557 | owner = row[6] 558 | return Owner(owner, '', '', '', ''),\ 559 | Aircraft(tail_n, msn=msn) 560 | return None, None 561 | except Exception as e: 562 | print('[!] ', e) 563 | return None, None 564 | 565 | def BR(tail_n): 566 | r = requests.get('https://sistemas.anac.gov.br'\ 567 | '/aeronaves/cons_rab_resposta.asp?textMarca=' + tail_n) 568 | if r.status_code == 200: 569 | soup = BeautifulSoup(r.text, features="html.parser") 570 | headings = soup.find_all('th', {'scope':'row'}) 571 | for heading in headings: 572 | if 'Proprietário' in heading.text: 573 | name = heading.parent.td.text.strip() 574 | return Owner(name, '', '', '', 'Brazil'), Aircraft(tail_n) 575 | raise Exception("Could not get info from BR register. " + r.url + r.status_code ) 576 | 577 | def BY(tail_n): 578 | register = register_from_config("BY") 579 | book = register.request_infos(tail_n) 580 | for line in book['Data']: 581 | if tail_n in line: 582 | print(line) 583 | 584 | def BZ(tail_n): 585 | register = register_from_config("BZ") 586 | infos = register.request_infos(tail_n) 587 | for page in infos['pages']: 588 | print(page) 589 | for content in page['elements']: 590 | if content['type'] == 'table': 591 | for item in content['content']: 592 | tail = item['content'][0]['content'][0]['content'] 593 | if tail == tail_n: 594 | # Owner infos 595 | own = item['content'][3]['content'][0]['content'] 596 | addr = ' '.join( 597 | item['content'][4]['content']\ 598 | [i]['content']\ 599 | for i in range(0, len(item['content'][4]['content']))) 600 | addr, city = addr.split(',') 601 | 602 | # Aircraft infos 603 | manufacturer = item['content'][1]['content'][0]['content'] 604 | return Owner(own, addr, city=city), \ 605 | Aircraft(tail_n, manufacturer=manufacturer) 606 | 607 | def CA(tail_n): 608 | tail_n = tail_n.lstrip('C-') 609 | r = requests.get('https://wwwapps.tc.gc.ca/'\ 610 | 'saf-sec-sur/2/ccarcs-riacc/RchSimpRes.aspx'\ 611 | '?cn=%7c%7c&mn=%7c%7c&sn=%7c%7c&on=%7c%7c&m=%7c'+tail_n+'%7c&rfr=RchSimp.aspx') 612 | if r.status_code == 200: 613 | soup = BeautifulSoup( 614 | r.content, features='html.parser') 615 | div_owner = soup.find('div', {'id':'dvOwnerName'}) 616 | if div_owner is not None: 617 | name = div_owner.find_all('div')[1].text.strip() 618 | div_addr = soup.find('div', {'id':'divOwnerAddress'}) 619 | addr = div_addr.find_all('div')[1].text.strip() 620 | div_city = soup.find('div', {'id':'divOwnerCity'}) 621 | city = div_city.find_all('div')[1].text.strip() 622 | return Owner(name, addr, city, '', 'Canada'), Aircraft(tail_n) 623 | else: 624 | print('[!] Error retrieving from CA register. Ensure tail number exists and try again') 625 | 626 | def CZ(tail_n): 627 | SwissRegister = register_from_config("CZ") 628 | jsonobj = SwissRegister.request_infos(tail_n) 629 | for obj in jsonobj['rows']: 630 | if obj['registration_number'] == tail_n[3:]: 631 | r = requests.get('https://lr.caa.cz/api/avreg/{}'.format(obj['id'])) 632 | if r.status_code == 200: 633 | j = json.loads(r.content) 634 | name = None 635 | if len(j['owners']) > 0: 636 | name = j['owners'][0]['display_name'] 637 | serial_n = j['serial_number'] 638 | manufacturer = j['manufacturer'] 639 | return Owner(name), Aircraft(tail_n, msn=serial_n, manufacturer=manufacturer) 640 | else: 641 | print("[!] Error while searching") 642 | raise Exception("Error while searching") 643 | 644 | def UK(tail_n): 645 | data = { 646 | 'Registration': tail_n[2:] 647 | } 648 | print( 649 | data) 650 | s = requests.session() 651 | r = s.get( 652 | 'https://siteapps.caa.co.uk/g-info/') 653 | r = s.post( 654 | 'https://ginfoapi.caa.co.uk/api/aircraft/search', data=data) 655 | if r.status_code == 200: 656 | print(r.content) 657 | 658 | return Owner(name=""), Aircraft(tail_n) 659 | 660 | def IE(tail_n): 661 | headers = { 662 | 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0'} 663 | # r = requests.get('https://www.iaa.ie/commercial-aviation/aircraft-registration-2/latest-register-and-monthly-changes-1', headers=headers) 664 | url_doc = 'https://www.iaa.ie/docs/default-source/publications/aircraft-registration/30-november-2019.xlsx' 665 | r = requests.get( 666 | url_doc, headers=headers) 667 | if r.status_code == 200: 668 | with open('/tmp/book.xlsx', 'wb') as f: 669 | f.write( 670 | r.content) 671 | book = load_workbook( 672 | '/tmp/book.xlsx') 673 | for sheet in book: 674 | plane = [ 675 | row for row in sheet.values if row[0] == tail_n] 676 | if len(plane) > 0: 677 | name = plane[ 678 | 0][12] 679 | addr = plane[ 680 | 0][13] 681 | return Owner(name, addr, '', '', 'Ireland'), Aircraft(tail_n) 682 | raise Exception( 683 | 'Error retrieving from Ireland register. Tail number may be wrong') 684 | 685 | 686 | def IM(tail_n): 687 | tail_n = tail_n.lstrip('M-') 688 | r = requests.get('https://ardis.iomaircraftregistry.com/register/search?prs_rm__ptt=1&prs_rm__tt=1&prs_as__v=2&prs_rm__v1='+tail_n+'&prs_rm__pv1='+tail_n) 689 | if r.status_code == 200: 690 | soup = BeautifulSoup( 691 | r.content, features="html.parser") 692 | link = soup.find( 693 | 'td', {'id': 'prp__rid__1__cid__2'}) 694 | if link is not None: 695 | href = link.contents[0]['href'] 696 | r = requests.get( 697 | 'https://ardis.iomaircraftregistry.com'+href) 698 | if r.status_code == 200: 699 | soup = BeautifulSoup( 700 | r.content, features="html.parser") 701 | own = soup.find( 702 | 'span', {'id': 'prv__11__value'}) 703 | name, infos = own.text.split(',') 704 | if len(infos) > 1: 705 | street = infos 706 | return Owner(name, street, '', '', ''), Aircraft(tail_n) 707 | 708 | 709 | def DE(tail_n): 710 | raise NotImplementedError( 711 | 'German register not implemented yet') 712 | 713 | def IT(tail_n): 714 | headers = { 715 | 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0'} 716 | s = requests.Session() 717 | r = s.get( 718 | 'http://gavs.it/') 719 | soup = BeautifulSoup( 720 | r.content, features="html.parser") 721 | csrf = soup.find( 722 | 'input', {'name': 'csrf_test_name'}) 723 | data = { 724 | 'csrf_test_name': csrf['value'], 725 | 'registration': tail_n[2:] 726 | } 727 | 728 | r = s.post( 729 | 'https://gavs.it/rci/search_registration', 730 | data=data, 731 | headers=headers) 732 | if r.status_code == 200: 733 | record_url = r.headers['Refresh'].split(';')[1][4:] 734 | r = s.get( 735 | record_url) 736 | if r.status_code == 200: 737 | soup = BeautifulSoup( 738 | r.text, features="html.parser") 739 | tab_owners = soup.find( 740 | 'div', {'id': 'htab2'}) 741 | tab_owner = tab_owners.find_all( 742 | 'dl', {'class': 'dl-horizontal'})[0] 743 | owner_name = tab_owner.find_all( 744 | 'dd')[-1].text 745 | return Owner(owner_name, '', '', '', ''), Aircraft(tail_n) 746 | raise Exception( 747 | "Could not get info from IT register") 748 | 749 | def RO(tail_n): 750 | raise NotImplementedError( 751 | 'Romanian register is the following pdf document http://www.caa.ro/media/docs/OPERATORI_AERIENI_ROMANI_28.11.2019_rom-eng.pdf. Documents parsing not implemented yet') 752 | 753 | def HR(tail_n): 754 | raise NotImplementedError( 755 | 'Croatian register is a pdf document. The document is available at https://www.ccaa.hr/english/popis-registriranih-zrakoplova_101/') 756 | 757 | 758 | def SG(tail_n): 759 | register = register_from_config("SG") 760 | book = register.request_infos(tail_n) 761 | infos_sheet = book["Aircraft Register"] 762 | for row in infos_sheet.values: 763 | if tail_n in row: 764 | return Owner(row[4], '', '', '', ''),\ 765 | Aircraft(tail_n, manufacturer=row[5], msn=row[2]) 766 | return None, None 767 | 768 | def NZ(tail_n): 769 | r = requests.get('https://www.aviation.govt.nz/aircraft/'\ 770 | 'aircraft-registration/aircraft-register/ShowDetails/'+tail_n[3:]) 771 | 772 | if r.status_code == 200: 773 | soup = BeautifulSoup(r.text, features="html.parser") 774 | owner_info = soup.find('div', {'class': 'col-md-9'}) 775 | name = owner_info.text.strip().split('\n')[1].strip() 776 | street = owner_info.text.strip().split('\n')[2].strip() 777 | return Owner(name, street, '', '', 'New Zealand'), Aircraft(tail_n) 778 | raise Exception("Could not get info from NZ register") 779 | 780 | def UA(tail_n): 781 | r = requests.get('http://avia.gov.ua/register_of_aircraft.xls') 782 | if r.status_code == 200: 783 | with open('/tmp/register.xls', 'wb') as f: 784 | f.write(r.content) 785 | book = xlrd.open_workbook('/tmp/register.xls') 786 | sheet_names = book.sheet_names() 787 | xl_sheet = book.sheet_by_name(sheet_names[0]) 788 | for i in range(0, xl_sheet.nrows): 789 | if xl_sheet.row(i)[2].value == tail_n: 790 | name = xl_sheet.row(i)[9].value 791 | return Owner(name, '', '', '', 'Ukraine') 792 | raise Exception("Could not get info from UA register") 793 | 794 | def TH(tail_n): 795 | # Register dates from 2019. TODO: find more recent one 796 | r = requests.get('https://www.caat.or.th/wp-content/uploads/'\ 797 | '2019/03/Aircraft-List_18-Mar-2019-Un-controlled-EN.xlsx') 798 | if r.status_code == 200: 799 | with open('/tmp/book.xlsx', 'wb') as f: 800 | f.write(r.content) 801 | book = load_workbook('/tmp/book.xlsx') 802 | for sheet in book: 803 | infos = [row for row in sheet.values if row[1] == tail_n] 804 | if len(infos) > 0: 805 | name = infos[0][2] 806 | return Owner(name, '', '', '', 'Thailand') 807 | else: 808 | raise Exception('Number not found in thai register') 809 | raise Exception("Could not get info from thai register") 810 | 811 | def RS(tail_n): 812 | items = [] 813 | for i in range(0, 2000, 100): 814 | r = requests.get('https://apps.cad.gov.rs/ords/dcvws'\ 815 | '/regvaz/site/listAircraft?p_reg_id=0&p_type_ac=0'\ 816 | '&p_manufacturer=0&p_man_code=0&p_sn=0&p_user=0'\ 817 | '&order_by=p_reg_id.asc&offset='+str(i), verify=False) 818 | if r.status_code == 200: 819 | j = json.loads(r.content) 820 | items.extend(j['items']) 821 | 822 | for item in items: 823 | if item['registarska_oznaka'] == tail_n: 824 | own = item['korisnik'] 825 | return Owner(own, '', '', '', 'Serbia'), Aircraft(tail_n) 826 | return None, None 827 | 828 | def DK(tail_n): 829 | register = register_from_config("DK") 830 | soup = register.request_infos(tail_n) 831 | tr = soup.find('tr', {'class':'ulige'}) 832 | td_link = tr.find('a') 833 | # No owner infos fallback by sending link 834 | r = requests.get('http://www-oy-reg.dk'+td_link['href']) 835 | if r.status_code == 200: 836 | print(r.content) 837 | return Owner(r.text), Aircraft(tail_n) 838 | else: 839 | return None, None 840 | 841 | def LV(tail_n): 842 | register = register_from_config("LV") 843 | book = register.request_infos(tail_n) 844 | infos_sheet = book["Sheet1"] 845 | for row in infos_sheet.values: 846 | if tail_n in row: 847 | return row 848 | 849 | 850 | def HR(tail_n): 851 | register = register_from_config("HR") 852 | infos = register.request_infos(tail_n) 853 | for page in infos['pages']: 854 | for element in page['elements']: 855 | if element['type'] == 'table': 856 | for line in element['content']: 857 | print(line) 858 | 859 | 860 | def CY(tail_n): 861 | register = register_from_config("CY") 862 | infos = register.request_infos(tail_n) 863 | for page in infos['pages']: 864 | for element in page['elements']: 865 | if element['type'] == 'table': 866 | for line in element['content']: 867 | if len(line['content']) >= 1: 868 | if len(line['content'][1]['content']) >= 1: 869 | reg = line['content'][1]['content'][0]['content'] 870 | if reg == tail_n: 871 | own_cell = line['content'][7]['content'] 872 | own = ' '.join([own_cell[i]['content'] \ 873 | for i in range(0,len(own_cell))]) 874 | return Owner(own, '', '', '', '') 875 | def MV(tail_n): 876 | register = register_from_config("MV") 877 | infos = register.request_infos(tail_n) 878 | for page in infos['pages']: 879 | for element in page['elements']: 880 | if element['type'] == 'table': 881 | for line in element['content']: 882 | if len(line['content']) > 2: 883 | if len(line['content'][3]['content']) > 0 \ 884 | and line['content'][3]['type'] != 'spanned-table-cell': 885 | if line['content'][3]['content'][0]['content'] == tail_n: 886 | owner = line['content'][17]['content'] 887 | own = ' '.join(owner[i]['content'] for i in range(0, len(owner))) 888 | return Owner(own, '', '', '', '') 889 | def CAY(tail_n): 890 | register = register_from_config("CAY") 891 | infos = register.request_infos(tail_n) 892 | for page in infos['pages']: 893 | for element in page['elements']: 894 | if element['type'] == 'table': 895 | for line in element['content']: 896 | print(line) 897 | print('---') 898 | 899 | def GG(tail_n): 900 | register = register_from_config("GG") 901 | infos = register.request_infos(tail_n) 902 | for page in infos['pages']: 903 | for e in page['elements']: 904 | if e['type'] == 'table': 905 | for content in e['content'][1:]: 906 | line_to_get = None 907 | for i, line in enumerate(content['content'][0]['content']): 908 | if line['content'] == tail_n: 909 | line_to_get = i 910 | own = ' '.join( 911 | content['content'][3]['content'][line_to_get+i]['content'] \ 912 | for i in range(1, 8)) 913 | return Owner(own, '', '', '', 'Guernsey'), Aircraft(tail_n) 914 | tail_column = content['content'][0]['content'] 915 | return "" 916 | 917 | 918 | def MT(tail_n): 919 | register = register_from_config("MT") 920 | infos = register.request_infos(tail_n) 921 | for page in infos['pages']: 922 | for e in page['elements']: 923 | if e['type'] == 'table': 924 | for content in e['content'][1:]: 925 | if content['type'] == 'table-row': 926 | row = content 927 | tail = row['content'][2]['content'][0]['content'] 928 | owner_arr = row['content'][7]['content'] 929 | if tail in tail_n: 930 | owner_name = ' '.join(c['content'] for c in owner_arr) 931 | owner_infos = owner_name.split(',') 932 | name = owner_infos[0] 933 | addr = owner_infos[1] 934 | city = owner_infos[2] 935 | return Owner(name, addr, city, '', country='Malta'), Aircraft(tail_n) 936 | return Owner('', country="Malta") 937 | 938 | def MD(tail_n): 939 | register = register_from_config("MD") 940 | infos = register.request_infos(tail_n) 941 | for page in infos['pages']: 942 | for content in page['elements']: 943 | if content['type'] == 'table': 944 | for item_row in content['content']: 945 | if item_row['type'] == 'table-row': 946 | ## print(item_row['content']) 947 | retrieved_tail = item_row['content'][2]['content'] 948 | owner = item_row['content'][4]['content'] 949 | if len(retrieved_tail) > 0: 950 | if retrieved_tail[0]['content'] == tail_n: 951 | owner_name = owner[0]['content'] 952 | return Owner(owner_name, country='Moldova') 953 | 954 | 955 | def VE(tail_n): 956 | register = register_from_config("VE") 957 | infos = register.request_infos(tail_n) 958 | format_tail = tail_n.replace('-', '') 959 | page = infos['Lista Master'] 960 | for line in page.values: 961 | if format_tail in str(line[0]): 962 | # Owner infos 963 | msn = str(line[2]) 964 | notes = str(line[4]) 965 | type=str(line[1]) 966 | return None, Aircraft(tail_n, msn=msn, notes=notes, manufacturer=type) 967 | return infos 968 | 969 | 970 | def SC(tail_n): 971 | register = register_from_config("SC") 972 | infos = register.request_infos(tail_n) 973 | ul = infos.find('ul', {'class':'jdb-tab-contents'}) 974 | li = ul.find('li') 975 | tables = li.find_all('table') 976 | for table in tables: 977 | rows = table.find_all('tr') 978 | for row in rows: 979 | spans = row.find_all('span') 980 | for span in spans: 981 | if tail_n in span.text: 982 | owner_name = spans[2].text 983 | # Only domesic owners are registered in this CAA 984 | country = 'Seychelles' 985 | return Owner(owner_name, country=country), Aircraft(tail_n) 986 | return None, None 987 | 988 | def EE(tail_n): 989 | register = register_from_config("EE") 990 | infos = register.request_infos(tail_n) 991 | table_rows = infos.find_all('tr') 992 | tail_formatted = tail_n.split('-')[0]+' - '+tail_n.split('-')[1] 993 | for row in table_rows: 994 | if tail_formatted in row.text: 995 | tds = row.find_all('td') 996 | own = Owner(tds[6].text) 997 | msn = tds[5].text 998 | return own, Aircraft(tail_n, msn=msn) 999 | return None, None 1000 | 1001 | def ES(tail_n): 1002 | register = register_from_config("ES") 1003 | infos = register.request_infos(tail_n) 1004 | for page in infos['pages']: 1005 | for element in page['elements']: 1006 | if element['type'] == 'table': 1007 | for line in element['content']: 1008 | for elem in line['content']: 1009 | if elem['content'][0]['content'] == tail_n: 1010 | msn = ' '.join( 1011 | line['content'][4]['content'][i]['content'] \ 1012 | for i in range(0, len(line['content'][4]['content']))) 1013 | manuf = ' '.join( 1014 | line['content'][4]['content'][i]['content'] \ 1015 | for i in range(0, len(line['content'][4]['content']))) 1016 | return Owner(''), Aircraft(tail_n, msn=msn, manufacturer=manuf) 1017 | 1018 | return None, None 1019 | 1020 | def ME(tail_n): 1021 | try: 1022 | register = register_from_config("ME") 1023 | infos = register.request_infos(tail_n) 1024 | if infos: 1025 | div_owner_name = infos.find('div', 1026 | {'class': 'field-name-field-ime'}) 1027 | div_owner_addr = infos.find('div', 1028 | {'class': 'field-name-field-adresa'}) 1029 | div_owner_city = infos.find('div', 1030 | {'class': 'field-name-field-po-tanski-broj-ulice'}) 1031 | div_msn = infos.find('div', 1032 | {'class': 'field-name-field-serijski-broj'}) 1033 | 1034 | owner_name = div_owner_name.text 1035 | owner_addr = div_owner_addr.text 1036 | owner_city = div_owner_city.text 1037 | msn = div_msn.text 1038 | return Owner(owner_name, owner_addr, 1039 | owner_city, country="Montenegro"), Aircraft(tail_n, msn=msn) 1040 | else: 1041 | return None, None 1042 | except Exception as e: 1043 | return None, None 1044 | 1045 | 1046 | def BM(tail_n): 1047 | try: 1048 | register = register_from_config("BM") 1049 | infos = register.request_infos(tail_n) 1050 | for page in infos['pages']: 1051 | for element in page['elements']: 1052 | if element['type'] == 'table': 1053 | for line in element['content']: 1054 | for elem in line['content']: 1055 | if elem['content'][0]['content'] == tail_n: 1056 | name = line['content'][4]['content'][0]['content'] 1057 | msn = line['content'][1]['content'][0]['content'] 1058 | return Owner(name), Aircraft(tail_n, msn=msn) 1059 | return None, None 1060 | except Exception as e: 1061 | return None, None 1062 | 1063 | 1064 | def KZ(tail_n): 1065 | try: 1066 | register = register_from_config("KZ") 1067 | book = register.request_infos(tail_n) 1068 | infos_sheet = book["aircraft"] 1069 | for row in infos_sheet.values: 1070 | if tail_n in row: 1071 | msn = row[9] 1072 | model = row[15] 1073 | owner = row[12] 1074 | return Owner(owner, '', '', '', ''),\ 1075 | Aircraft(tail_n, msn=msn, manufacturer=model) 1076 | return None, None 1077 | except Exception as e: 1078 | return None, None 1079 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | parsr-client 3 | bs4 4 | openpyxl 5 | 6 | -------------------------------------------------------------------------------- /tail_to_register.py: -------------------------------------------------------------------------------- 1 | from registers import * 2 | 3 | tail_to_register_function = { 4 | "HB-": CH, 5 | "F-" : FR, 6 | "TF-": IS, 7 | "N" : US, 8 | "OO-": BE, 9 | "OE-": AT, 10 | "SE-": SW, 11 | "OK-": CZ, 12 | "G-" : UK, 13 | "EI-": IE, 14 | "M-" : IM, 15 | "I-" : IT, 16 | "C-" : CA, 17 | "YR-": RO, 18 | "YU-": RS, 19 | "VH-": AU, 20 | "9A-": HR, 21 | "9V-": SG, 22 | "ZK-": NZ, 23 | "PP-": BR, 24 | "PS-": BR, 25 | "PR-": BR, 26 | "PT-": BR, 27 | "PU-": BR, 28 | "D-" : DE, 29 | "UR-": UA, 30 | "HS-": TH, 31 | "U-" : TH, 32 | "OY-": DK, 33 | "YL-": LV, 34 | "E7-": BA, 35 | "9A-": HR, 36 | "5B-": CY, 37 | "8Q-": MV, 38 | "VP-": CA, 39 | "2-" : GG, 40 | "9H-": MT, 41 | "ER-": MD, 42 | "V3-": BZ, 43 | "YV-": VE, 44 | "S7-": SC, 45 | "ES-": EE, 46 | "EC-": ES, 47 | "4O-": ME, 48 | "VP-": BM, 49 | "VQ-": BM, 50 | "LZ-": BG, 51 | "UP-": KZ, 52 | "RB-": BY, 53 | "EW-": BY, 54 | } 55 | -------------------------------------------------------------------------------- /wiki_api.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | def search_wiki_commons(tail_n): 5 | r = requests.get('https://commons.wikimedia.org/wiki/Category:{}_(aircraft)'.format(tail_n)) 6 | 7 | if r.status_code == 200: 8 | return r.url 9 | else: 10 | return None 11 | --------------------------------------------------------------------------------