├── udger ├── tests │ ├── __init__.py │ ├── downloadtest.py │ ├── runperftest.py │ ├── runtests.py │ ├── test_ip.json │ └── test_ua.json ├── __init__.py ├── wdetector.py ├── downloader.py ├── parser.py ├── base.py └── queries.py ├── MANIFEST.in ├── setup.py ├── LICENSE └── README.md /udger/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /udger/__init__.py: -------------------------------------------------------------------------------- 1 | from .parser import Udger 2 | 3 | __version__ = '4.0.2' 4 | 5 | __all__ = ['Udger'] 6 | -------------------------------------------------------------------------------- /udger/tests/downloadtest.py: -------------------------------------------------------------------------------- 1 | from udger import UdgerDownloader 2 | 3 | downloader = UdgerDownloader('test_data') 4 | downloader.download() -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include MANIFEST.in 2 | include README.rst 3 | include setup.py 4 | include LICENSE 5 | 6 | global-exclude *.pyc 7 | global-exclude *.pyo 8 | -------------------------------------------------------------------------------- /udger/wdetector.py: -------------------------------------------------------------------------------- 1 | 2 | class WordDetector(object): 3 | 4 | _wdict = {} 5 | 6 | def add_word(self, wid, word): 7 | prefix = word[:3] 8 | if prefix not in self._wdict: 9 | self._wdict[prefix] = [(wid,word),] 10 | else: 11 | self._wdict[prefix].append((wid,word),) 12 | 13 | def find_words(self, text): 14 | found_words = set() 15 | found_words.add(0) 16 | s = text.lower() 17 | for x in range(0, len(s) - 2): 18 | word_infos = self._wdict.get(s[x:x+3], ()) 19 | found_words.update(wi[0] for wi in word_infos if s.startswith(wi[1], x)) 20 | return found_words -------------------------------------------------------------------------------- /udger/tests/runperftest.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from udger import Udger 4 | 5 | """ 6 | Running in both Python 2 and Python 3 like this: 7 | 8 | $ PYTHONPATH=../.. python2 runperftest.py 9 | $ PYTHONPATH=../.. python3 runperftest.py 10 | 11 | """ 12 | 13 | 14 | udger = Udger('') 15 | 16 | uas = [line.rstrip('\n') for line in open('ua.txt')] 17 | 18 | start_time = time.time() 19 | step_time = start_time 20 | for i in range(10000): 21 | ua_string = uas[i % len(uas)] 22 | udger.parse_ua(ua_string) 23 | if (i % 1000) == 0: 24 | print ("step: ", i, " avg: ", 1000.0/(time.time() - step_time) , "/sec") 25 | step_time = time.time() 26 | print("Total time: ", i / (time.time() - start_time), "/sec") -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from os.path import dirname, join 2 | from setuptools import setup, find_packages 3 | 4 | from udger import __version__ 5 | 6 | 7 | setup( 8 | name='udger', 9 | version=__version__, 10 | license='BSD', 11 | author='The Udger Team', 12 | author_email='info@udger.com', 13 | description="Fast and reliable User Agent parser and IP classifier for Python", 14 | long_description_content_type='text/markdown', 15 | url='https://github.com/udger/udger-python', 16 | packages=find_packages( 17 | exclude=('udger.tests',), 18 | ), 19 | platforms='any', 20 | zip_safe=True, 21 | classifiers=[ 22 | "Development Status :: 5 - Production/Stable", 23 | "Intended Audience :: Developers", 24 | "License :: OSI Approved :: BSD License", 25 | "Operating System :: OS Independent", 26 | "Programming Language :: Python", 27 | "Programming Language :: Python :: 2", 28 | "Programming Language :: Python :: 3", 29 | "Topic :: Software Development :: Libraries", 30 | ], 31 | ) 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) [year] [fullname] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /udger/downloader.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from .base import cached_property 4 | from .base import UdgerBase 5 | 6 | import gzip 7 | import io 8 | import struct 9 | import tempfile 10 | import urllib.request 11 | 12 | class UdgerDownloader(object): 13 | 14 | udger_data_file = 'udgerdb_v3.dat.gz' 15 | download_url = 'http://data.udger.com/' 16 | 17 | def __init__(self, client_key, data_dir=None): 18 | self.client_key = client_key 19 | self.data_dir = data_dir or tempfile.gettempdir() 20 | 21 | def download(self): 22 | url = UdgerDownloader.download_url + self.client_key + '/' + UdgerDownloader.udger_data_file 23 | response = urllib.request.urlopen(url) 24 | compressed_file = io.BytesIO(response.read()) 25 | decompressed_file = gzip.GzipFile(fileobj=compressed_file) 26 | 27 | with open(self.new_filename, 'wb') as outfile: 28 | outfile.write(decompressed_file.read()) 29 | os.rename(self.new_filename, os.path.join(self.data_dir, UdgerBase.db_filename)) 30 | 31 | @cached_property 32 | def new_filename(self): 33 | return os.path.join(self.data_dir, UdgerBase.db_filename + '.new') 34 | -------------------------------------------------------------------------------- /udger/tests/runtests.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import json 3 | import operator 4 | import sys 5 | 6 | from udger import Udger 7 | 8 | try: 9 | reduce 10 | except NameError: 11 | from functools import reduce # Python 3 12 | 13 | """ 14 | Running in both Python 2 and Python 3 like this: 15 | 16 | $ PYTHONPATH=../.. python2 runtests.py 17 | $ PYTHONPATH=../.. python3 runtests.py 18 | 19 | Compare Python 2 to Python 3: 20 | $ diff <(PYTHONPATH=../.. python3 runtests.py | sort) <(PYTHONPATH=../.. python2 runtests.py | sort) 21 | 22 | """ 23 | 24 | 25 | def iter_compare_dicts(dict1, dict2, only_common_keys=False, comparison_op=operator.ne): 26 | """ 27 | A generator for comparation of values in the given two dicts. 28 | 29 | Yields the tuples (key, pair of values positively compared). 30 | 31 | By default, the *difference* of values is evaluated using the usual != op, but can be changed 32 | by passing other comparison_op (a function of two arguments returning True/False). 33 | 34 | For example: operator.eq for equal values, operator.is_not for not identical objects. 35 | 36 | You can also require comparison only over keys existing in both dicts (only_common_keys=True). 37 | Otherwise, you will get the pair with the Python built-in Ellipsis placed for dict with 38 | that key missing. (Be sure to test for Ellipsis using the 'is' operator.) 39 | 40 | >>> d1 = dict(a=1, b=2, c=3) 41 | >>> d2 = dict(a=1, b=20, d=4) 42 | >>> dict(iter_compare_dicts(d1, d2, only_common_keys=True)) 43 | {'b': (2, 20)} 44 | >>> dict(iter_compare_dicts(d1, d2, only_common_keys=True, comparison_op=operator.eq)) 45 | {'a': (1, 1)} 46 | >>> dict(iter_compare_dicts(d1, d2)) 47 | {'c': (3, Ellipsis), 'b': (2, 20), 'd': (Ellipsis, 4)} 48 | >>> dict(iter_compare_dicts(d1, d2, comparison_op=operator.eq)) 49 | {'a': (1, 1), 'c': (3, Ellipsis), 'd': (Ellipsis, 4)} 50 | """ 51 | keyset1, keyset2 = set(dict1), set(dict2) 52 | 53 | for key in (keyset1 & keyset2): 54 | pair = (dict1[key], dict2[key]) 55 | if reduce(comparison_op, pair): 56 | yield key, pair 57 | 58 | if not only_common_keys: 59 | for key in (keyset1 - keyset2): 60 | yield key, (dict1[key], Ellipsis) 61 | for key in (keyset2 - keyset1): 62 | yield key, (Ellipsis, dict2[key]) 63 | 64 | 65 | header_written = False 66 | writer = csv.writer(sys.stdout, delimiter=';', quotechar='"') 67 | 68 | 69 | def do_tests(json_filepath, test_func): 70 | global header_written, writer 71 | 72 | for testcase in json.load(open(json_filepath)): 73 | test_string = testcase['test']['teststring'] 74 | 75 | expected = testcase['ret'] 76 | 77 | resolved = test_func(test_string) 78 | resolved.pop('crawler_last_seen', None) 79 | resolved.pop('ip_last_seen', None) 80 | 81 | for field, (value_expected, value_returned) in iter_compare_dicts(expected, resolved): 82 | if (value_expected or '') == (value_returned or '') == '': 83 | continue 84 | 85 | # this client replaces ' ' with '%20' in strings; ignore the mismatch with test data 86 | if value_expected is not Ellipsis and (value_expected or '').startswith('https://'): 87 | if value_expected.replace(' ', '%20') == value_returned: 88 | continue 89 | 90 | # tolerate number in strings 91 | if str(value_returned) == value_expected: 92 | continue 93 | 94 | if not header_written: 95 | writer.writerow(('STRING', 'RESPONSE_FIELD', 'VALUE_EXPECTED', 'VALUE_RETURNED')) 96 | header_written = True 97 | 98 | writer.writerow((test_string, field, value_expected, value_returned)) 99 | 100 | udger = Udger() 101 | 102 | do_tests('test_ua.json', udger.parse_ua) 103 | do_tests('test_ip.json', udger.parse_ip) 104 | -------------------------------------------------------------------------------- /udger/parser.py: -------------------------------------------------------------------------------- 1 | from .base import UdgerBase 2 | 3 | from .queries import Queries 4 | 5 | class Udger(UdgerBase): 6 | 7 | def parse_ua(self, ua_string): 8 | 9 | if self.lru_cache is not None: 10 | cached = self.lru_cache.get(ua_string, None) 11 | if cached is not None: 12 | return cached 13 | 14 | ua, class_id, client_id = self._client_detector(ua_string) 15 | is_crawler = (ua['ua_class'] == 'Crawler') 16 | 17 | opsys = self._os_detector(ua_string, client_id) if not is_crawler else None 18 | ua.update(opsys or self.os_emptyrow) 19 | 20 | dev = self._device_detector(ua_string, class_id) if not is_crawler else None 21 | ua.update(dev or self.device_emptyrow) 22 | 23 | marketname = None 24 | if not is_crawler and ua['os_family_code']: 25 | # must complete first so cursors don't collide 26 | rows = tuple(self.db_iter_rows( 27 | Queries.devicename_sql, 28 | ua['os_family_code'], 29 | ua['os_code'], 30 | )) 31 | 32 | for dn_row in rows: 33 | if self.regexp_func(dn_row['regstring'], ua_string): 34 | match = self.last_regexp_match.group(1) 35 | 36 | marketname = self.db_get_first_row( 37 | Queries.marketname_sql, 38 | dn_row['regex_id'], 39 | match.strip(), 40 | ) 41 | if marketname: 42 | break 43 | 44 | ua.update(marketname or self.marketname_emptyrow) 45 | 46 | ua['ua_string'] = ua_string 47 | 48 | self.lru_cache[ua_string] = ua 49 | 50 | return ua 51 | 52 | def parse_ip(self, ip_string): 53 | ip = self.ip_datacenter_emptyrow.copy() 54 | ip['ip'] = ip_string 55 | 56 | try: 57 | ip_string, ipv4_int, ipv6_words = self.normalize_ipaddress(ip_string) 58 | except: 59 | pass 60 | else: 61 | ip.update( 62 | ip_classification="Unrecognized", 63 | ip_classification_code="unrecognized", 64 | ) 65 | 66 | ip_row = self.db_get_first_row(Queries.ip_sql, ip_string) 67 | if ip_row: 68 | if ip_row['ip_classification_code'] != 'crawler': 69 | ip_row.pop('crawler_family_info_url') 70 | 71 | ip.update(ip_row) 72 | 73 | if ipv4_int is not None: 74 | ip['ip_ver'] = 4 75 | dc = self.db_get_first_row(Queries.datacenter_sql, ipv4_int, ipv4_int) 76 | 77 | else: 78 | ip['ip_ver'] = 6 79 | ipv6_words *= 2 80 | dc = self.db_get_first_row(Queries.datacenter6_sql, *ipv6_words) 81 | 82 | if dc: 83 | ip.update(dc) 84 | 85 | return ip 86 | def _client_detector(self, ua_string): 87 | ua = self.db_get_first_row(Queries.crawler_sql, ua_string) 88 | if ua: 89 | del ua['class_id'] 90 | del ua['client_id'] 91 | class_id = 99 92 | client_id = -1 93 | else: 94 | rowid = self._find_id_from_list(ua_string, self.client_word_detector.find_words(ua_string), self.client_regstring_list) 95 | if rowid != -1: 96 | ua = self.db_get_first_row(Queries.client_sql, rowid) 97 | self._patch_versions(ua) 98 | else: 99 | ua = self.client_emptyrow.copy() 100 | class_id = ua.pop('class_id', -1) 101 | client_id = ua.pop('client_id', 0) 102 | return (ua, class_id, client_id) 103 | 104 | def _os_detector(self, ua_string, client_id): 105 | rowid = self._find_id_from_list(ua_string, self.os_word_detector.find_words(ua_string), self.os_regstring_list) 106 | if rowid != -1: 107 | return self.db_get_first_row(Queries.os_sql, rowid) 108 | return client_id != 0 and self.db_get_first_row(Queries.client_os_sql, client_id) 109 | 110 | def _device_detector(self, ua_string, class_id): 111 | rowid = self._find_id_from_list(ua_string, self.device_word_detector.find_words(ua_string), self.device_regstring_list) 112 | if rowid != -1: 113 | return self.db_get_first_row(Queries.device_sql, rowid) 114 | return class_id != -1 and self.db_get_first_row(Queries.client_class_sql, class_id) 115 | 116 | def _find_id_from_list(self, ua_string, found_client_words, reg_string_list): 117 | self.last_regexp_match = None 118 | for irs in reg_string_list: 119 | if (irs.word_id in found_client_words) and (irs.word2_id in found_client_words): 120 | m = irs.pattern.search(ua_string) 121 | if not m is None: 122 | self.last_regexp_match = m 123 | return irs.rowid 124 | return -1 125 | 126 | def _patch_versions(self, ua): 127 | if self.last_regexp_match: 128 | try: 129 | ver = self.last_regexp_match.group(1) 130 | except IndexError: 131 | ver = '' 132 | 133 | ua['ua_version'] = ver 134 | ua['ua'] += " " + ver 135 | ua['ua_version_major'] = ver.split('.')[0] 136 | else: 137 | ua['ua_version'] = ua['ua_version_major'] = None 138 | 139 | -------------------------------------------------------------------------------- /udger/tests/test_ip.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "test": { 4 | "teststring": "108.61.199.93" 5 | }, 6 | "ret": { 7 | "ip": "108.61.199.93", 8 | "ip_ver": "4", 9 | "ip_classification": "Crawler", 10 | "ip_classification_code": "crawler", 11 | "ip_hostname": "108.61.199.93.vultr.com", 12 | "ip_country": "Netherlands", 13 | "ip_country_code": "NL", 14 | "ip_city": "Amsterdam", 15 | "crawler_name": "PINGOMETER", 16 | "crawler_ver": "", 17 | "crawler_ver_major": "", 18 | "crawler_family": "PINGOMETER", 19 | "crawler_family_code": "pingometer", 20 | "crawler_family_homepage": "", 21 | "crawler_family_vendor": "Pingometer, LLC", 22 | "crawler_family_vendor_code": "pingometer_llc", 23 | "crawler_family_vendor_homepage": "http:\/\/pingometer.com\/", 24 | "crawler_family_icon": "bot_pingometer.png", 25 | "crawler_family_info_url": "https:\/\/udger.com\/resources\/ua-list\/bot-detail?bot=PINGOMETER#id20112", 26 | "crawler_category": "Site monitor", 27 | "crawler_category_code": "site_monitor", 28 | "crawler_respect_robotstxt": "no", 29 | "datacenter_name": "Choopa, LLC.", 30 | "datacenter_name_code": "choopa", 31 | "datacenter_homepage": "https:\/\/www.choopa.com\/" 32 | } 33 | }, 34 | { 35 | "test": { 36 | "teststring": "2a02:598:111::9" 37 | }, 38 | "ret": { 39 | "ip": "2a02:598:111::9", 40 | "ip_ver": "6", 41 | "ip_classification": "Crawler", 42 | "ip_classification_code": "crawler", 43 | "ip_hostname": "fulltextrobot-dev-2a02-598-111--9.seznam.cz", 44 | "ip_country": "Czech Republic", 45 | "ip_country_code": "CZ", 46 | "ip_city": "Prague", 47 | "crawler_name": "SeznamBot\/3.2-test1", 48 | "crawler_ver": "3.2-test1", 49 | "crawler_ver_major": "3", 50 | "crawler_family": "SeznamBot", 51 | "crawler_family_code": "seznambot", 52 | "crawler_family_homepage": "http:\/\/napoveda.seznam.cz\/en\/seznambot-intro\/", 53 | "crawler_family_vendor": "Seznam.cz, a.s.", 54 | "crawler_family_vendor_code": "seznam-cz_as", 55 | "crawler_family_vendor_homepage": "http:\/\/onas.seznam.cz\/", 56 | "crawler_family_icon": "seznam.png", 57 | "crawler_family_info_url": "https:\/\/udger.com\/resources\/ua-list\/bot-detail?bot=SeznamBot#id28914", 58 | "crawler_category": "Search engine bot", 59 | "crawler_category_code": "search_engine_bot", 60 | "crawler_respect_robotstxt": "unknown", 61 | "datacenter_name": "Seznam.cz", 62 | "datacenter_name_code": "seznam_cz", 63 | "datacenter_homepage": "http:\/\/onas.seznam.cz\/" 64 | } 65 | }, 66 | { 67 | "test": { 68 | "teststring": "2001:41d0:8:d54c::1" 69 | }, 70 | "ret": { 71 | "ip": "2001:41d0:8:d54c::1", 72 | "ip_ver": "6", 73 | "ip_classification": "Cgi proxy", 74 | "ip_classification_code": "cgi_proxy", 75 | "ip_hostname": "", 76 | "ip_country": "France", 77 | "ip_country_code": "FR", 78 | "ip_city": "Cachan", 79 | "crawler_name": "", 80 | "crawler_ver": "", 81 | "crawler_ver_major": "", 82 | "crawler_family": "", 83 | "crawler_family_code": "", 84 | "crawler_family_homepage": "", 85 | "crawler_family_vendor": "", 86 | "crawler_family_vendor_code": "", 87 | "crawler_family_vendor_homepage": "", 88 | "crawler_family_icon": "", 89 | "crawler_family_info_url": "", 90 | "crawler_category": "", 91 | "crawler_category_code": "", 92 | "crawler_respect_robotstxt": "", 93 | "datacenter_name": "OVH", 94 | "datacenter_name_code": "ovh", 95 | "datacenter_homepage": "http:\/\/www.ovh.com\/" 96 | } 97 | }, 98 | { 99 | "test": { 100 | "teststring": "66.249.64.73" 101 | }, 102 | "ret": { 103 | "ip": "66.249.64.73", 104 | "ip_ver": "4", 105 | "ip_classification": "Crawler", 106 | "ip_classification_code": "crawler", 107 | "ip_hostname": "crawl-66-249-64-73.googlebot.com", 108 | "ip_country": "United States", 109 | "ip_country_code": "US", 110 | "ip_city": "Mountain View", 111 | "crawler_name": "Googlebot\/2.1", 112 | "crawler_ver": "2.1", 113 | "crawler_ver_major": "2", 114 | "crawler_family": "Googlebot", 115 | "crawler_family_code": "googlebot", 116 | "crawler_family_homepage": "http:\/\/www.google.com\/bot.html", 117 | "crawler_family_vendor": "Google Inc.", 118 | "crawler_family_vendor_code": "google_inc", 119 | "crawler_family_vendor_homepage": "https:\/\/www.google.com\/about\/company\/", 120 | "crawler_family_icon": "bot_googlebot.png", 121 | "crawler_family_info_url": "https:\/\/udger.com\/resources\/ua-list\/bot-detail?bot=Googlebot#id31", 122 | "crawler_category": "Search engine bot", 123 | "crawler_category_code": "search_engine_bot", 124 | "crawler_respect_robotstxt": "yes", 125 | "datacenter_name": "Google sites", 126 | "datacenter_name_code": "google_sites", 127 | "datacenter_homepage": "http:\/\/sites.google.com\/" 128 | } 129 | }, 130 | { 131 | "test": { 132 | "teststring": "90.177.52.111" 133 | }, 134 | "ret": { 135 | "ip": "90.177.52.111", 136 | "ip_ver": "4", 137 | "ip_classification": "Unrecognized", 138 | "ip_classification_code": "unrecognized", 139 | "ip_hostname": "", 140 | "ip_country": "", 141 | "ip_country_code": "", 142 | "ip_city": "", 143 | "crawler_name": "", 144 | "crawler_ver": "", 145 | "crawler_ver_major": "", 146 | "crawler_family": "", 147 | "crawler_family_code": "", 148 | "crawler_family_homepage": "", 149 | "crawler_family_vendor": "", 150 | "crawler_family_vendor_code": "", 151 | "crawler_family_vendor_homepage": "", 152 | "crawler_family_icon": "", 153 | "crawler_family_info_url": "", 154 | "crawler_category": "", 155 | "crawler_category_code": "", 156 | "crawler_respect_robotstxt": "", 157 | "datacenter_name": "", 158 | "datacenter_name_code": "", 159 | "datacenter_homepage": "" 160 | } 161 | } 162 | ] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Udger client for Python (data ver. 3) 2 | 3 | Local parser is very fast and accurate useragent string detection solution. Enables developers to locally install and integrate a highly-scalable product. 4 | We provide the detection of the devices (personal computer, tablet, Smart TV, Game console etc.), operating system and client SW type (browser, e-mail client etc.). 5 | It also provides information about IP addresses (Public proxies, VPN services, Tor exit nodes, Fake crawlers, Web scrapers .. etc.) 6 | 7 | - Tested with more the 50.000 unique user agents. 8 | - Up to date data provided by https://udger.com/ 9 | - Support for Python 3 10 | 11 | Enjoy! 12 | 13 | ### Install using pip 14 | 15 | $ pip install udger 16 | 17 | ### Install from git repo 18 | 19 | $ git clone https://github.com/udger/udger-python 20 | $ cd udger-python/ 21 | # python setup.py install 22 | 23 | ### Automatic updates download 24 | 25 | For data auto update, please use Udger data updater (https://udger.com/support/documentation/?doc=62) 26 | 27 | ### Help us 28 | 29 | Feel free to send us a Pull Request on GitHub to help us make Udger for Python better. 30 | Or just let us know of any issues you encounter. 31 | 32 | Thank you! 33 | 34 | ### Usage 35 | 36 | ``` 37 | $ python 38 | >>> from pprint import pprint 39 | >>> from udger import Udger 40 | >>> udger = Udger() 41 | >>> 42 | >>> result = udger.parse_ua( 43 | ... 'Mozilla/5.0 (iPad; CPU OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53' 44 | ... ) 45 | >>> pprint(result) 46 | {'crawler_category': None, 47 | 'crawler_category_code': None, 48 | 'crawler_last_seen': None, 49 | 'crawler_respect_robotstxt': None, 50 | 'device_brand': 'Apple', 51 | 'device_brand_code': 'apple', 52 | 'device_brand_homepage': 'http://www.apple.com/', 53 | 'device_brand_icon': 'apple.png', 54 | 'device_brand_icon_big': 'apple_big.png', 55 | 'device_brand_info_url': 'https://udger.com/resources/ua-list/devices-brand-detail?brand=apple', 56 | 'device_class': 'Tablet', 57 | 'device_class_code': 'tablet', 58 | 'device_class_icon': 'tablet.png', 59 | 'device_class_icon_big': 'tablet_big.png', 60 | 'device_class_info_url': 'https://udger.com/resources/ua-list/device-detail?device=Tablet', 61 | 'device_marketname': 'iPad', 62 | 'os': 'iOS 7', 63 | 'os_code': 'ios_7', 64 | 'os_family': 'iOS', 65 | 'os_family_code': 'ios', 66 | 'os_family_vendor': 'Apple Inc.', 67 | 'os_family_vendor_code': 'apple_inc', 68 | 'os_family_vendor_homepage': 'http://www.apple.com/', 69 | 'os_homepage': 'https://en.wikipedia.org/wiki/IOS_7', 70 | 'os_icon': 'iphone.png', 71 | 'os_icon_big': 'iphone_big.png', 72 | 'os_info_url': 'https://udger.com/resources/ua-list/os-detail?os=iOS%207', 73 | 'ua': 'Safari mobile 7.0', 74 | 'ua_class': 'Mobile browser', 75 | 'ua_class_code': 'mobile_browser', 76 | 'ua_engine': 'WebKit', 77 | 'ua_family': 'Safari mobile', 78 | 'ua_family_code': 'safari_mobile', 79 | 'ua_family_homepage': 'https://en.wikipedia.org/wiki/Safari_%28web_browser%29', 80 | 'ua_family_icon': 'safari.png', 81 | 'ua_family_icon_big': 'safari_big.png', 82 | 'ua_family_info_url': 'https://udger.com/resources/ua-list/browser-detail?browser=Safari%20mobile', 83 | 'ua_family_vendor': 'Apple Inc.', 84 | 'ua_family_vendor_code': 'apple_inc', 85 | 'ua_family_vendor_homepage': 'http://www.apple.com/', 86 | 'ua_string': 'Mozilla/5.0 (iPad; CPU OS 7_0 like Mac OS X) ' 87 | 'AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 ' 88 | 'Mobile/11A465 Safari/9537.53', 89 | 'ua_uptodate_current_version': '', 90 | 'ua_version': '7.0', 91 | 'ua_version_major': '7'} 92 | >>> 93 | >>> result = udger.parse_ua('Some thing') 94 | >>> pprint(result) 95 | {'crawler_category': None, 96 | 'crawler_category_code': None, 97 | 'crawler_last_seen': None, 98 | 'crawler_respect_robotstxt': None, 99 | 'device_brand': None, 100 | 'device_brand_code': None, 101 | 'device_brand_homepage': None, 102 | 'device_brand_icon': None, 103 | 'device_brand_icon_big': None, 104 | 'device_brand_info_url': None, 105 | 'device_class': None, 106 | 'device_class_code': None, 107 | 'device_class_icon': None, 108 | 'device_class_icon_big': None, 109 | 'device_class_info_url': None, 110 | 'device_marketname': None, 111 | 'os': None, 112 | 'os_code': None, 113 | 'os_family': None, 114 | 'os_family_code': None, 115 | 'os_family_vendor': None, 116 | 'os_family_vendor_code': None, 117 | 'os_family_vendor_homepage': None, 118 | 'os_homepage': None, 119 | 'os_icon': None, 120 | 'os_icon_big': None, 121 | 'os_info_url': None, 122 | 'ua': None, 123 | 'ua_class': 'Unrecognized', 124 | 'ua_class_code': 'unrecognized', 125 | 'ua_engine': None, 126 | 'ua_family': None, 127 | 'ua_family_code': None, 128 | 'ua_family_homepage': None, 129 | 'ua_family_icon': None, 130 | 'ua_family_icon_big': None, 131 | 'ua_family_info_url': None, 132 | 'ua_family_vendor': None, 133 | 'ua_family_vendor_code': None, 134 | 'ua_family_vendor_homepage': None, 135 | 'ua_string': 'Some thing', 136 | 'ua_uptodate_current_version': None, 137 | 'ua_version': None, 138 | 'ua_version_major': None} 139 | >>> 140 | >>> result = udger.parse_ip('69.89.31.120') 141 | >>> pprint(result) 142 | {'crawler_category': None, 143 | 'crawler_category_code': None, 144 | 'crawler_family': None, 145 | 'crawler_family_code': None, 146 | 'crawler_family_homepage': None, 147 | 'crawler_family_icon': None, 148 | 'crawler_family_info_url': None, 149 | 'crawler_family_vendor': None, 150 | 'crawler_family_vendor_code': None, 151 | 'crawler_family_vendor_homepage': None, 152 | 'crawler_last_seen': None, 153 | 'crawler_name': None, 154 | 'crawler_respect_robotstxt': None, 155 | 'crawler_ver': None, 156 | 'crawler_ver_major': None, 157 | 'datacenter_homepage': 'https://www.bluehost.com/', 158 | 'datacenter_name': 'Bluehost Inc.', 159 | 'datacenter_name_code': 'bluehost', 160 | 'ip': '69.89.31.120', 161 | 'ip_city': 'Provo', 162 | 'ip_classification': 'Web scraper', 163 | 'ip_classification_code': 'web_scraper', 164 | 'ip_country': 'United States', 165 | 'ip_country_code': 'US', 166 | 'ip_hostname': 'box320.bluehost.com', 167 | 'ip_last_seen': '2016-09-17 12:13:25', 168 | 'ip_ver': 4} 169 | >>> 170 | >>> result = udger.parse_ip('108.61.199.93') 171 | >>> pprint(result) 172 | {'crawler_category': 'Site monitor', 173 | 'crawler_category_code': 'site_monitor', 174 | 'crawler_family': 'PINGOMETER', 175 | 'crawler_family_code': 'pingometer', 176 | 'crawler_family_homepage': '', 177 | 'crawler_family_icon': 'bot_pingometer.png', 178 | 'crawler_family_info_url': 'https://udger.com/resources/ua-list/bot-detail?bot=PINGOMETER', 179 | 'crawler_family_vendor': 'Pingometer, LLC', 180 | 'crawler_family_vendor_code': 'pingometer_llc', 181 | 'crawler_family_vendor_homepage': 'http://pingometer.com/', 182 | 'crawler_last_seen': '2016-09-17 12:15:38', 183 | 'crawler_name': 'PINGOMETER', 184 | 'crawler_respect_robotstxt': 'no', 185 | 'crawler_ver': '', 186 | 'crawler_ver_major': '', 187 | 'datacenter_homepage': 'https://www.choopa.com/', 188 | 'datacenter_name': 'Choopa, LLC.', 189 | 'datacenter_name_code': 'choopa', 190 | 'ip': '108.61.199.93', 191 | 'ip_city': 'Amsterdam', 192 | 'ip_classification': 'Crawler', 193 | 'ip_classification_code': 'crawler', 194 | 'ip_country': 'Netherlands', 195 | 'ip_country_code': 'NL', 196 | 'ip_hostname': '108.61.199.93.vultr.com', 197 | 'ip_last_seen': '2016-09-17 12:00:31', 198 | 'ip_ver': 4} 199 | ``` 200 | 201 | ### Data directory 202 | 203 | ``Udger()`` parser expects the data file to be placed in the system temporary 204 | directory as returned by the ``tempfile.gettempdir()``. 205 | 206 | You may override the path using the argument like this: 207 | 208 | udger = Udger('/var/cache/udger/') 209 | 210 | ### Documentation for developers 211 | 212 | https://udger.com/pub/documentation/parser/Python/html/ 213 | 214 | ### Author 215 | 216 | The Udger.com Team (info@udger.com) 217 | 218 | ### Old v1 format 219 | 220 | If you still use the previous format of the db (v1), please see the branch ``old_format_v1`` 221 | -------------------------------------------------------------------------------- /udger/base.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import os 3 | import re 4 | import sys 5 | import socket 6 | import sqlite3 7 | import struct 8 | import tempfile 9 | 10 | from .queries import Queries 11 | from .wdetector import * 12 | 13 | if sys.version_info.major == 3 and sys.version_info.minor >= 10: 14 | 15 | from collections.abc import MutableMapping 16 | else: 17 | from collections import MutableMapping 18 | 19 | if sys.version_info.major == 3 and sys.version_info.minor >= 10: 20 | from collections.abc import MutableMapping 21 | else: 22 | from collections import MutableMapping 23 | 24 | 25 | unperlize_re = re.compile('^/?(.*)/([si]*)$') 26 | 27 | def make_empty_row(columns_dict): 28 | return dict((col, None) for col in columns_dict) 29 | 30 | class cached_property(object): 31 | """ 32 | Decorator that converts a method with a single self argument into a 33 | property cached on the instance. 34 | """ 35 | def __init__(self, func): 36 | self.func = func 37 | 38 | def __get__(self, instance, unused_type=None): 39 | if instance is None: 40 | return self 41 | res = instance.__dict__[self.func.__name__] = self.func(instance) 42 | return res 43 | 44 | 45 | class LRUDict(MutableMapping): 46 | def __init__(self, maxlen, *a, **k): 47 | self.maxlen = maxlen 48 | self.d = dict(*a, **k) 49 | while len(self) > maxlen: 50 | self.popitem() 51 | 52 | def __iter__(self): 53 | return iter(self.d) 54 | 55 | def __len__(self): 56 | return len(self.d) 57 | 58 | def __getitem__(self, k): 59 | return self.d[k] 60 | 61 | def __delitem__(self, k): 62 | del self.d[k] 63 | 64 | def __setitem__(self, k, v): 65 | if k not in self and len(self) == self.maxlen: 66 | self.popitem() 67 | self.d[k] = v 68 | 69 | class IdRegString(object): 70 | def __init__(self, rowid, word_id, word2_id, pattern): 71 | self.rowid = rowid 72 | self.word_id = word_id 73 | self.word2_id = word2_id 74 | self.pattern = pattern 75 | 76 | class UdgerBase(object): 77 | db_filename = 'udgerdb_v3.dat' 78 | 79 | _client_word_detector = None 80 | _os_word_detector = None 81 | _device_word_detector = None 82 | 83 | _client_regstring_list = None 84 | _os_regstring_list = None 85 | _device_regstring_list = None 86 | 87 | client_emptyrow = make_empty_row(Queries.client_columns) 88 | client_emptyrow.update( 89 | ua_class="Unrecognized", 90 | ua_class_code="unrecognized", 91 | ) 92 | 93 | os_emptyrow = make_empty_row(Queries.os_columns) 94 | device_emptyrow = make_empty_row(Queries.device_columns) 95 | marketname_emptyrow = make_empty_row(Queries.marketname_columns) 96 | 97 | ip_datacenter_emptyrow = make_empty_row(Queries.ip_columns) 98 | ip_datacenter_emptyrow.update( 99 | make_empty_row(Queries.datacenter_columns), 100 | ip_ver=None, 101 | ) 102 | 103 | def __init__(self, data_dir=None, lru_cache_size=10000): 104 | self.data_dir = data_dir or tempfile.gettempdir() 105 | self.regexp_cache = {} 106 | if lru_cache_size > 0: 107 | self.lru_cache = LRUDict(lru_cache_size) 108 | 109 | @staticmethod 110 | def dict_factory(cursor, row): 111 | return dict( 112 | (col[0], row[idx]) 113 | for idx, col in enumerate(cursor.description) 114 | ) 115 | 116 | perl_flags = { 117 | 's': re.DOTALL, 118 | 'i': re.IGNORECASE, 119 | 'm': re.MULTILINE, 120 | 'x': re.VERBOSE, 121 | } 122 | 123 | def regexp_func(self, expr, item): 124 | global unperlize_re 125 | 126 | expr_re = self.regexp_cache.get(expr) 127 | if expr_re is None: 128 | m = unperlize_re.match(expr) 129 | old_expr = expr 130 | flags = 0 131 | if m: 132 | # strip / from the beginning and /(si...) from the end 133 | expr, opts = m.groups() 134 | # this fails for unsupported Perl flag 135 | flags = sum(map(self.perl_flags.get, opts)) 136 | 137 | expr_re = re.compile(expr, flags) 138 | self.regexp_cache[old_expr] = expr_re 139 | 140 | self.last_regexp_match = expr_re.search(item) # this does not take flags! 141 | 142 | return bool(self.last_regexp_match) 143 | 144 | @cached_property 145 | def db_cursor(self): 146 | db_filepath = os.path.join(self.data_dir, self.db_filename) 147 | db = sqlite3.connect(db_filepath) 148 | # db.create_function("REGEXP", 2, self.regexp_func) 149 | 150 | cursor = db.cursor() 151 | cursor.row_factory = self.dict_factory 152 | 153 | return cursor 154 | 155 | @cached_property 156 | def client_regstring_list(self): 157 | if UdgerBase._client_regstring_list is None: 158 | UdgerBase._client_regstring_list = UdgerBase.prepare_regexp_struct(self.db_cursor, "udger_client_regex") 159 | return UdgerBase._client_regstring_list 160 | 161 | @cached_property 162 | def os_regstring_list(self): 163 | if UdgerBase._os_regstring_list is None: 164 | UdgerBase._os_regstring_list = UdgerBase.prepare_regexp_struct(self.db_cursor, "udger_os_regex") 165 | return UdgerBase._os_regstring_list 166 | 167 | @cached_property 168 | def device_regstring_list(self): 169 | if UdgerBase._device_regstring_list is None: 170 | UdgerBase._device_regstring_list = UdgerBase.prepare_regexp_struct(self.db_cursor, "udger_deviceclass_regex") 171 | return UdgerBase._device_regstring_list 172 | 173 | @staticmethod 174 | def prepare_regexp_struct(db_cursor, regexp_table_name): 175 | result = [] 176 | global unperlize_re 177 | for row in db_cursor.execute('SELECT rowid, regstring, word_id, word2_id FROM %s ORDER BY sequence' % regexp_table_name): 178 | regstring = row['regstring'] 179 | m = unperlize_re.match(regstring) 180 | flags = 0 181 | if m: 182 | # strip / from the beginning and /(si...) from the end 183 | regstring, opts = m.groups() 184 | # this fails for unsupported Perl flag 185 | flags = sum(map(UdgerBase.perl_flags.get, opts)) 186 | expr_re = re.compile(regstring, flags) 187 | rs = IdRegString(row['rowid'], row['word_id'], row['word2_id'], expr_re) 188 | result.append(rs) 189 | 190 | return result; 191 | 192 | @cached_property 193 | def client_word_detector(self): 194 | if UdgerBase._client_word_detector is None: 195 | UdgerBase._client_word_detector = UdgerBase.create_word_detector(self.db_cursor, 'udger_client_regex', 'udger_client_regex_words') 196 | return UdgerBase._client_word_detector 197 | 198 | @cached_property 199 | def device_word_detector(self): 200 | if UdgerBase._device_word_detector is None: 201 | UdgerBase._device_word_detector = UdgerBase.create_word_detector(self.db_cursor, 'udger_deviceclass_regex', 'udger_deviceclass_regex_words') 202 | return UdgerBase._device_word_detector 203 | 204 | @cached_property 205 | def os_word_detector(self): 206 | if UdgerBase._os_word_detector is None: 207 | UdgerBase._os_word_detector = UdgerBase.create_word_detector(self.db_cursor, 'udger_os_regex', 'udger_os_regex_words') 208 | return UdgerBase._os_word_detector 209 | 210 | @staticmethod 211 | def create_word_detector(db_cursor, regex_table, word_table_name): 212 | wdetector = WordDetector() 213 | sql = "SELECT %s FROM " + regex_table 214 | used_words = set(row['word_id'] for row in db_cursor.execute(sql % 'word_id')) 215 | used_words |= set(row['word2_id'] for row in db_cursor.execute(sql % 'word2_id')) 216 | 217 | for row in db_cursor.execute('SELECT * FROM %s' % word_table_name): 218 | if row['id'] in used_words: 219 | wdetector.add_word(row['id'], row['word']) 220 | 221 | return wdetector 222 | 223 | def db_get_first_row(self, sql, *params): 224 | # self.last_regexp_match = None 225 | 226 | self.db_cursor.execute(sql, params) 227 | 228 | for row in self.db_cursor: 229 | return row 230 | 231 | def db_iter_rows(self, sql, *params): 232 | # self.last_regexp_match = None 233 | 234 | self.db_cursor.execute(sql, params) 235 | 236 | for row in self.db_cursor: 237 | yield row 238 | 239 | # self.last_regexp_match = None 240 | 241 | @staticmethod 242 | def normalize_ipaddress(ip_string): 243 | try: 244 | packed = socket.inet_pton(socket.AF_INET, ip_string) 245 | ip_string = socket.inet_ntop(socket.AF_INET, packed) 246 | 247 | ipv6_words = None 248 | ipv4_int = struct.unpack("!L", packed)[0] 249 | except socket.error: 250 | packed = socket.inet_pton(socket.AF_INET6, ip_string) 251 | ip_string = socket.inet_ntop(socket.AF_INET6, packed) 252 | 253 | ipv6_words = struct.unpack("!8H", packed) 254 | ipv4_int = None 255 | 256 | return ip_string, ipv4_int, ipv6_words 257 | -------------------------------------------------------------------------------- /udger/queries.py: -------------------------------------------------------------------------------- 1 | 2 | def join_sql_columns(columns_dict, set_index=None): 3 | def columns(): 4 | for col_name, col_expression in columns_dict.items(): 5 | if set_index is not None: 6 | col_expression = col_expression[set_index] 7 | 8 | if col_expression == col_name: 9 | yield col_name 10 | else: 11 | yield '{0} AS {1}'.format(col_expression or 'NULL', col_name) 12 | 13 | return ',\n '.join(columns()) 14 | 15 | class Queries(object): 16 | client_columns = { 17 | 'client_id': (None, 'client_id'), 18 | 'class_id': (None, 'class_id'), 19 | 20 | 'ua_class': ('"Crawler"', 'client_classification'), 21 | 'ua_class_code': ('"crawler"', 'client_classification_code'), 22 | 'ua': ('name', 'name'), 23 | 'ua_engine': (None, 'engine'), 24 | 'ua_version': ('ver', None), 25 | 'ua_version_major': ('ver_major', None), 26 | 'crawler_last_seen': ('last_seen', None), 27 | 'crawler_respect_robotstxt': ('respect_robotstxt', None), 28 | 'crawler_category': ('crawler_classification', None), 29 | 'crawler_category_code': ('crawler_classification_code', None), 30 | 'ua_uptodate_current_version': (None, 'uptodate_current_version'), 31 | 'ua_family': ('family', 'name'), 32 | 'ua_family_code': ('family_code', 'name_code'), 33 | 'ua_family_homepage': ('family_homepage', 'homepage'), 34 | 'ua_family_icon': ('family_icon', 'icon'), 35 | 'ua_family_icon_big': (None, 'icon_big'), 36 | 'ua_family_vendor': ('vendor', 'vendor'), 37 | 'ua_family_vendor_code': ('vendor_code', 'vendor_code'), 38 | 'ua_family_vendor_homepage': ('vendor_homepage', 'vendor_homepage'), 39 | 'ua_family_info_url': ('"https://udger.com/resources/ua-list/bot-detail?bot=" || ' 40 | 'REPLACE(family, " ", "%20") || "#id" || udger_crawler_list.id', 41 | '"https://udger.com/resources/ua-list/browser-detail?browser=" || ' 42 | 'REPLACE(name, " ", "%20")'), 43 | } 44 | 45 | crawler_sql = """ 46 | SELECT 47 | %s 48 | FROM 49 | udger_crawler_list 50 | LEFT JOIN 51 | udger_crawler_class ON udger_crawler_class.id = udger_crawler_list.class_id 52 | WHERE 53 | ua_string = ? 54 | """ % join_sql_columns(client_columns, 0) 55 | 56 | client_sql = """ 57 | SELECT 58 | %s 59 | FROM 60 | udger_client_regex ur 61 | JOIN 62 | udger_client_list ON udger_client_list.id = ur.client_id 63 | JOIN 64 | udger_client_class ON udger_client_class.id = udger_client_list.class_id 65 | WHERE 66 | ur.rowid=? 67 | """ % join_sql_columns(client_columns, 1) 68 | 69 | os_columns = { 70 | 'os_family': 'family', 71 | 'os_family_code': 'family_code', 72 | 'os': 'name', 73 | 'os_code': 'name_code', 74 | 'os_homepage': 'homepage', 75 | 'os_icon': 'icon', 76 | 'os_icon_big': 'icon_big', 77 | 'os_family_vendor': 'vendor', 78 | 'os_family_vendor_code': 'vendor_code', 79 | 'os_family_vendor_homepage': 'vendor_homepage', 80 | 'os_info_url': '"https://udger.com/resources/ua-list/os-detail?os=" || ' 81 | 'REPLACE(name, " ", "%20")', 82 | } 83 | 84 | os_sql = """ 85 | SELECT 86 | %s 87 | FROM 88 | udger_os_regex ur 89 | JOIN 90 | udger_os_list ON udger_os_list.id = ur.os_id 91 | WHERE 92 | ur.rowid=? 93 | """ % join_sql_columns(os_columns) 94 | 95 | client_os_sql = """ 96 | SELECT 97 | %s 98 | FROM 99 | udger_client_os_relation 100 | JOIN 101 | udger_os_list ON udger_os_list.id = udger_client_os_relation.os_id 102 | WHERE 103 | client_id = ? 104 | """ % join_sql_columns(os_columns) 105 | 106 | device_columns = { 107 | 'device_class': 'name', 108 | 'device_class_code': 'name_code', 109 | 'device_class_icon': 'icon', 110 | 'device_class_icon_big': 'icon_big', 111 | 'device_class_info_url': '"https://udger.com/resources/ua-list/device-detail?device=" || ' 112 | 'REPLACE(name, " ", "%20")', 113 | } 114 | 115 | device_sql = """ 116 | SELECT 117 | %s 118 | FROM 119 | udger_deviceclass_regex ur 120 | JOIN 121 | udger_deviceclass_list ON udger_deviceclass_list.id = ur.deviceclass_id 122 | WHERE 123 | ur.rowid=? 124 | """ % join_sql_columns(device_columns) 125 | 126 | client_class_sql = """ 127 | SELECT 128 | %s 129 | FROM 130 | udger_deviceclass_list 131 | JOIN 132 | udger_client_class ON udger_client_class.deviceclass_id = udger_deviceclass_list.id 133 | WHERE 134 | udger_client_class.id = ? 135 | """ % join_sql_columns(device_columns) 136 | 137 | devicename_sql = """ 138 | SELECT 139 | id AS regex_id, 140 | regstring 141 | FROM 142 | udger_devicename_regex 143 | WHERE 144 | os_family_code = ? AND 145 | os_code IN ('-all-', ?) 146 | ORDER BY 147 | sequence 148 | """ 149 | 150 | marketname_columns = { 151 | 'device_marketname': 'marketname', 152 | 'device_brand': 'brand', 153 | 'device_brand_code': 'brand_code', 154 | 'device_brand_homepage': 'brand_url', 155 | 'device_brand_icon': 'icon', 156 | 'device_brand_icon_big': 'icon_big', 157 | 'device_brand_info_url': '"https://udger.com/resources/ua-list/devices-brand-detail?brand=" || ' 158 | 'REPLACE(brand_code, " ", "%20")', 159 | } 160 | 161 | marketname_sql = """ 162 | SELECT 163 | %s 164 | FROM 165 | udger_devicename_list 166 | JOIN 167 | udger_devicename_brand ON udger_devicename_brand.id = udger_devicename_list.brand_id 168 | WHERE 169 | regex_id = ? AND code = ? 170 | LIMIT 1 171 | """ % join_sql_columns(marketname_columns) 172 | 173 | ip_columns = { 174 | 'ip_classification': 'ip_classification', 175 | 'ip_classification_code': 'ip_classification_code', 176 | 'ip_last_seen': 'ip_last_seen', 177 | 'ip_hostname': 'ip_hostname', 178 | 'ip_country': 'ip_country', 179 | 'ip_country_code': 'ip_country_code', 180 | 'ip_city': 'ip_city', 181 | 'crawler_name': 'name', 182 | 'crawler_ver': 'ver', 183 | 'crawler_ver_major': 'ver_major', 184 | 'crawler_family': 'family', 185 | 'crawler_family_code': 'family_code', 186 | 'crawler_family_homepage': 'family_homepage', 187 | 'crawler_family_vendor': 'vendor', 188 | 'crawler_family_vendor_code': 'vendor_code', 189 | 'crawler_family_vendor_homepage': 'vendor_homepage', 190 | 'crawler_family_icon': 'family_icon', 191 | 'crawler_family_info_url': '"https://udger.com/resources/ua-list/bot-detail?bot=" || ' 192 | 'REPLACE(family, " ", "%20") || "#id" || udger_crawler_list.id', 193 | 'crawler_last_seen': 'last_seen', 194 | 'crawler_category': 'crawler_classification', 195 | 'crawler_category_code': 'crawler_classification_code', 196 | 'crawler_respect_robotstxt': 'respect_robotstxt', 197 | } 198 | 199 | ip_sql = """ 200 | SELECT 201 | %s 202 | FROM 203 | udger_ip_list 204 | JOIN 205 | udger_ip_class ON udger_ip_class.id=udger_ip_list.class_id 206 | LEFT JOIN 207 | udger_crawler_list ON udger_crawler_list.id=udger_ip_list.crawler_id 208 | LEFT JOIN 209 | udger_crawler_class ON udger_crawler_class.id=udger_crawler_list.class_id 210 | WHERE 211 | ip = ? 212 | ORDER BY 213 | sequence 214 | """ % join_sql_columns(ip_columns) 215 | 216 | datacenter_columns = { 217 | 'datacenter_name': 'name', 218 | 'datacenter_name_code': 'name_code', 219 | 'datacenter_homepage': 'homepage', 220 | } 221 | 222 | datacenter_sql = """ 223 | SELECT 224 | %s 225 | FROM 226 | udger_datacenter_range 227 | JOIN 228 | udger_datacenter_list ON udger_datacenter_range.datacenter_id = udger_datacenter_list.id 229 | WHERE 230 | iplong_from <= ? AND iplong_to >= ? 231 | """ % join_sql_columns(datacenter_columns) 232 | 233 | datacenter6_sql = """ 234 | SELECT 235 | %s 236 | FROM 237 | udger_datacenter_range6 238 | JOIN 239 | udger_datacenter_list ON udger_datacenter_range6.datacenter_id = udger_datacenter_list.id 240 | WHERE 241 | iplong_from0 <= ? AND iplong_from1 <= ? AND iplong_from2 <= ? AND iplong_from3 <= ? AND 242 | iplong_from4 <= ? AND iplong_from5 <= ? AND iplong_from6 <= ? AND iplong_from7 <= ? AND 243 | iplong_to0 >= ? AND iplong_to1 >= ? AND iplong_to2 >= ? AND iplong_to3 >= ? AND 244 | iplong_to4 >= ? AND iplong_to5 >= ? AND iplong_to6 >= ? AND iplong_to7 >= ? 245 | """ % join_sql_columns(datacenter_columns) 246 | 247 | -------------------------------------------------------------------------------- /udger/tests/test_ua.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "test": { 4 | "teststring": "Mozilla\/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko\/20100101 Firefox\/40.0" 5 | }, 6 | "ret": { 7 | "ua_string": "Mozilla\/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko\/20100101 Firefox\/40.0", 8 | "ua_class": "Browser", 9 | "ua_class_code": "browser", 10 | "ua": "Firefox 40.0", 11 | "ua_version": "40.0", 12 | "ua_version_major": "40", 13 | "ua_uptodate_current_version": "50", 14 | "ua_family": "Firefox", 15 | "ua_family_code": "firefox", 16 | "ua_family_homepage": "http:\/\/www.firefox.com\/", 17 | "ua_family_vendor": "Mozilla Foundation", 18 | "ua_family_vendor_code": "mozilla_foundation", 19 | "ua_family_vendor_homepage": "http:\/\/www.mozilla.org\/", 20 | "ua_family_icon": "firefox.png", 21 | "ua_family_icon_big": "firefox_big.png", 22 | "ua_family_info_url": "https:\/\/udger.com\/resources\/ua-list\/browser-detail?browser=Firefox", 23 | "ua_engine": "Gecko", 24 | "os": "Windows 10", 25 | "os_code": "windows_10", 26 | "os_homepage": "https:\/\/en.wikipedia.org\/wiki\/Windows_10", 27 | "os_icon": "windows10.png", 28 | "os_icon_big": "windows10_big.png", 29 | "os_info_url": "https:\/\/udger.com\/resources\/ua-list\/os-detail?os=Windows 10", 30 | "os_family": "Windows", 31 | "os_family_code": "windows", 32 | "os_family_vendor": "Microsoft Corporation.", 33 | "os_family_vendor_code": "microsoft_corporation", 34 | "os_family_vendor_homepage": "https:\/\/www.microsoft.com\/about\/", 35 | "device_class": "Desktop", 36 | "device_class_code": "desktop", 37 | "device_class_icon": "desktop.png", 38 | "device_class_icon_big": "desktop_big.png", 39 | "device_class_info_url": "https:\/\/udger.com\/resources\/ua-list\/device-detail?device=Desktop", 40 | "device_marketname": "", 41 | "device_brand": "", 42 | "device_brand_code": "", 43 | "device_brand_homepage": "", 44 | "device_brand_icon": "", 45 | "device_brand_icon_big": "", 46 | "device_brand_info_url": "", 47 | "crawler_category": "", 48 | "crawler_category_code": "", 49 | "crawler_respect_robotstxt": "" 50 | } 51 | }, 52 | { 53 | "test": { 54 | "teststring": "Mozilla\/5.0 (compatible; SeznamBot\/3.2; +http:\/\/fulltext.sblog.cz\/)" 55 | }, 56 | "ret": { 57 | "ua_string": "Mozilla\/5.0 (compatible; SeznamBot\/3.2; +http:\/\/fulltext.sblog.cz\/)", 58 | "ua_class": "Crawler", 59 | "ua_class_code": "crawler", 60 | "ua": "SeznamBot\/3.2", 61 | "ua_version": "3.2", 62 | "ua_version_major": "3", 63 | "ua_uptodate_current_version": "", 64 | "ua_family": "SeznamBot", 65 | "ua_family_code": "seznambot", 66 | "ua_family_homepage": "http:\/\/napoveda.seznam.cz\/en\/seznambot-intro\/", 67 | "ua_family_vendor": "Seznam.cz, a.s.", 68 | "ua_family_vendor_code": "seznam-cz_as", 69 | "ua_family_vendor_homepage": "http:\/\/onas.seznam.cz\/", 70 | "ua_family_icon": "seznam.png", 71 | "ua_family_icon_big": "", 72 | "ua_family_info_url": "https:\/\/udger.com\/resources\/ua-list\/bot-detail?bot=SeznamBot#id12571", 73 | "ua_engine": "", 74 | "os": "", 75 | "os_code": "", 76 | "os_homepage": "", 77 | "os_icon": "", 78 | "os_icon_big": "", 79 | "os_info_url": "", 80 | "os_family": "", 81 | "os_family_code": "", 82 | "os_family_vendor": "", 83 | "os_family_vendor_code": "", 84 | "os_family_vendor_homepage": "", 85 | "device_class": "Unrecognized", 86 | "device_class_code": "unrecognized", 87 | "device_class_icon": "other.png", 88 | "device_class_icon_big": "other_big.png", 89 | "device_class_info_url": "https:\/\/udger.com\/resources\/ua-list\/device-detail?device=Unrecognized", 90 | "device_marketname": "", 91 | "device_brand": "", 92 | "device_brand_code": "", 93 | "device_brand_homepage": "", 94 | "device_brand_icon": "", 95 | "device_brand_icon_big": "", 96 | "device_brand_info_url": "", 97 | "crawler_category": "Search engine bot", 98 | "crawler_category_code": "search_engine_bot", 99 | "crawler_respect_robotstxt": "yes" 100 | } 101 | }, 102 | { 103 | "test": { 104 | "teststring": "Mozilla\/5.0 (iPad; CPU OS 7_0 like Mac OS X) AppleWebKit\/537.51.1 (KHTML, like Gecko) Version\/7.0 Mobile\/11A465 Safari\/9537.53" 105 | }, 106 | "ret": { 107 | "ua_string": "Mozilla\/5.0 (iPad; CPU OS 7_0 like Mac OS X) AppleWebKit\/537.51.1 (KHTML, like Gecko) Version\/7.0 Mobile\/11A465 Safari\/9537.53", 108 | "ua_class": "Mobile browser", 109 | "ua_class_code": "mobile_browser", 110 | "ua": "Safari mobile 7.0", 111 | "ua_version": "7.0", 112 | "ua_version_major": "7", 113 | "ua_uptodate_current_version": "", 114 | "ua_family": "Safari mobile", 115 | "ua_family_code": "safari_mobile", 116 | "ua_family_homepage": "https:\/\/en.wikipedia.org\/wiki\/Safari_%28web_browser%29", 117 | "ua_family_vendor": "Apple Inc.", 118 | "ua_family_vendor_code": "apple_inc", 119 | "ua_family_vendor_homepage": "http:\/\/www.apple.com\/", 120 | "ua_family_icon": "safari.png", 121 | "ua_family_icon_big": "safari_big.png", 122 | "ua_family_info_url": "https:\/\/udger.com\/resources\/ua-list\/browser-detail?browser=Safari mobile", 123 | "ua_engine": "WebKit", 124 | "os": "iOS 7", 125 | "os_code": "ios_7", 126 | "os_homepage": "https:\/\/en.wikipedia.org\/wiki\/IOS_7", 127 | "os_icon": "iphone.png", 128 | "os_icon_big": "iphone_big.png", 129 | "os_info_url": "https:\/\/udger.com\/resources\/ua-list\/os-detail?os=iOS 7", 130 | "os_family": "iOS", 131 | "os_family_code": "ios", 132 | "os_family_vendor": "Apple Inc.", 133 | "os_family_vendor_code": "apple_inc", 134 | "os_family_vendor_homepage": "http:\/\/www.apple.com\/", 135 | "device_class": "Tablet", 136 | "device_class_code": "tablet", 137 | "device_class_icon": "tablet.png", 138 | "device_class_icon_big": "tablet_big.png", 139 | "device_class_info_url": "https:\/\/udger.com\/resources\/ua-list\/device-detail?device=Tablet", 140 | "device_marketname": "iPad", 141 | "device_brand": "Apple", 142 | "device_brand_code": "apple", 143 | "device_brand_homepage": "http:\/\/www.apple.com\/", 144 | "device_brand_icon": "apple.png", 145 | "device_brand_icon_big": "apple_big.png", 146 | "device_brand_info_url": "https:\/\/udger.com\/resources\/ua-list\/devices-brand-detail?brand=apple", 147 | "crawler_category": "", 148 | "crawler_category_code": "", 149 | "crawler_respect_robotstxt": "" 150 | } 151 | }, 152 | { 153 | "test": { 154 | "teststring": "Mozilla\/5.0 (Linux; U; Android 4.0.4; sk-sk; Luna TAB474 Build\/LunaTAB474) AppleWebKit\/534.30 (KHTML, like Gecko) Version\/4.0 Safari\/534.30" 155 | }, 156 | "ret": { 157 | "ua_string": "Mozilla\/5.0 (Linux; U; Android 4.0.4; sk-sk; Luna TAB474 Build\/LunaTAB474) AppleWebKit\/534.30 (KHTML, like Gecko) Version\/4.0 Safari\/534.30", 158 | "ua_class": "Mobile browser", 159 | "ua_class_code": "mobile_browser", 160 | "ua": "Android browser 4.0", 161 | "ua_version": "4.0", 162 | "ua_version_major": "4", 163 | "ua_uptodate_current_version": "", 164 | "ua_family": "Android browser", 165 | "ua_family_code": "android_browser", 166 | "ua_family_homepage": "http:\/\/developer.android.com\/reference\/android\/webkit\/package-summary.html", 167 | "ua_family_vendor": "Google Inc.", 168 | "ua_family_vendor_code": "google_inc", 169 | "ua_family_vendor_homepage": "https:\/\/www.google.com\/about\/company\/", 170 | "ua_family_icon": "androidWebkit.png", 171 | "ua_family_icon_big": "androidWebkit_big.png", 172 | "ua_family_info_url": "https:\/\/udger.com\/resources\/ua-list\/browser-detail?browser=Android browser", 173 | "ua_engine": "WebKit", 174 | "os": "Android 4.0.x Ice Cream Sandwich", 175 | "os_code": "android_4", 176 | "os_homepage": "https:\/\/en.wikipedia.org\/wiki\/Android_%28operating_system%29", 177 | "os_icon": "android.png", 178 | "os_icon_big": "android_big.png", 179 | "os_info_url": "https:\/\/udger.com\/resources\/ua-list\/os-detail?os=Android 4.0.x Ice Cream Sandwich", 180 | "os_family": "Android", 181 | "os_family_code": "android", 182 | "os_family_vendor": "Google, Inc.", 183 | "os_family_vendor_code": "google_inc", 184 | "os_family_vendor_homepage": "https:\/\/www.google.com\/about\/company\/", 185 | "device_class": "Tablet", 186 | "device_class_code": "tablet", 187 | "device_class_icon": "tablet.png", 188 | "device_class_icon_big": "tablet_big.png", 189 | "device_class_info_url": "https:\/\/udger.com\/resources\/ua-list\/device-detail?device=Tablet", 190 | "device_marketname": "Luna 10", 191 | "device_brand": "Yarvik", 192 | "device_brand_code": "yarvik", 193 | "device_brand_homepage": "http:\/\/yarvik.com\/", 194 | "device_brand_icon": "yarvik.png", 195 | "device_brand_icon_big": "", 196 | "device_brand_info_url": "https:\/\/udger.com\/resources\/ua-list\/devices-brand-detail?brand=yarvik", 197 | "crawler_category": "", 198 | "crawler_category_code": "", 199 | "crawler_respect_robotstxt": "" 200 | } 201 | }, 202 | { 203 | "test": { 204 | "teststring": "Mozilla\/5.0 (Playstation Vita 1.61) AppleWebKit\/531.22.8 (KHTML, like Gecko) Silk\/3.2" 205 | }, 206 | "ret": { 207 | "ua_string": "Mozilla\/5.0 (Playstation Vita 1.61) AppleWebKit\/531.22.8 (KHTML, like Gecko) Silk\/3.2", 208 | "ua_class": "Mobile browser", 209 | "ua_class_code": "mobile_browser", 210 | "ua": "PS Vita browser 3.2", 211 | "ua_version": "3.2", 212 | "ua_version_major": "3", 213 | "ua_uptodate_current_version": "", 214 | "ua_family": "PS Vita browser", 215 | "ua_family_code": "ps_vita_browser", 216 | "ua_family_homepage": "", 217 | "ua_family_vendor": "Sony Computer Entertainment", 218 | "ua_family_vendor_code": "sony_computer_entertainment", 219 | "ua_family_vendor_homepage": "http:\/\/www.scei.co.jp\/", 220 | "ua_family_icon": "ps-vita-browser.png", 221 | "ua_family_icon_big": "", 222 | "ua_family_info_url": "https:\/\/udger.com\/resources\/ua-list\/browser-detail?browser=PS Vita browser", 223 | "ua_engine": "WebKit", 224 | "os": "LiveArea", 225 | "os_code": "live_area", 226 | "os_homepage": "https:\/\/en.wikipedia.org\/wiki\/LiveArea", 227 | "os_icon": "ps-vitaLiveArea.png", 228 | "os_icon_big": "", 229 | "os_info_url": "https:\/\/udger.com\/resources\/ua-list\/os-detail?os=LiveArea", 230 | "os_family": "LiveArea", 231 | "os_family_code": "live_area", 232 | "os_family_vendor": "Sony Computer Entertainment", 233 | "os_family_vendor_code": "sony", 234 | "os_family_vendor_homepage": "http:\/\/www.scei.co.jp\/", 235 | "device_class": "Game console", 236 | "device_class_code": "game_console", 237 | "device_class_icon": "console.png", 238 | "device_class_icon_big": "console_big.png", 239 | "device_class_info_url": "https:\/\/udger.com\/resources\/ua-list\/device-detail?device=Game console", 240 | "device_marketname": "Playstation Vita", 241 | "device_brand": "Sony", 242 | "device_brand_code": "sony", 243 | "device_brand_homepage": "http:\/\/www.sony.com\/", 244 | "device_brand_icon": "sony.png", 245 | "device_brand_icon_big": "sony_big.png", 246 | "device_brand_info_url": "https:\/\/udger.com\/resources\/ua-list\/devices-brand-detail?brand=sony", 247 | "crawler_category": "", 248 | "crawler_category_code": "", 249 | "crawler_respect_robotstxt": "" 250 | } 251 | }, 252 | { 253 | "test": { 254 | "teststring": "Mozilla\/5.0 (CrKey armv7l 1.4.15250) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/31.0.1650.0 Safari\/537.36" 255 | }, 256 | "ret": { 257 | "ua_string": "Mozilla\/5.0 (CrKey armv7l 1.4.15250) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/31.0.1650.0 Safari\/537.36", 258 | "ua_class": "Browser", 259 | "ua_class_code": "browser", 260 | "ua": "Chrome 31.0.1650.0", 261 | "ua_version": "31.0.1650.0", 262 | "ua_version_major": "31", 263 | "ua_uptodate_current_version": "55", 264 | "ua_family": "Chrome", 265 | "ua_family_code": "chrome", 266 | "ua_family_homepage": "http:\/\/www.google.com\/chrome\/", 267 | "ua_family_vendor": "Google Inc.", 268 | "ua_family_vendor_code": "google_inc", 269 | "ua_family_vendor_homepage": "https:\/\/www.google.com\/about\/company\/", 270 | "ua_family_icon": "chrome.png", 271 | "ua_family_icon_big": "chrome_big.png", 272 | "ua_family_info_url": "https:\/\/udger.com\/resources\/ua-list\/browser-detail?browser=Chrome", 273 | "ua_engine": "WebKit\/Blink", 274 | "os": "Android", 275 | "os_code": "android", 276 | "os_homepage": "https:\/\/en.wikipedia.org\/wiki\/Android_%28operating_system%29", 277 | "os_icon": "android.png", 278 | "os_icon_big": "android_big.png", 279 | "os_info_url": "https:\/\/udger.com\/resources\/ua-list\/os-detail?os=Android", 280 | "os_family": "Android", 281 | "os_family_code": "android", 282 | "os_family_vendor": "Google, Inc.", 283 | "os_family_vendor_code": "google_inc", 284 | "os_family_vendor_homepage": "https:\/\/www.google.com\/about\/company\/", 285 | "device_class": "Smart TV", 286 | "device_class_code": "smart_tv", 287 | "device_class_icon": "smarttv.png", 288 | "device_class_icon_big": "smarttv_big.png", 289 | "device_class_info_url": "https:\/\/udger.com\/resources\/ua-list\/device-detail?device=Smart TV", 290 | "device_marketname": "Chromecast", 291 | "device_brand": "Google", 292 | "device_brand_code": "google", 293 | "device_brand_homepage": "https:\/\/www.google.com\/", 294 | "device_brand_icon": "google.png", 295 | "device_brand_icon_big": "google_big.png", 296 | "device_brand_info_url": "https:\/\/udger.com\/resources\/ua-list\/devices-brand-detail?brand=google", 297 | "crawler_category": "", 298 | "crawler_category_code": "", 299 | "crawler_respect_robotstxt": "" 300 | } 301 | }, 302 | { 303 | "test": { 304 | "teststring": "Mozilla\/5.0 (compatible; MSIE 10.0; Windows Phone 8.0; Trident\/6.0; IEMobile\/10.0; ARM; Touch; NOKIA; Lumia 820)" 305 | }, 306 | "ret": { 307 | "ua_string": "Mozilla\/5.0 (compatible; MSIE 10.0; Windows Phone 8.0; Trident\/6.0; IEMobile\/10.0; ARM; Touch; NOKIA; Lumia 820)", 308 | "ua_class": "Mobile browser", 309 | "ua_class_code": "mobile_browser", 310 | "ua": "IE Mobile 10.0", 311 | "ua_version": "10.0", 312 | "ua_version_major": "10", 313 | "ua_uptodate_current_version": "", 314 | "ua_family": "IE Mobile", 315 | "ua_family_code": "ie_mobile", 316 | "ua_family_homepage": "https:\/\/en.wikipedia.org\/wiki\/Internet_Explorer_Mobile", 317 | "ua_family_vendor": "Microsoft Corporation.", 318 | "ua_family_vendor_code": "microsoft_corporation", 319 | "ua_family_vendor_homepage": "https:\/\/www.microsoft.com\/about\/", 320 | "ua_family_icon": "iemobile.png", 321 | "ua_family_icon_big": "", 322 | "ua_family_info_url": "https:\/\/udger.com\/resources\/ua-list\/browser-detail?browser=IE Mobile", 323 | "ua_engine": "Trident", 324 | "os": "Windows Phone 8", 325 | "os_code": "windows_phone_8", 326 | "os_homepage": "https:\/\/en.wikipedia.org\/wiki\/Windows_Phone_8", 327 | "os_icon": "windowsPhone8.png", 328 | "os_icon_big": "", 329 | "os_info_url": "https:\/\/udger.com\/resources\/ua-list\/os-detail?os=Windows Phone 8", 330 | "os_family": "Windows", 331 | "os_family_code": "windows", 332 | "os_family_vendor": "Microsoft Corporation.", 333 | "os_family_vendor_code": "microsoft_corporation", 334 | "os_family_vendor_homepage": "https:\/\/www.microsoft.com\/about\/", 335 | "device_class": "Smartphone", 336 | "device_class_code": "smartphone", 337 | "device_class_icon": "phone.png", 338 | "device_class_icon_big": "phone_big.png", 339 | "device_class_info_url": "https:\/\/udger.com\/resources\/ua-list\/device-detail?device=Smartphone", 340 | "device_marketname": "Lumia 820", 341 | "device_brand": "Nokia", 342 | "device_brand_code": "nokia", 343 | "device_brand_homepage": "http:\/\/www.nokia.com\/", 344 | "device_brand_icon": "nokia.png", 345 | "device_brand_icon_big": "nokia_big.png", 346 | "device_brand_info_url": "https:\/\/udger.com\/resources\/ua-list\/devices-brand-detail?brand=nokia", 347 | "crawler_category": "", 348 | "crawler_category_code": "", 349 | "crawler_respect_robotstxt": "" 350 | } 351 | }, 352 | { 353 | "test": { 354 | "teststring": "Mozilla\/5.0 (Linux; U; Android 4.0.4; en-us; Glass 1 Build\/IMM76L; XE9) AppleWebKit\/534.30 (KHTML, like Gecko) Version\/4.0 Mobile Safari\/534.30" 355 | }, 356 | "ret": { 357 | "ua_string": "Mozilla\/5.0 (Linux; U; Android 4.0.4; en-us; Glass 1 Build\/IMM76L; XE9) AppleWebKit\/534.30 (KHTML, like Gecko) Version\/4.0 Mobile Safari\/534.30", 358 | "ua_class": "Mobile browser", 359 | "ua_class_code": "mobile_browser", 360 | "ua": "Android browser 4.0", 361 | "ua_version": "4.0", 362 | "ua_version_major": "4", 363 | "ua_uptodate_current_version": "", 364 | "ua_family": "Android browser", 365 | "ua_family_code": "android_browser", 366 | "ua_family_homepage": "http:\/\/developer.android.com\/reference\/android\/webkit\/package-summary.html", 367 | "ua_family_vendor": "Google Inc.", 368 | "ua_family_vendor_code": "google_inc", 369 | "ua_family_vendor_homepage": "https:\/\/www.google.com\/about\/company\/", 370 | "ua_family_icon": "androidWebkit.png", 371 | "ua_family_icon_big": "androidWebkit_big.png", 372 | "ua_family_info_url": "https:\/\/udger.com\/resources\/ua-list\/browser-detail?browser=Android browser", 373 | "ua_engine": "WebKit", 374 | "os": "Android 4.0.x Ice Cream Sandwich", 375 | "os_code": "android_4", 376 | "os_homepage": "https:\/\/en.wikipedia.org\/wiki\/Android_%28operating_system%29", 377 | "os_icon": "android.png", 378 | "os_icon_big": "android_big.png", 379 | "os_info_url": "https:\/\/udger.com\/resources\/ua-list\/os-detail?os=Android 4.0.x Ice Cream Sandwich", 380 | "os_family": "Android", 381 | "os_family_code": "android", 382 | "os_family_vendor": "Google, Inc.", 383 | "os_family_vendor_code": "google_inc", 384 | "os_family_vendor_homepage": "https:\/\/www.google.com\/about\/company\/", 385 | "device_class": "Wearable computer", 386 | "device_class_code": "wearable_computer", 387 | "device_class_icon": "wearable.png", 388 | "device_class_icon_big": "wearable_big.png", 389 | "device_class_info_url": "https:\/\/udger.com\/resources\/ua-list\/device-detail?device=Wearable computer", 390 | "device_marketname": "Google Glass", 391 | "device_brand": "Google", 392 | "device_brand_code": "google", 393 | "device_brand_homepage": "https:\/\/www.google.com\/", 394 | "device_brand_icon": "google.png", 395 | "device_brand_icon_big": "google_big.png", 396 | "device_brand_info_url": "https:\/\/udger.com\/resources\/ua-list\/devices-brand-detail?brand=google", 397 | "crawler_category": "", 398 | "crawler_category_code": "", 399 | "crawler_respect_robotstxt": "" 400 | } 401 | }, 402 | { 403 | "test": { 404 | "teststring": "Microsoft Office\/16.0 (Windows NT 10.0; Microsoft Outlook 16.0.6326; Pro)" 405 | }, 406 | "ret": { 407 | "ua_string": "Microsoft Office\/16.0 (Windows NT 10.0; Microsoft Outlook 16.0.6326; Pro)", 408 | "ua_class": "E-mail client", 409 | "ua_class_code": "email_client", 410 | "ua": "Outlook 2016 ", 411 | "ua_version": "", 412 | "ua_version_major": "", 413 | "ua_uptodate_current_version": "", 414 | "ua_family": "Outlook 2016", 415 | "ua_family_code": "outlook_2016", 416 | "ua_family_homepage": "https:\/\/en.wikipedia.org\/wiki\/Microsoft_Outlook", 417 | "ua_family_vendor": "Microsoft Corporation.", 418 | "ua_family_vendor_code": "microsoft_corporation", 419 | "ua_family_vendor_homepage": "https:\/\/www.microsoft.com\/about\/", 420 | "ua_family_icon": "outlook2016.png", 421 | "ua_family_icon_big": "", 422 | "ua_family_info_url": "https:\/\/udger.com\/resources\/ua-list\/browser-detail?browser=Outlook 2016", 423 | "ua_engine": "", 424 | "os": "Windows 10", 425 | "os_code": "windows_10", 426 | "os_homepage": "https:\/\/en.wikipedia.org\/wiki\/Windows_10", 427 | "os_icon": "windows10.png", 428 | "os_icon_big": "windows10_big.png", 429 | "os_info_url": "https:\/\/udger.com\/resources\/ua-list\/os-detail?os=Windows 10", 430 | "os_family": "Windows", 431 | "os_family_code": "windows", 432 | "os_family_vendor": "Microsoft Corporation.", 433 | "os_family_vendor_code": "microsoft_corporation", 434 | "os_family_vendor_homepage": "https:\/\/www.microsoft.com\/about\/", 435 | "device_class": "Desktop", 436 | "device_class_code": "desktop", 437 | "device_class_icon": "desktop.png", 438 | "device_class_icon_big": "desktop_big.png", 439 | "device_class_info_url": "https:\/\/udger.com\/resources\/ua-list\/device-detail?device=Desktop", 440 | "device_marketname": "", 441 | "device_brand": "", 442 | "device_brand_code": "", 443 | "device_brand_homepage": "", 444 | "device_brand_icon": "", 445 | "device_brand_icon_big": "", 446 | "device_brand_info_url": "", 447 | "crawler_category": "", 448 | "crawler_category_code": "", 449 | "crawler_respect_robotstxt": "" 450 | } 451 | }, 452 | { 453 | "test": { 454 | "teststring": "Mozilla\/5.0 (compatible; Googlebot\/2.1; +http:\/\/www.google.com\/bot.html)" 455 | }, 456 | "ret": { 457 | "ua_string": "Mozilla\/5.0 (compatible; Googlebot\/2.1; +http:\/\/www.google.com\/bot.html)", 458 | "ua_class": "Crawler", 459 | "ua_class_code": "crawler", 460 | "ua": "Googlebot\/2.1", 461 | "ua_version": "2.1", 462 | "ua_version_major": "2", 463 | "ua_uptodate_current_version": "", 464 | "ua_family": "Googlebot", 465 | "ua_family_code": "googlebot", 466 | "ua_family_homepage": "http:\/\/www.google.com\/bot.html", 467 | "ua_family_vendor": "Google Inc.", 468 | "ua_family_vendor_code": "google_inc", 469 | "ua_family_vendor_homepage": "https:\/\/www.google.com\/about\/company\/", 470 | "ua_family_icon": "bot_googlebot.png", 471 | "ua_family_icon_big": "", 472 | "ua_family_info_url": "https:\/\/udger.com\/resources\/ua-list\/bot-detail?bot=Googlebot#id31", 473 | "ua_engine": "", 474 | "os": "", 475 | "os_code": "", 476 | "os_homepage": "", 477 | "os_icon": "", 478 | "os_icon_big": "", 479 | "os_info_url": "", 480 | "os_family": "", 481 | "os_family_code": "", 482 | "os_family_vendor": "", 483 | "os_family_vendor_code": "", 484 | "os_family_vendor_homepage": "", 485 | "device_class": "Unrecognized", 486 | "device_class_code": "unrecognized", 487 | "device_class_icon": "other.png", 488 | "device_class_icon_big": "other_big.png", 489 | "device_class_info_url": "https:\/\/udger.com\/resources\/ua-list\/device-detail?device=Unrecognized", 490 | "device_marketname": "", 491 | "device_brand": "", 492 | "device_brand_code": "", 493 | "device_brand_homepage": "", 494 | "device_brand_icon": "", 495 | "device_brand_icon_big": "", 496 | "device_brand_info_url": "", 497 | "crawler_category": "Search engine bot", 498 | "crawler_category_code": "search_engine_bot", 499 | "crawler_respect_robotstxt": "yes" 500 | } 501 | }, 502 | { 503 | "test": { 504 | "teststring": "PINGOMETER_BOT_(HTTPS:\/\/PINGOMETER.COM)" 505 | }, 506 | "ret": { 507 | "ua_string": "PINGOMETER_BOT_(HTTPS:\/\/PINGOMETER.COM)", 508 | "ua_class": "Crawler", 509 | "ua_class_code": "crawler", 510 | "ua": "PINGOMETER", 511 | "ua_version": "", 512 | "ua_version_major": "", 513 | "ua_uptodate_current_version": "", 514 | "ua_family": "PINGOMETER", 515 | "ua_family_code": "pingometer", 516 | "ua_family_homepage": "", 517 | "ua_family_vendor": "Pingometer, LLC", 518 | "ua_family_vendor_code": "pingometer_llc", 519 | "ua_family_vendor_homepage": "http:\/\/pingometer.com\/", 520 | "ua_family_icon": "bot_pingometer.png", 521 | "ua_family_icon_big": "", 522 | "ua_family_info_url": "https:\/\/udger.com\/resources\/ua-list\/bot-detail?bot=PINGOMETER#id20112", 523 | "ua_engine": "", 524 | "os": "", 525 | "os_code": "", 526 | "os_homepage": "", 527 | "os_icon": "", 528 | "os_icon_big": "", 529 | "os_info_url": "", 530 | "os_family": "", 531 | "os_family_code": "", 532 | "os_family_vendor": "", 533 | "os_family_vendor_code": "", 534 | "os_family_vendor_homepage": "", 535 | "device_class": "Unrecognized", 536 | "device_class_code": "unrecognized", 537 | "device_class_icon": "other.png", 538 | "device_class_icon_big": "other_big.png", 539 | "device_class_info_url": "https:\/\/udger.com\/resources\/ua-list\/device-detail?device=Unrecognized", 540 | "device_marketname": "", 541 | "device_brand": "", 542 | "device_brand_code": "", 543 | "device_brand_homepage": "", 544 | "device_brand_icon": "", 545 | "device_brand_icon_big": "", 546 | "device_brand_info_url": "", 547 | "crawler_category": "Site monitor", 548 | "crawler_category_code": "site_monitor", 549 | "crawler_respect_robotstxt": "no" 550 | } 551 | }, 552 | { 553 | "test": { 554 | "teststring": "Monogram\/1.3 CFNetwork\/758.2.8 Darwin\/15.0.0" 555 | }, 556 | "ret": { 557 | "ua_string": "Monogram\/1.3 CFNetwork\/758.2.8 Darwin\/15.0.0", 558 | "ua_class": "Unrecognized", 559 | "ua_class_code": "unrecognized", 560 | "ua": "", 561 | "ua_version": "", 562 | "ua_version_major": "", 563 | "ua_uptodate_current_version": "", 564 | "ua_family": "", 565 | "ua_family_code": "", 566 | "ua_family_homepage": "", 567 | "ua_family_vendor": "", 568 | "ua_family_vendor_code": "", 569 | "ua_family_vendor_homepage": "", 570 | "ua_family_icon": "", 571 | "ua_family_icon_big": "", 572 | "ua_family_info_url": "", 573 | "ua_engine": "", 574 | "os": "iOS 9", 575 | "os_code": "ios_9", 576 | "os_homepage": "https:\/\/en.wikipedia.org\/wiki\/IOS_9", 577 | "os_icon": "iphone.png", 578 | "os_icon_big": "iphone_big.png", 579 | "os_info_url": "https:\/\/udger.com\/resources\/ua-list\/os-detail?os=iOS 9", 580 | "os_family": "iOS", 581 | "os_family_code": "ios", 582 | "os_family_vendor": "Apple Inc.", 583 | "os_family_vendor_code": "apple_inc", 584 | "os_family_vendor_homepage": "http:\/\/www.apple.com\/", 585 | "device_class": "", 586 | "device_class_code": "", 587 | "device_class_icon": "", 588 | "device_class_icon_big": "", 589 | "device_class_info_url": "", 590 | "device_marketname": "", 591 | "device_brand": "", 592 | "device_brand_code": "", 593 | "device_brand_homepage": "", 594 | "device_brand_icon": "", 595 | "device_brand_icon_big": "", 596 | "device_brand_info_url": "", 597 | "crawler_category": "", 598 | "crawler_category_code": "", 599 | "crawler_respect_robotstxt": "" 600 | } 601 | }, 602 | { 603 | "test": { 604 | "teststring": "lorem ipsum dolor sit amet" 605 | }, 606 | "ret": { 607 | "ua_string": "lorem ipsum dolor sit amet", 608 | "ua_class": "Unrecognized", 609 | "ua_class_code": "unrecognized", 610 | "ua": "", 611 | "ua_version": "", 612 | "ua_version_major": "", 613 | "ua_uptodate_current_version": "", 614 | "ua_family": "", 615 | "ua_family_code": "", 616 | "ua_family_homepage": "", 617 | "ua_family_vendor": "", 618 | "ua_family_vendor_code": "", 619 | "ua_family_vendor_homepage": "", 620 | "ua_family_icon": "", 621 | "ua_family_icon_big": "", 622 | "ua_family_info_url": "", 623 | "ua_engine": "", 624 | "os": "", 625 | "os_code": "", 626 | "os_homepage": "", 627 | "os_icon": "", 628 | "os_icon_big": "", 629 | "os_info_url": "", 630 | "os_family": "", 631 | "os_family_code": "", 632 | "os_family_vendor": "", 633 | "os_family_vendor_code": "", 634 | "os_family_vendor_homepage": "", 635 | "device_class": "", 636 | "device_class_code": "", 637 | "device_class_icon": "", 638 | "device_class_icon_big": "", 639 | "device_class_info_url": "", 640 | "device_marketname": "", 641 | "device_brand": "", 642 | "device_brand_code": "", 643 | "device_brand_homepage": "", 644 | "device_brand_icon": "", 645 | "device_brand_icon_big": "", 646 | "device_brand_info_url": "", 647 | "crawler_category": "", 648 | "crawler_category_code": "", 649 | "crawler_respect_robotstxt": "" 650 | } 651 | } 652 | ] --------------------------------------------------------------------------------