├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── pyipip ├── __init__.py └── ipipdb.py ├── setup.cfg ├── setup.py ├── tests ├── 17monipdb.dat ├── 17monipdb.datx ├── bench.py ├── ips.gz ├── reqs_bench.txt └── test_pyipip.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | tags 3 | .cache 4 | 5 | venv 6 | dev_venv 7 | *.egg-info 8 | dist 9 | build 10 | .tox 11 | 12 | *.dat 13 | *.datx 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | 4 | matrix: 5 | include: 6 | - python: 2.7 7 | - python: 3.4 8 | - python: 3.5 9 | - python: 3.6 10 | - python: 3.7 11 | dist: xenial 12 | - python: nightly 13 | - python: pypy 14 | - python: pypy3 15 | 16 | install: 17 | - pip install tox 18 | 19 | script: 20 | - tox -e py 21 | 22 | cache: 23 | - pip 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: help clean test bench package publish 2 | 3 | help: 4 | @echo targets: test bench publish clean 5 | 6 | venv: 7 | virtualenv -q venv 8 | ./venv/bin/pip install -q -e . 9 | 10 | test: venv 11 | @./venv/bin/python tests/test_pyipip.py 12 | 13 | bench: venv 14 | ./venv/bin/pip install -q -r tests/reqs_bench.txt 15 | ./venv/bin/python tests/bench.py 16 | 17 | package: clean 18 | python setup.py sdist bdist_wheel 19 | 20 | publish: package 21 | twine upload dist/* 22 | 23 | clean: 24 | rm -rf venv 25 | rm -rf build dist *.egg-info 26 | find . -name '*.pyc' -delete 27 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | PyIPIP 2 | ====== 3 | 4 | .. image:: https://travis-ci.org/georgexsh/pyipip.svg?branch=master 5 | :target: https://travis-ci.org/georgexsh/pyipip 6 | 7 | .. image:: https://img.shields.io/pypi/v/pyipip.svg 8 | :target: https://pypi.org/project/pyipip/ 9 | 10 | 11 | Description 12 | ----------- 13 | 14 | ipip.net IPv4 地址归属地数据库 Python 查询库。同时支持 dat 与 datx 15 | 格式的数据文件，支持 Python 2 与 3。 需要先去 `ipip.net 16 | 官方网站 `__ 下载数据文件。 17 | 18 | 性能较官方库为高，在 E5-2682 2.5GHz 下 QPS 约为 490k:: 19 | 20 | pyipip 491507.74 21 | lxyu 101062.99 22 | official 5701.01 23 | 24 | 25 | Usage 26 | ----- 27 | 28 | .. code:: python 29 | 30 | >>> from pyipip import IPIPDatabase 31 | >>> db = IPIPDatabase('/path/to/your/ipipdb.dat') 32 | >>> db.lookup('202.112.80.106') 33 | '中国\t北京\t北京\t' 34 | 35 | Install 36 | ~~~~~~~ 37 | 38 | :: 39 | 40 | pip install pyipip 41 | 42 | Test & Benchmark 43 | ~~~~~~~~~~~~~~~~ 44 | 45 | :: 46 | 47 | make test 48 | 49 | :: 50 | 51 | make bench 52 | 53 | Note 54 | ---- 55 | 56 | 数据文件时时更新，请自行下载使用，代码仓库随附的数据文件只用于测试。 57 | -------------------------------------------------------------------------------- /pyipip/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.1.1' 2 | 3 | import logging 4 | logging.getLogger(__name__).addHandler(logging.NullHandler()) 5 | 6 | from .ipipdb import IPIPDatabase 7 | 8 | -------------------------------------------------------------------------------- /pyipip/ipipdb.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | import os 4 | import sys 5 | import time 6 | import array 7 | import struct 8 | import logging 9 | import platform 10 | 11 | # speed-up attr lookup 12 | from struct import unpack 13 | from socket import inet_aton 14 | from bisect import bisect_left 15 | 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | IS_PY3 = sys.version_info[0] == 3 20 | IS_PYPY = platform.python_implementation() == 'PyPy' 21 | 22 | unpack_uint32_little = lambda b: struct.unpack("L", b)[0] 24 | unpack_uint16_big = lambda b: struct.unpack(">H", b)[0] 25 | if IS_PY3: 26 | unpack_uint8 = lambda b: b 27 | else: 28 | unpack_uint8 = lambda b: struct.unpack("B", b)[0] 29 | 30 | 31 | class IPIPDatabase(object): 32 | 33 | def __init__(self, filename): 34 | self.filename = os.path.abspath(filename) 35 | self._ranges = array.array('I') 36 | self._offsets = array.array('I') 37 | self._strings = '' 38 | self.load_db() 39 | 40 | def __repr__(self): 41 | return '' % (self.filename) 42 | 43 | def load_db(self): 44 | start = time.time() 45 | filename = self.filename 46 | with open(filename, 'rb') as f: 47 | buff = f.read() 48 | if filename.endswith('.dat'): 49 | index_size = 256 * 4 50 | text_length_size = 1 51 | elif filename.endswith('.datx'): 52 | index_size = 256 * 256 * 4 53 | text_length_size = 2 54 | else: 55 | raise Exception('unknown database format') 56 | self._load_db(buff, index_size, text_length_size) 57 | logger.debug('loaded ipip data file: %s, in %.2fs, ranges: %s', 58 | filename, time.time()-start, len(self._ranges)) 59 | if not IS_PYPY: 60 | logger.debug('mem usage: idx %.2fM, offsets %.2fM, strings %.2fM', 61 | sys.getsizeof(self._ranges)/1048576.0, 62 | sys.getsizeof(self._offsets)/1048576.0, 63 | sys.getsizeof(self._strings)/1048576.0, 64 | ) 65 | 66 | def _load_db(self, buff, index_size, text_length_size): 67 | ns_buff = [] 68 | ns_offset = 0 69 | offset_old_to_new = {} 70 | text_start = unpack_uint32_big(buff[:4]) - index_size 71 | offset = index_size + 4 72 | while offset < text_start: 73 | ip_range = unpack_uint32_big(buff[offset:offset+4]) 74 | offset += 4 75 | self._ranges.append(ip_range) 76 | text_offset = unpack_uint32_little(buff[offset:offset+3] + b'\0') + text_start 77 | offset += 3 78 | if text_length_size == 1: 79 | text_length = unpack_uint8(buff[offset]) 80 | elif text_length_size == 2: 81 | text_length = unpack_uint16_big(buff[offset:offset+2]) 82 | offset += text_length_size 83 | if text_offset not in offset_old_to_new: 84 | s = buff[text_offset:text_offset+text_length].decode('utf-8') 85 | offset_old_to_new[text_offset] = (ns_offset, ns_offset+len(s)) 86 | ns_buff.append(s) 87 | ns_offset += len(s) 88 | self._offsets.extend(offset_old_to_new[text_offset]) 89 | self._strings = ''.join(ns_buff) 90 | 91 | def lookup(self, ip): 92 | n = unpack(">L", inet_aton(ip))[0] 93 | i = bisect_left(self._ranges, n) 94 | return self._strings[self._offsets[2*i]:self._offsets[2*i+1]] 95 | 96 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [metadata] 5 | license_file = LICENSE 6 | description-file = README.rst 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | def read_readme(): 5 | with open('README.rst', 'rb') as f: 6 | return f.read().decode('utf-8') 7 | 8 | 9 | setup(name='pyipip', 10 | version='0.1.1', 11 | description="ipip.net IP address geolocation database Python library", 12 | long_description=read_readme(), 13 | url='https://github.com/georgexsh/pyipip', 14 | author='georgexsh', 15 | author_email='georgexsh@gmail.com', 16 | license='MIT', 17 | packages=['pyipip'], 18 | classifiers=[ 19 | 'Development Status :: 4 - Beta', 20 | 'Intended Audience :: Developers', 21 | 'License :: OSI Approved :: MIT License', 22 | 'Topic :: Software Development :: Libraries', 23 | 'Programming Language :: Python :: 2', 24 | 'Programming Language :: Python :: 3', 25 | 'Programming Language :: Python :: Implementation :: PyPy', 26 | ], 27 | ) 28 | -------------------------------------------------------------------------------- /tests/17monipdb.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/georgexsh/pyipip/48f3ab44f11ad2cd41a478f08bfdccedcecf2247/tests/17monipdb.dat -------------------------------------------------------------------------------- /tests/17monipdb.datx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/georgexsh/pyipip/48f3ab44f11ad2cd41a478f08bfdccedcecf2247/tests/17monipdb.datx -------------------------------------------------------------------------------- /tests/bench.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import gzip 4 | import time 5 | import logging 6 | import resource 7 | import socket 8 | import os 9 | 10 | 11 | CWD = os.path.dirname(__file__) 12 | DATA_FILE = os.path.join(CWD, '17monipdb.dat') 13 | 14 | # logging.basicConfig(level=logging.DEBUG) 15 | 16 | 17 | with gzip.open(os.path.join(CWD, 'ips.gz'), 'rb') as f: 18 | testdata = [l.strip().decode('latin1') for l in f] 19 | 20 | 21 | class BenchBase(object): 22 | pass 23 | 24 | 25 | class Bench_official(BenchBase): 26 | 27 | def setup(self): 28 | import ipip 29 | self.db = ipip.IP 30 | self.db.load(DATA_FILE) 31 | 32 | def run(self): 33 | db = self.db 34 | for ip in testdata: 35 | db.find(ip) 36 | 37 | 38 | class Bench_lxyu(BenchBase): 39 | 40 | def setup(self): 41 | import IP 42 | self.db = IP.IPv4Database(DATA_FILE) 43 | 44 | def run(self): 45 | db = self.db 46 | for ip in testdata: 47 | db.find(ip) 48 | 49 | 50 | class Bench_pyipip(BenchBase): 51 | 52 | def setup(self): 53 | from pyipip import IPIPDatabase 54 | self.db = IPIPDatabase(DATA_FILE) 55 | 56 | def run(self): 57 | db = self.db 58 | for ip in testdata: 59 | db.lookup(ip) 60 | 61 | 62 | def main(): 63 | N = 3 64 | for b in BenchBase.__subclasses__(): 65 | n = b.__name__.split('_')[1] 66 | c = b() 67 | c.setup() 68 | c.run() # warm-up 69 | s = time.time() 70 | for _ in range(N): 71 | c.run() 72 | e = time.time() - s 73 | print(n, '%.2f' % (N*len(testdata) / e)) 74 | 75 | 76 | if __name__ == '__main__': 77 | main() 78 | -------------------------------------------------------------------------------- /tests/ips.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/georgexsh/pyipip/48f3ab44f11ad2cd41a478f08bfdccedcecf2247/tests/ips.gz -------------------------------------------------------------------------------- /tests/reqs_bench.txt: -------------------------------------------------------------------------------- 1 | -e git+https://github.com/myiwen/ipip.git@c5aea92c14e0f900020ffb06aa8fcec23f87e55c#egg=ipip-master 2 | 17MonIP==0.2.7 3 | -------------------------------------------------------------------------------- /tests/test_pyipip.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from __future__ import unicode_literals 4 | import unittest 5 | import logging 6 | import os 7 | 8 | import pyipip 9 | 10 | 11 | # logging.basicConfig(level=logging.DEBUG) 12 | 13 | CWD = os.path.dirname(__file__) 14 | DAT_FILE = os.path.join(CWD, '17monipdb.dat') 15 | DATX_FILE = os.path.join(CWD, '17monipdb.datx') 16 | 17 | 18 | class TestPyIPIP(unittest.TestCase): 19 | 20 | def test_dat(self): 21 | self._check(DAT_FILE) 22 | 23 | def test_datx(self): 24 | self._check(DATX_FILE) 25 | 26 | def _check(self, db_file): 27 | db = pyipip.IPIPDatabase(db_file) 28 | self.assertEqual(db.lookup('0.0.0.0'), u"保留地址\t保留地址\t\t") 29 | self.assertEqual(db.lookup('127.0.0.1'), u"本机地址\t本机地址\t\t") 30 | self.assertEqual(db.lookup('198.18.0.0'), u"保留地址\t保留地址\t\t") 31 | self.assertIn('北京', db.lookup('202.112.80.106')) 32 | self.assertEqual(db.lookup('255.255.255.255').split()[0], u"IPIP.NET") 33 | 34 | 35 | if __name__ == '__main__': 36 | unittest.main() 37 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py{37,36,35,34,27,py,py3} 3 | 4 | [testenv] 5 | commands = python tests/test_pyipip.py 6 | --------------------------------------------------------------------------------