├── .gitignore ├── README.md ├── index.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | demo.js 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Binary Structure of ipdb Data Structure 2 | Based on the provided code, we can outline the binary representation of the ipdb data structure as follows: 3 | 4 | # Binary Structure Overview 5 | The binary representation of the ipdb data structure mainly consists of the following parts: 6 | 7 | **Header**: Contains the JSON string of metadata (meta). The header starts with a 4-byte unsigned integer (uint32) indicating the length of the JSON string, followed by the JSON string itself. 8 | 9 | **Node Chunk**: Contains the node data of the IP address binary tree structure. The length of the node chunk is equal to the number of nodes multiplied by 8 (each node occupies 8 bytes). Each node is represented by two 4-byte unsigned integers (uint32), indicating the offsets of the left and right child nodes. 10 | 11 | **Loopback Chunk**: A special node representing the root of the IP address range. The loopback chunk is 8 bytes long and contains two 4-byte unsigned integers (uint32), both indicating the length of the node chunk. 12 | 13 | **Data Chunk**: Contains data related to IP addresses (such as geolocation information). Each element in the data chunk is a Buffer, containing the length of the data (2-byte unsigned integer, uint16) and the actual data. 14 | 15 | # Example of Binary Structure 16 | 17 | Here is a simplified example of the binary representation of the ipdb data structure: 18 | 19 | ``` 20 | +-----------------+-----------------+-----------------+-----------------+ 21 | | Header Length | Header | Node Chunk | Loopback Chunk | 22 | | (4 bytes) | (JSON string) | (variable size) | (8 bytes) | 23 | +-----------------+-----------------+-----------------+-----------------+ 24 | | Data Chunk | Data Chunk | ... | ... | 25 | | (variable size) | (variable size) | | | 26 | +-----------------+-----------------+-----------------+-----------------+ 27 | ``` 28 | 29 | ## Node Chunk 30 | 31 | ``` 32 | +-----------------+-----------------+ 33 | | Left Child | Right Child | 34 | | Offset (4 bytes)| Offset (4 bytes)| 35 | +-----------------+-----------------+ 36 | ``` 37 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const net = require('net') 2 | 3 | class Packer { 4 | constructor(options = {}) { 5 | this.node = [[null, null]] 6 | this.data = [] 7 | this.data_hash = {} 8 | this.data_size = 0 9 | this.options = options 10 | this.meta = { 11 | build: Math.floor(new Date() / 1000), 12 | ip_version: options.ipv6 ? 0x03 : 0x01, 13 | languages: { 14 | 'CN': 0, 15 | } 16 | } 17 | } 18 | 19 | insert(cidr, data) { 20 | let [node, idx] = this._getNode(cidr) 21 | let offset = this._getOffset(data) 22 | this.node[node][idx] = -offset 23 | return this.node.length 24 | } 25 | 26 | output(fields) { 27 | // meta fields 28 | this.meta.fields = fields 29 | 30 | // meta node_count 31 | let node_count = this.node.length 32 | this.meta.node_count = node_count 33 | 34 | // fix null node 35 | let null_count = 0 36 | let null_offset = this._getOffset(new Array(fields.length).fill('')) 37 | for (let i = 0; i < this.node.length; i += 1) { 38 | for (let j = 0; j < 2; j += 1) { 39 | if (this.node[i][j] === null) { 40 | null_count += 1 41 | this.node[i][j] = -null_offset 42 | } 43 | } 44 | } 45 | 46 | // make data chunk 47 | this.data = Buffer.concat(this.data) 48 | 49 | // v4 node offset 50 | let v4node = this._getNode('0.0.0.0/1')[0] 51 | this.meta.v4node = v4node 52 | 53 | // node chunk 54 | let node_chunk = Buffer.alloc(this.node.length << 3) 55 | for (let i = 0; i < this.node.length; i += 1) { 56 | for (let j = 0; j < 2; j += 1) { 57 | if (this.node[i][j] <= 0) { 58 | this.node[i][j] = node_count - this.node[i][j] 59 | } 60 | let buf = this._toNodeBuffer(this.node[i][0], this.node[i][1]) 61 | buf.copy(node_chunk, i << 3) 62 | } 63 | } 64 | 65 | // loopback chunk 66 | let loopback_chunk = Buffer.alloc(1 << 3) 67 | let buf = this._toNodeBuffer(node_count, node_count) 68 | buf.copy(loopback_chunk, 0) 69 | 70 | // header chunk 71 | this.meta.total_size = node_chunk.length + loopback_chunk.length + this.data.length 72 | let header = Buffer.from(JSON.stringify(this.meta)) 73 | let header_len = header.length 74 | let header_chunk = Buffer.alloc(4) 75 | header_chunk.writeUInt32BE(header_len) 76 | header_chunk = Buffer.concat([header_chunk, header]) 77 | 78 | // final 79 | return Buffer.concat([header_chunk, node_chunk, loopback_chunk, this.data]) 80 | } 81 | 82 | _toNodeBuffer(left, right) { 83 | return Buffer.from([ 84 | (left >> 24) & 0xff, 85 | (left >> 16) & 0xff, 86 | (left >> 8) & 0xff, 87 | (left) & 0xff, 88 | (right >> 24) & 0xff, 89 | (right >> 16) & 0xff, 90 | (right >> 8) & 0xff, 91 | (right) & 0xff, 92 | ]) 93 | } 94 | 95 | _getNode(cidr) { 96 | let [ip, mask] = this._toBin(cidr) 97 | if (mask < 1) { 98 | mask = 1 99 | } 100 | let node = 0 101 | for (let i = 0; i < mask - 1; i += 1) { 102 | if (this.node[node][ip[i]] === null || this.node[node][ip[i]] < 0) { 103 | this.node.push([null, null]) 104 | this.node[node][ip[i]] = this.node.length - 1 105 | } 106 | node = this.node[node][ip[i]] 107 | } 108 | return [node, ip[mask - 1]] 109 | } 110 | 111 | _getOffset(data) { 112 | data = data.join('\t') 113 | if (!this.data_hash[data]) { 114 | let buf = Buffer.from(data) 115 | let len = Buffer.alloc(2) 116 | len.writeUInt16BE(buf.length) 117 | this.data_hash[data] = this.data_size + 8 118 | this.data.push(Buffer.concat([len, buf])) 119 | this.data_size += len.length + buf.length 120 | } 121 | return this.data_hash[data] 122 | } 123 | 124 | _toBin(cidr) { 125 | let [ip, mask] = cidr.split('/') 126 | mask = parseInt(mask, 10) 127 | 128 | let bin = net.isIP(ip) === 4 ? this._toBin4(ip) : this._toBin6(ip) 129 | if (bin.length === 32) { 130 | bin = [...(new Array(80).fill(0)), ...(new Array(16).fill(1)), ...bin] 131 | mask += 96 132 | } 133 | return [bin, mask] 134 | } 135 | 136 | _toBin4(ip) { 137 | const result = [] 138 | const items = ip.split('.') 139 | for (const item of items) { 140 | const num = parseInt(item, 10) 141 | for (let i = 7; i >= 0; i -= 1) { 142 | result.push((num >> i) & 1) 143 | } 144 | } 145 | return result 146 | } 147 | 148 | _toBin6(ip) { 149 | const result = [[], []] 150 | const parts = ip.split('::', 2) 151 | for (let index = 0; index < 2; index += 1) { 152 | if (parts[index]) { 153 | const items = parts[index].split(':') 154 | for (const item of items) { 155 | const num = parseInt(item, 16) 156 | for (let i = 15; i >= 0; i -= 1) { 157 | result[index].push((num >> i) & 1) 158 | } 159 | } 160 | } 161 | } 162 | const pad = 128 - result[0].length - result[1].length 163 | return [...result[0], ...(new Array(pad).fill(0)), ...result[1]] 164 | } 165 | } 166 | 167 | module.exports = Packer 168 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@ipdb/packer", 3 | "version": "1.0.4", 4 | "main": "index.js", 5 | "files": [ 6 | "index.js" 7 | ], 8 | "repository": { 9 | "type": "git", 10 | "url": "git+ssh://git@github.com/metowolf/ipdb-packer.git" 11 | }, 12 | "author": "metowolf ", 13 | "license": "MIT" 14 | } 15 | --------------------------------------------------------------------------------