├── .gitignore ├── LICENSE ├── README.md ├── datas ├── datas.txt ├── pyjce ├── __init__.py ├── bytebuffer.py ├── exception.py ├── stream.py └── struct.py ├── sample.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | #Visual Studio Code 132 | .vscode/ 133 | 134 | # Custom 135 | Jce/ 136 | jce_github/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 washingtown 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyJce 2 | 腾讯JCE协议解析Python版 3 | -------------------------------------------------------------------------------- /datas: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/washingtown/PyJce/8d9044329e3c792baf4a968486b176ae803088be/datas -------------------------------------------------------------------------------- /datas.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/washingtown/PyJce/8d9044329e3c792baf4a968486b176ae803088be/datas.txt -------------------------------------------------------------------------------- /pyjce/__init__.py: -------------------------------------------------------------------------------- 1 | from .bytebuffer import ByteBuffer 2 | from .struct import JceStruct 3 | from .stream import JceInputStream 4 | 5 | name = "PyJce" -------------------------------------------------------------------------------- /pyjce/bytebuffer.py: -------------------------------------------------------------------------------- 1 | import struct 2 | 3 | class ByteBuffer(object): 4 | _bytes = None 5 | _position = 0 6 | 7 | @property 8 | def bytes(self)->bytes: 9 | return self._bytes 10 | 11 | @property 12 | def position(self): 13 | return self._position 14 | @position.setter 15 | def position(self, value): 16 | if not isinstance(value, int): 17 | raise TypeError("'position' attribute must be a integer") 18 | elif value < 0: 19 | raise ValueError("'position' attribute must be a positive number") 20 | elif value > len(self._bytes): 21 | raise ValueError('') 22 | else: 23 | self._position = value 24 | 25 | def __init__(self, bs: bytes): 26 | if isinstance(bs, bytes): 27 | self._bytes = bs 28 | elif isinstance(bs, bytearray): 29 | self._bytes = bytes(bs) 30 | else: 31 | raise TypeError("'bs' argument must be bytes or bytesarray") 32 | 33 | def get(self): 34 | if self._position >= len(self._bytes): 35 | raise BufferError('reached end of bytes') 36 | b = self._bytes[self._position] 37 | self._position += 1 38 | return b 39 | 40 | def get_bytes(self, size): 41 | if size < 0: 42 | raise ValueError("'size' attribute must be a positive number") 43 | if self._position > len(self._bytes): 44 | raise BufferError('reached end of bytes') 45 | if self.position + size > len(self._bytes): 46 | raise BufferError('reached end of bytes') 47 | b = self._bytes[self.position:self.position + size] 48 | self.position = self.position + size 49 | return b 50 | 51 | def get_int2(self): 52 | b = self.get_bytes(2) 53 | return struct.unpack('>h', b)[0] 54 | 55 | def get_int4(self): 56 | b = self.get_bytes(4) 57 | return struct.unpack('>i', b)[0] 58 | 59 | def get_int8(self): 60 | b = self.get_bytes(4) 61 | return struct.unpack('>q', b)[0] 62 | 63 | def get_float(self): 64 | b = self.get_bytes(4) 65 | return struct.unpack('>f', b)[0] 66 | 67 | def get_double(self): 68 | b = self.get_bytes(8) 69 | return struct.unpack('>d', b)[0] 70 | 71 | 72 | def duplicate(self): 73 | bb = ByteBuffer(self._bytes) 74 | bb.position = self.position 75 | return bb 76 | 77 | def clear(self): 78 | self._position = 0 -------------------------------------------------------------------------------- /pyjce/exception.py: -------------------------------------------------------------------------------- 1 | # class ByteBufferExcetion(ValueError): 2 | # def __init__(self,ErrorInfo): 3 | # super().__init__(self) 4 | # self.errorinfo = ErrorInfo 5 | 6 | class JceDecodeException(ValueError): 7 | pass -------------------------------------------------------------------------------- /pyjce/stream.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple, List 2 | from pyjce.bytebuffer import ByteBuffer 3 | from pyjce.exception import JceDecodeException 4 | from pyjce.struct import JceStruct, JceStructStatics 5 | 6 | class HeadData(object): 7 | tag: int=0 8 | type: int = 0 9 | 10 | def __init__(self): 11 | self.tag = 0 12 | self.type = 0 13 | 14 | def __str__(self): 15 | return '{tag: %d, type: %d}' % (self.tag, self.type) 16 | 17 | def __repr__(self): 18 | return '{tag: %d, type: %d}' % (self.tag, self.type) 19 | 20 | 21 | def clear(self): 22 | self.tag = 0 23 | self.type = 0 24 | 25 | def read_head(byte_buffer: ByteBuffer) -> Tuple[HeadData, int]: 26 | head_data = HeadData() 27 | b = byte_buffer.get() 28 | head_data.type = b & 0x0F #低4位位类型 29 | head_data.tag = (b & 0xF0) >> 4 #高4位为tag, 30 | if head_data.tag != 15: #如果tag为15 则下一个字段为tag 31 | return (head_data, 1) 32 | else: 33 | head_data.tag = byte_buffer.get() & 0xFF 34 | return (head_data, 2) 35 | 36 | 37 | class JceInputStream(object): 38 | _bs:ByteBuffer = None 39 | encoding = 'utf-8' 40 | 41 | def __init__(self, bs, i=0): 42 | self.encoding = 'utf-8' 43 | if isinstance(bs, ByteBuffer): 44 | self._bs = bs 45 | elif isinstance(bs, (bytearray, bytes)): 46 | self._bs = ByteBuffer(bs) 47 | self._bs.position = i 48 | else: 49 | raise TypeError("'bs' argument must be bytes, bytesarray or ByteBuffer") 50 | 51 | def read_head(self): 52 | return read_head(self._bs) 53 | 54 | def peak_head(self): 55 | return read_head(self._bs.duplicate()) 56 | 57 | def skip(self, i: int): 58 | self._bs.position = self._bs.position + i 59 | 60 | def skip_to_struct_end(self): 61 | head_data, _ = self.read_head() 62 | self.skip_field(head_data.type) 63 | while (head_data.type != 11): 64 | head_data, _ = self.read_head() 65 | self.skip_field(head_data.type) 66 | 67 | def skip_field(self, field_type=None): 68 | if not field_type: 69 | head_data, _ = self.read_head() 70 | field_type = head_data.type 71 | i = 0 72 | read_value = None 73 | if field_type == 0: 74 | self.skip(1) 75 | elif field_type == 1: 76 | self.skip(2) 77 | elif field_type == 2: 78 | self.skip(4) 79 | elif field_type == 3: 80 | self.skip(8) 81 | elif field_type == 4: 82 | self.skip(4) 83 | elif field_type == 5: 84 | self.skip(8) 85 | elif field_type == 6: 86 | i = self._bs.get() 87 | if i < 0: 88 | i += 256 89 | self.skip(i) 90 | elif field_type == 7: 91 | i = self._bs.get_int4() 92 | self.skip(i) 93 | elif field_type == 8: 94 | read_value = self._read_int(0, 0, True) 95 | while i < read_value * 2: 96 | self.skip_field() 97 | i += 1 98 | elif field_type == 9: 99 | read_value = self._read_int(0, 0, True) 100 | while i < read_value: 101 | self.skip_field() 102 | i += i 103 | elif field_type == 10: 104 | self.skip_to_struct_end() 105 | elif field_type == 11 or field_type == 12: 106 | return 107 | elif field_type == 13: 108 | head_data,_=self.read_head() 109 | if head_data.type != 0: 110 | raise JceDecodeException("skipField with invalid type, type value: " + field_type + ", " + head_data.type) 111 | i = self._read_int(0, 0, True) 112 | self.skip(i) 113 | else: 114 | raise JceDecodeException("invalid type.") 115 | 116 | def skip_to_tag(self, tag: int)->bool: 117 | try: 118 | while True: 119 | head_data, length = self.peak_head() 120 | if tag > head_data.tag and head_data.type != 0x0B: 121 | self.skip(length) 122 | self.skip_field(head_data.type) 123 | else: 124 | break 125 | if head_data.type == 0X0B or tag != head_data.tag: 126 | return False 127 | return True 128 | except (JceDecodeException, BufferError): 129 | return False 130 | 131 | def re_init(self): 132 | self.encoding = 'utf-8' 133 | self._bs.clear() 134 | 135 | def get_tags(self) -> List[int]: 136 | position = self._bs.position 137 | # self.re_init() 138 | tags=[] 139 | while True: 140 | try: 141 | head_data, _ = self.read_head() 142 | tags.append(head_data.tag) 143 | self.skip_field(head_data.type) 144 | except: 145 | print('exception occured in position: %d, quit' % (self._bs.position)) 146 | break 147 | self._bs.position = position 148 | return tags 149 | 150 | 151 | def _read_bool(self, b: bool, tag: int, is_require: bool) -> bool: 152 | c = self._read_int(0, tag, is_require) 153 | return c != 0 154 | 155 | def _read_int(self, c: int, tag: int, is_require) -> int: 156 | if self.skip_to_tag(tag): 157 | head_data, _ = self.read_head() 158 | if head_data.type == 12: 159 | c = 0 160 | elif head_data.type == 0: 161 | c = self._bs.get() 162 | elif head_data.type == 1: 163 | c = self._bs.get_int2() 164 | elif head_data.type == 2: 165 | c = self._bs.get_int4() 166 | elif head_data.type == 3: 167 | c = self._bs.get_int8() 168 | else: 169 | raise JceDecodeException("type mismatch.") 170 | elif is_require: 171 | raise JceDecodeException("require field not exist.") 172 | return c 173 | 174 | def _read_float(self, n: float, tag: int, is_require: bool) -> float: 175 | if self.skip_to_tag(tag): 176 | head_data, _ = self.read_head() 177 | if head_data.type == 12: 178 | n = 0.0 179 | elif head_data.type == 4: 180 | n = self._bs.get_float() 181 | if head_data.type == 5: 182 | n = self._bs.get_double() 183 | else: 184 | raise JceDecodeException("type mismatch.") 185 | elif is_require: 186 | raise JceDecodeException("require field not exist.") 187 | return n 188 | 189 | def _read_string(self, s: str, tag: int, is_require: bool) -> str: 190 | if self.skip_to_tag(tag): 191 | head_data, _ = self.read_head() 192 | if head_data.type == 6: 193 | length = self._bs.get() 194 | if length < 0: 195 | length += 256 196 | ss = self._bs.get_bytes(length) 197 | try: 198 | s = ss.decode(self.encoding) 199 | except UnicodeDecodeError: 200 | s = ss.decode() 201 | elif head_data.type == 7: 202 | length = self._bs.get_int4() 203 | if length > JceStructStatics.JCE_MAX_STRING_LENGTH or length < 0: 204 | raise JceDecodeException("String too long: " + len) 205 | ss = self._bs.get_bytes(length) 206 | try: 207 | s = ss.decode(self.encoding) 208 | except UnicodeDecodeError: 209 | s = ss.decode() 210 | else: 211 | raise JceDecodeException("type mismatch.") 212 | elif is_require: 213 | raise JceDecodeException("require field not exist.") 214 | return s 215 | 216 | def _read_struct(self, o: JceStruct, tag: int, is_require: bool) -> JceStruct: 217 | ref = None 218 | if self.skip_to_tag(tag): 219 | ref = o 220 | head_data, _ = self.read_head() 221 | if head_data.type != 10: 222 | raise JceDecodeException("type mismatch.") 223 | ref.read_from(self) 224 | self.skip_to_struct_end() 225 | elif is_require: 226 | raise JceDecodeException("require field not exist.") 227 | return ref 228 | 229 | def _read_list(self, mt, tag: int, is_require: bool) -> list: 230 | if self.skip_to_tag(tag): 231 | head_data, _ = self.read_head() 232 | if head_data.type == 9: 233 | size = self._read_int(0, 0, True) 234 | if size < 0: 235 | raise JceDecodeException("size invalid: " + size) 236 | lr = [] 237 | for i in range(size): 238 | t = self.read_current(True) 239 | lr.append(t) 240 | return lr 241 | raise JceDecodeException("type mismatch.") 242 | elif is_require: 243 | raise JceDecodeException("require field not exist.") 244 | return None 245 | 246 | def _read_map(self, m: dict, tag: int, is_require: bool): 247 | mr = {} 248 | if self.skip_to_tag(tag): 249 | head_data, _ = self.read_head() 250 | if head_data.type == 8: 251 | size = self._read_int(0, 0, True) 252 | if size < 0: 253 | raise JceDecodeException("size invalid: " + size) 254 | for i in range(size): 255 | k = self.read_current(True) 256 | v = self.read_current(True) 257 | mr[k] = v 258 | else: 259 | raise JceDecodeException("type mismatch.") 260 | elif is_require: 261 | raise JceDecodeException("require field not exist.") 262 | return mr 263 | 264 | def _read_simple_list(self, l, tag: int, is_require: bool): 265 | lr = b'' 266 | if self.skip_to_tag(tag): 267 | head_data, _ = self.read_head() 268 | if head_data.type == 13: 269 | hh, _ = self.read_head() 270 | if hh.type != 0: 271 | raise JceDecodeException("type mismatch, tag: " + tag + ", type: " + head_data.type + ", " + hh.type) 272 | size = self._read_int(0, 0, True) 273 | if size < 0: 274 | raise JceDecodeException("invalid size, tag: " + tag + ", type: " + head_data.type + ", " + hh.type + ", size: " + size) 275 | lr=self._bs.get_bytes(size) 276 | else: 277 | raise JceDecodeException("type mismatch.") 278 | elif is_require: 279 | raise JceDecodeException("require field not exist.") 280 | return lr 281 | 282 | def read(self, o, tag: int, is_require: bool): 283 | if isinstance(o, bool): 284 | return self._read_bool(o, tag, is_require) 285 | if isinstance(o, int): 286 | return self._read_int(o, tag, is_require) 287 | if isinstance(o, float): 288 | return self._read_float(o, tag, is_require) 289 | if isinstance(o, str): 290 | return self._read_string(o, tag, is_require) 291 | if isinstance(o, list): 292 | return self._read_list(o, tag, True) 293 | if isinstance(o, dict): 294 | return self._read_map(o, tag, True) 295 | raise JceDecodeException("read object error: unsupport type.") 296 | 297 | def read_current(self, is_require: bool): 298 | head_data, _ = self.peak_head() 299 | if head_data.type in (0, 1, 2, 3): 300 | return self._read_int(0, head_data.tag, is_require) 301 | elif head_data.type in (4, 5): 302 | return self._read_float(0.0, head_data.tag, is_require) 303 | elif head_data.type in (6, 7): 304 | return self._read_string('', head_data.tag, is_require) 305 | elif head_data.type == 8: 306 | return self._read_map({}, head_data.tag, is_require) 307 | elif head_data.type == 9: 308 | return self._read_list([], head_data.tag, is_require) 309 | elif head_data.type == 10: 310 | return self._read_struct(JceStruct(), head_data.tag, is_require) 311 | elif head_data.type == 11: 312 | self.read_head() 313 | return None 314 | elif head_data.type == 12: 315 | self.read_head() 316 | return 0 317 | elif head_data.type == 13: 318 | return self._read_simple_list(b'', head_data.tag, is_require) 319 | else: 320 | raise JceDecodeException("read object error: unsupport type.") -------------------------------------------------------------------------------- /pyjce/struct.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | class JceStructStatics: 4 | BYTE = 0 5 | SHORT = 1 6 | INT = 2 7 | LONG = 3 8 | FLOAT = 4 9 | DOUBLE = 5 10 | STRING1 = 6 11 | STRING4 = 7 12 | MAP = 8 13 | LIST = 9 14 | STRUCT_BEGIN = 10 15 | STRUCT_END = 11 16 | ZERO_TAG = 12 17 | SIMPLE_LIST = 13 18 | JCE_MAX_STRING_LENGTH = 104857600 19 | 20 | class JceStruct(object): 21 | 22 | def __init__(self): 23 | self.data = {} 24 | 25 | def __getitem__(self, key): 26 | return self.data[key] 27 | 28 | def __setitem__(self, key, value): 29 | self.data[key] = value 30 | 31 | def read_from(self, stream): 32 | self.data.clear() 33 | head_data, _ = stream.peak_head() 34 | while head_data.type != 11 and stream._bs.position < len(stream._bs.bytes): 35 | tag = head_data.tag 36 | value = stream.read_current(True) 37 | if isinstance(value, bytes) and len(value) > 0: 38 | try: 39 | s = JceStruct() 40 | from pyjce.stream import JceInputStream 41 | s.read_from(JceInputStream(value)) 42 | self.data[tag] = s 43 | except: 44 | self.data[tag] = value 45 | else: 46 | self.data[tag] = value 47 | if stream._bs.position >= len(stream._bs.bytes): 48 | break 49 | head_data, _ = stream.peak_head() 50 | 51 | def to_json(self): 52 | return json.dumps(self,cls=JceStructEnconding,ensure_ascii=False) 53 | 54 | class JceStructEnconding(json.JSONEncoder): 55 | def default(self, o): # pylint: disable=E0202 56 | if isinstance(o, JceStruct): 57 | if len(o.data) == 1: 58 | return list(o.data.values())[0] 59 | else: 60 | return o.data 61 | if isinstance(o, (bytes, bytearray)): 62 | return str(o) -------------------------------------------------------------------------------- /sample.py: -------------------------------------------------------------------------------- 1 | from pyjce import JceInputStream,JceStruct 2 | 3 | f = open('datas', 'rb') 4 | stream = JceInputStream(f.read()) 5 | s=JceStruct() 6 | s.read_from(stream) 7 | ff = open('datas.txt', 'w') 8 | print(s.to_json()) 9 | ff.write(s.to_json()) 10 | f.close() 11 | ff.close() 12 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r", encoding='utf-8') as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="PyJce", 8 | version="0.0.1", 9 | author="washingtown", 10 | author_email="sharp-shoot2@126.com", 11 | description="Tecent JCE parser for Python", 12 | long_description=long_description, 13 | long_description_content_type="text/markdown", 14 | url="https://github.com/washingtown/PyJce", 15 | packages=setuptools.find_packages(), 16 | classifiers=[ 17 | "Programming Language :: Python :: 3", 18 | "License :: OSI Approved :: MIT License", 19 | "Operating System :: OS Independent", 20 | ], 21 | ) --------------------------------------------------------------------------------