├── .gitignore
├── requirements.txt
├── pysproto
    ├── __init__.py
    ├── compat.h
    ├── msvcint.h
    ├── sproto.h
    ├── sproto.py
    ├── sprotodump.py
    ├── sprotoparser.py
    ├── core.pyx
    └── sproto.c
├── README.md
├── setup.py
└── LICENSE


/.gitignore:
--------------------------------------------------------------------------------
1 | build
2 | dist
3 | MANIFEST
4 | 
5 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | setuptools>=20
2 | Cython>=0.24
3 | 


--------------------------------------------------------------------------------
/pysproto/__init__.py:
--------------------------------------------------------------------------------
1 | from .sproto import SprotoRpc
2 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # pysproto
 2 | python binding for cloudwu‘s sproto with rpc support
 3 | 
 4 | make it one python egg<br>
 5 | installation:
 6 | 
 7 | 0. sudo pip install -r requirements.txt
 8 | 1. python setup.py build
 9 | 2. python setup.py install
10 | 
11 | or pip
12 | 
13 |  sudo pip install 'setuptools>=20' 'cython>=0.24' git+git://github.com/bttscut/pysproto.git#egg=pysproto 
14 | 
15 | - sproto.h, sproto.c, msvcint.h copied from https://github.com/cloudwu/sproto
16 | - sprotoparser.py copied from https://github.com/tzngit/python_sproto_parser
17 | 
18 | contact me if any problem.
19 | 


--------------------------------------------------------------------------------
/pysproto/compat.h:
--------------------------------------------------------------------------------
 1 | #include "Python.h"
 2 | 
 3 | typedef void (*capsule_dest)(PyObject *);
 4 | typedef void (*cobj_dest)(void *);
 5 | 
 6 | #if PY_MAJOR_VERSION != 2
 7 |     #error "need python 2.*"
 8 | #elif PY_MINOR_VERSION > 6
 9 |     #define CAP_NEW PyCapsule_New
10 |     #define DEST_FUNC_TYPE capsule_dest
11 |     #define CAP_GET_POINTER PyCapsule_GetPointer
12 | #else
13 |     #define CAP_NEW(a,b,c) PyCObject_FromVoidPtr(a,c)
14 |     #define DEST_FUNC_TYPE cobj_dest
15 |     #define CAP_GET_POINTER(a,b) PyCObject_AsVoidPtr(a)
16 | #endif
17 | 
18 | PyObject* make_capsule(void *p, const char *name, capsule_dest dest) {
19 |     return CAP_NEW(p, name, (DEST_FUNC_TYPE)dest);
20 | }
21 | void* get_pointer(PyObject *cap, const char *name) {
22 |     return CAP_GET_POINTER(cap, name);
23 | }
24 | 


--------------------------------------------------------------------------------
/pysproto/msvcint.h:
--------------------------------------------------------------------------------
 1 | #ifndef msvc_int_h
 2 | #define msvc_int_h
 3 | 
 4 | #ifdef _MSC_VER
 5 | # define inline __inline
 6 | # ifndef _MSC_STDINT_H_
 7 | #  if (_MSC_VER < 1300)
 8 | typedef signed char       int8_t;
 9 | typedef signed short      int16_t;
10 | typedef signed int        int32_t;
11 | typedef unsigned char     uint8_t;
12 | typedef unsigned short    uint16_t;
13 | typedef unsigned int      uint32_t;
14 | #  else
15 | typedef signed __int8     int8_t;
16 | typedef signed __int16    int16_t;
17 | typedef signed __int32    int32_t;
18 | typedef unsigned __int8   uint8_t;
19 | typedef unsigned __int16  uint16_t;
20 | typedef unsigned __int32  uint32_t;
21 | #  endif
22 | typedef signed __int64       int64_t;
23 | typedef unsigned __int64     uint64_t;
24 | # endif
25 | 
26 | #else
27 | 
28 | #include <stdint.h>
29 | 
30 | #endif
31 | 
32 | #endif
33 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | from setuptools import setup, Extension
 2 | 
 3 | # core = Extension('pysproto.core',
 4 |         # sources = ["pysproto/python_sproto.c", "pysproto/sproto.c"],
 5 |         # extra_compile_args = ["-g3", "-O0"],
 6 |         # )
 7 | from Cython.Build import cythonize
 8 | ext = Extension("pysproto.core",
 9 |         sources = ["pysproto/core.pyx", "pysproto/sproto.c"],
10 |         # extra_compile_args = ["-g3", "-O0"],
11 |         )
12 | core = cythonize(ext)
13 | 
14 | setup(
15 |         name = "pysproto",
16 |         version = '0.6',
17 |         packages = ["pysproto"],
18 |         description = "python binding for cloudwu's sproto",
19 |         author = "bttscut",
20 |         license = "MIT",
21 |         url="http://github.com/bttscut/pysproto",
22 |         keywords=["sproto", "python"],
23 |         # py_modules = ["sproto.py"],
24 |         ext_modules = core,
25 |         )
26 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2014 codingnow.com
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of
 6 | this software and associated documentation files (the "Software"), to deal in
 7 | the Software without restriction, including without limitation the rights to
 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 9 | the Software, and to permit persons to whom the Software is furnished to do so,
10 | subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | Copyright (c) 2016 bttsuct
16 | 
17 | Permission is hereby granted, free of charge, to any person obtaining a copy
18 | of this software and associated documentation files (the "Software"), to deal
19 | in the Software without restriction, including without limitation the rights
20 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
21 | copies of the Software, and to permit persons to whom the Software is
22 | furnished to do so, subject to the following conditions:
23 | 
24 | The above copyright notice and this permission notice shall be included in all
25 | copies or substantial portions of the Software.
26 | 
27 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
28 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
29 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
30 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
31 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
32 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 | SOFTWARE.
34 | 


--------------------------------------------------------------------------------
/pysproto/sproto.h:
--------------------------------------------------------------------------------
 1 | #ifndef sproto_h
 2 | #define sproto_h
 3 | 
 4 | #include <stddef.h>
 5 | 
 6 | struct sproto;
 7 | struct sproto_type;
 8 | 
 9 | #define SPROTO_REQUEST 0
10 | #define SPROTO_RESPONSE 1
11 | 
12 | // type (sproto_arg.type)
13 | #define SPROTO_TINTEGER 0
14 | #define SPROTO_TBOOLEAN 1
15 | #define SPROTO_TSTRING 2
16 | #define SPROTO_TSTRUCT 3
17 | 
18 | // sub type of string (sproto_arg.extra)
19 | #define SPROTO_TSTRING_STRING 0
20 | #define SPROTO_TSTRING_BINARY 1
21 | 
22 | #define SPROTO_CB_ERROR -1
23 | #define SPROTO_CB_NIL -2
24 | #define SPROTO_CB_NOARRAY -3
25 | 
26 | struct sproto * sproto_create(const void * proto, size_t sz);
27 | void sproto_release(struct sproto *);
28 | 
29 | int sproto_prototag(const struct sproto *, const char * name);
30 | const char * sproto_protoname(const struct sproto *, int proto);
31 | // SPROTO_REQUEST(0) : request, SPROTO_RESPONSE(1): response
32 | struct sproto_type * sproto_protoquery(const struct sproto *, int proto, int what);
33 | int sproto_protoresponse(const struct sproto *, int proto);
34 | 
35 | struct sproto_type * sproto_type(const struct sproto *, const char * type_name);
36 | 
37 | int sproto_pack(const void * src, int srcsz, void * buffer, int bufsz);
38 | int sproto_unpack(const void * src, int srcsz, void * buffer, int bufsz);
39 | 
40 | struct sproto_arg {
41 | 	void *ud;
42 | 	const char *tagname;
43 | 	int tagid;
44 | 	int type;
45 | 	struct sproto_type *subtype;
46 | 	void *value;
47 | 	int length;
48 | 	int index;	// array base 1
49 | 	int mainindex;	// for map
50 | 	int extra; // SPROTO_TINTEGER: decimal ; SPROTO_TSTRING 0:utf8 string 1:binary
51 | };
52 | 
53 | typedef int (*sproto_callback)(const struct sproto_arg *args);
54 | 
55 | int sproto_decode(const struct sproto_type *, const void * data, int size, sproto_callback cb, void *ud);
56 | int sproto_encode(const struct sproto_type *, void * buffer, int size, sproto_callback cb, void *ud);
57 | 
58 | // for debug use
59 | void sproto_dump(struct sproto *);
60 | const char * sproto_name(struct sproto_type *);
61 | 
62 | #endif
63 | 


--------------------------------------------------------------------------------
/pysproto/sproto.py:
--------------------------------------------------------------------------------
 1 | from . import core
 2 | 
 3 | __all__ = ["SprotoRpc"]
 4 | 
 5 | class SprotoObj(object):
 6 |     def __init__(self, chunk):
 7 |         self.sp = core.newproto(chunk)
 8 |         self.st = {}
 9 |         self.proto = {}
10 | 
11 |     def query_type(self, tagname):
12 |         if not tagname in self.st:
13 |             self.st[tagname] = core.query_type(self.sp, tagname)
14 |         return self.st[tagname]
15 | 
16 |     def protocol(self, protoname):
17 |         # print "protocol", protoname
18 |         if not protoname in self.proto:
19 |             self.proto[protoname] = core.protocol(self.sp, protoname)
20 |         return self.proto[protoname]
21 | 
22 |     def encode(self, st, data):
23 |         # print "encode", data
24 |         return core.encode(st, data)
25 | 
26 |     def decode(self, st, chunk):
27 |         # print "decode"
28 |         return core.decode(st, chunk)
29 | 
30 |     def pack(self, chunk):
31 |         return core.pack(chunk)
32 | 
33 |     def unpack(self, chunk):
34 |         return core.unpack(chunk)
35 | 
36 | class SprotoRpc(object):
37 |     def __init__(self, protobin, packagename):
38 |         self._sp = SprotoObj(protobin)
39 |         self._package = self._sp.query_type(packagename)
40 |         self._session = {}
41 | 
42 |     def _gen_response_f(self, resp, session):
43 |         def f(args, ud):
44 |             header = self._sp.encode(self._package, {"type":0, "session":session, "ud":ud})
45 |             content = self._sp.encode(resp, args)
46 |             return self._sp.pack(header+content)
47 |         return f
48 | 
49 |     def dispatch(self, data):
50 |         sp = self._sp
51 |         data = sp.unpack(data)
52 |         header,size = sp.decode(self._package, data)
53 |         content = data[size:]
54 |         if header.get("type", 0):
55 |             # request
56 |             protoname, req, resp = sp.protocol(header["type"])
57 |             result,_ = sp.decode(req, content) if req else (None, 0)
58 |             ret = {"type":"REQUEST", "proto": protoname, "msg":result, "session":None}
59 |             session = header.get("session")
60 |             if session:
61 |                 ret["response"] = self._gen_response_f(resp, session)
62 |         else:
63 |             # response
64 |             session = header["session"]
65 |             if not session in self._session:
66 |                 raise ValueError("unknown session", session)
67 |             response = self._session[session]
68 |             del self._session[session]
69 |             ret = {"type":"RESPONSE", "session":session, "msg":None}
70 |             ret["msg"], _ = sp.decode(response, content)
71 |         ret["header"] = header
72 |         return ret
73 |             
74 |     def request(self, protoname, args = None, session = 0, ud = None):
75 |         sp = self._sp
76 |         tag, req, resp = sp.protocol(protoname)
77 |         header = sp.encode(self._package, {"type":tag, "session":session, "ud":ud})
78 |         if session and not resp:
79 |             raise ValueError("proto no response")
80 |         if session:
81 |             self._session[session] = resp
82 |         content = sp.encode(req, args) if req else ""
83 |         return sp.pack(header + content)
84 | 


--------------------------------------------------------------------------------
/pysproto/sprotodump.py:
--------------------------------------------------------------------------------
  1 | #coding:utf-8
  2 | import sprotoparser
  3 | import struct 
  4 | import sys, argparse, os, codecs
  5 | 
  6 | def packbytes(s):
  7 |     s = str(s)
  8 |     return struct.pack("<I%ds" % len(s), len(s), s) 
  9 | 
 10 | def packvalue(v):
 11 |     v = (v + 1) * 2
 12 |     return struct.pack("<H", v)
 13 | 
 14 | def packfield(f):
 15 |     strtbl = []
 16 |     if f["array"]:
 17 |         if f["key"]: #if has no "key" already set to f["key"] = None
 18 |             strtbl.append("\6\0")
 19 |         else:
 20 |             strtbl.append("\5\0")
 21 |     else:
 22 |         strtbl.append("\4\0")
 23 |     strtbl.append("\0\0")
 24 |     if f["builtin"] != None:
 25 |         strtbl.append(packvalue(f["builtin"]))
 26 |         strtbl.append("\1\0")
 27 |         strtbl.append(packvalue(f["tag"]))
 28 |     else:
 29 |         strtbl.append("\1\0")
 30 |         strtbl.append(packvalue(f["type"]))
 31 |         strtbl.append(packvalue(f["tag"]))
 32 |     if f["array"]:
 33 |         strtbl.append(packvalue(1))
 34 |     if f["key"]:
 35 |         strtbl.append(packvalue(f["key"]))
 36 |     strtbl.append(packbytes(f["name"]))
 37 |     return packbytes("".join(strtbl))
 38 | 
 39 | def packtype(name, t, alltypes):
 40 |     fields = []
 41 |     tmp = {}
 42 |     for _, f in enumerate(t):
 43 |         tmp["array"] = f["array"]
 44 |         tmp["name"] = f["name"]
 45 |         tmp["tag"] = f["tag"]
 46 |         tname = f["typename"]
 47 |         tmp["builtin"] = sprotoparser.builtin_types[tname] if tname in sprotoparser.builtin_types else None
 48 |         subtype = None
 49 | 
 50 |         if tmp["builtin"] == None:
 51 |             assert alltypes[tname], "type %s not exists" % tname
 52 |             subtype = alltypes[tname]
 53 |             tmp["type"] = subtype["id"]
 54 |         else:
 55 |             tmp["type"] = None
 56 |         if "key" in f:
 57 |             tmp["key"] = subtype["fields"][f["key"]["name"]]
 58 |             assert tmp["key"], "Invalid map index %d" % f["key"]["name"]
 59 |         else:
 60 |             tmp["key"] = None
 61 |         fields.append(packfield(tmp))
 62 |     data = None
 63 |     if not fields:
 64 |         data = ["\1\0", "\0\0", packbytes(name)]
 65 |     else:
 66 |         data = ["\2\0", "\0\0", "\0\0", packbytes(name), packbytes("".join(fields))]
 67 |     return packbytes("".join(data))
 68 | 
 69 | def packproto(name, p, alltypes):
 70 |     if "request" in p:
 71 |         request = alltypes[p["request"]]
 72 |         assert request != None, "Protocol %s request types not found" % (name, p["request"])
 73 |         request = request["id"]
 74 | 
 75 |     tmp = ["\4\0", "\0\0", packvalue(p["tag"])]
 76 |     if "request" not in p and "response" not in p:
 77 |         tmp[0] = "\2\0"
 78 |     else:
 79 |         if "request" in p:
 80 |             tmp.append(packvalue(alltypes[p["request"]]["id"]))
 81 |         else:
 82 |             tmp.append("\1\0")
 83 |         if "response" in p:
 84 |             tmp.append(packvalue(alltypes[p["response"]]["id"]))
 85 |         else:
 86 |             tmp[0] = "\3\0"
 87 |     tmp.append(packbytes(name))
 88 |     return packbytes("".join(tmp))
 89 | 
 90 | def packgroup(t, p):
 91 |     if not t:
 92 |         assert p
 93 |         return "\0\0"
 94 |     tp = None
 95 |     alltypes = {}
 96 |     alltype_names = []
 97 |     for name in t:
 98 |         alltype_names.append(name)
 99 |     alltype_names.sort()
100 |     for idx, name in enumerate(alltype_names):
101 |         fields = {}
102 |         for _, type_fields in enumerate(t[name]):
103 |             if type_fields["typename"] in sprotoparser.builtin_types:
104 |                 fields[type_fields["name"]] = type_fields["tag"]
105 |         alltypes[name] = { "id":idx, "fields":fields }
106 | 
107 |     tt = []
108 |     for _, name in enumerate(alltype_names):
109 |         tt.append(packtype(name, t[name], alltypes))
110 | 
111 |     tt = packbytes("".join(tt))
112 |     if p:
113 |         tmp = []
114 |         for name, tbl in p.iteritems():
115 |             tmp.append(tbl)
116 |             tbl["name"] = name
117 |         tmp = sorted(tmp, key=lambda k: k["tag"])
118 |         tp = []
119 |         for _, tbl in enumerate(tmp):
120 |             tp.append(packproto(tbl["name"], tbl, alltypes))
121 |         tp = packbytes("".join(tp))
122 |     result = None
123 |     if tp == None:
124 |         result = ["\1\0","\0\0",tt]
125 |     else:
126 |         result = ["\2\0","\0\0","\0\0",tt,tp]
127 |     return "".join(result)
128 | 
129 | def encodeall(r):
130 |     return packgroup(r["type"], r["protocol"])
131 | 
132 | def parse_ast(ast):
133 |     return encodeall(ast)
134 | 
135 | def dump(build, outfile):
136 |     data = parse_ast(build)
137 |     f = open(outfile, "wb")
138 |     f.write(data)
139 |     f.close()
140 | 
141 | if __name__ == "__main__":
142 |     parser = argparse.ArgumentParser()
143 |     parser.add_argument("-d", "--directory", dest="src_dir", help="sproto source files")
144 |     parser.add_argument("-f", "--file", dest="src_file", help="sproto single file")
145 |     parser.add_argument("-o", "--out", dest="outfile", default="sproto.spb", help="specific dump binary file")
146 |     parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="show more info")
147 |     args = parser.parse_args()
148 | 
149 |     build = None
150 |     if args.src_file:
151 |         text = codecs.open(args.src_file, encoding="utf-8").read()
152 |         build = sprotoparser.parse(text, os.path.basename(args.src_file))
153 |     else:
154 |         sproto_list = []
155 |         for f in os.listdir(args.src_dir):
156 |             file_path = os.path.join(args.src_dir, f)
157 |             if os.path.isfile(file_path) and f.endswith(".sproto"):
158 |                 text = codecs.open(file_path, encoding="utf-8").read()
159 |                 sproto_list.append((text, f))
160 | 
161 |         build = sprotoparser.parse_list(sproto_list)
162 | 
163 |     if args.verbose == True:
164 |         import json
165 |         print(json.dumps(build, indent=4))
166 |     dump(build, args.outfile)
167 | 


--------------------------------------------------------------------------------
/pysproto/sprotoparser.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | from __future__ import unicode_literals, print_function
  3 | from pypeg2 import * 
  4 | 
  5 | pypeg2_parse = parse #rename it,for avoiding name conflict
  6 | tag = re.compile(r"\d+")
  7 | nomeaning = blank, maybe_some(comment_sh), blank
  8 | fullname = re.compile(r"[\w+\.]*\w+")
  9 | 
 10 | class MainKey(str):
 11 |     grammar = "(", word, ")"
 12 | 
 13 | class TypeName(object):
 14 |     grammar = flag("is_arr", "*"), attr("fullname", fullname)
 15 | 
 16 | class Filed(List):
 17 |     grammar = attr("filed", word), attr("tag", tag), ":", attr("typename", TypeName),\
 18 |             optional(MainKey), nomeaning, endl
 19 | 
 20 | class Struct(List):pass
 21 | class Type(List):pass
 22 | 
 23 | Struct.grammar = "{", nomeaning, attr("fileds", maybe_some([Filed, Type])), "}"
 24 | Type.grammar = nomeaning, ".", name(), attr("struct", Struct), nomeaning
 25 | 
 26 | class Sub_pro_type(Keyword):
 27 |     grammar = Enum(K("request"), K("response"))
 28 | 
 29 | class Subprotocol(List):
 30 |     grammar = attr("subpro_type", Sub_pro_type), attr("pro_filed", [TypeName, Struct]), nomeaning
 31 | 
 32 | class Protocol(List):
 33 |     grammar = nomeaning, attr("name", word), attr("tag",tag), "{", nomeaning, attr("fileds", maybe_some(Subprotocol)), "}", nomeaning
 34 | 
 35 | class Sproto(List):
 36 |     grammar = attr("items", maybe_some([Type, Protocol]))
 37 | #====================================================================
 38 | 
 39 | builtin_types = {"integer":0, "boolean":1, "string":2}
 40 | 
 41 | import re as rawre
 42 | def checktype(types, ptype, t):
 43 |     if t in builtin_types:
 44 |         return t
 45 | 
 46 |     fullname = "%s.%s" % (ptype, t)
 47 |     if fullname in types:
 48 |         return fullname
 49 |     else:
 50 |         sobj = rawre.search("(.+)\..+$", ptype)
 51 |         if sobj:
 52 |             return checktype(types, sobj.group(1), t)
 53 |         elif t in types:
 54 |             return t
 55 | 
 56 | 
 57 | def flattypename(r):
 58 |     for typename, t in r["type"].iteritems():
 59 |         for _, f in enumerate(t):
 60 |             ftype = f["typename"]
 61 |             fullname = checktype(r["type"], typename, ftype)
 62 |             assert fullname != None, "Undefined type %s in type %s" % (ftype, typename)
 63 |             f["typename"] = fullname
 64 |             if "array" in f and "key" in f:
 65 |                 pass
 66 | 
 67 | 
 68 | class Convert:
 69 |     group = {}
 70 |     type_dict = {}
 71 |     protocol_dict = {}
 72 |     protocol_tags = {} #just for easiliy check
 73 | 
 74 |     @staticmethod
 75 |     def parse(text, name):
 76 |         Convert.group = {}
 77 |         Convert.type_dict = {}
 78 |         Convert.protocol_dict = {}
 79 |         Convert.protocol_tags = {}
 80 | 
 81 |         obj = pypeg2_parse(text, Sproto)
 82 |         for i in obj.items:
 83 |             if hasattr(i,"tag"):
 84 |                 Convert.convert_protocol(i)
 85 |             else:
 86 |                 Convert.convert_type(i)
 87 | 
 88 |         Convert.group["type"] = Convert.type_dict
 89 |         Convert.group["protocol"] = Convert.protocol_dict
 90 | 
 91 |         #import json
 92 |         #print(json.dumps(Convert.group, indent=4))
 93 |         return Convert.group
 94 | 
 95 |     @staticmethod
 96 |     def convert_type(obj, parent_name = ""):
 97 |         if parent_name != "":
 98 |             obj.name = parent_name + "." + obj.name
 99 |         type_name = obj.name
100 |         if type_name in Convert.type_dict.keys():
101 |             print("Error:redifine %s\n" % (type_name))
102 |             return False
103 |         Convert.type_dict[type_name] = Convert.convert_struct(obj.struct, type_name)
104 | 
105 |     @staticmethod
106 |     def convert_struct(obj, name = ""):
107 |         struct = []
108 |         for filed in obj.fileds:
109 |             if type(filed) == Filed:
110 |                 filed_typename = filed.typename.fullname
111 |                 filed_type = Convert.get_typename(filed_typename)
112 |                 filed_info = {}
113 |                 filed_info["name"] = filed.filed
114 |                 filed_info["tag"] = int(filed.tag)
115 |                 filed_info["array"] = filed.typename.is_arr
116 |                 filed_info["typename"] = filed_typename
117 |                 filed_info["type"] = filed_type
118 |                 struct.append(filed_info)
119 |             elif type(filed) == Type:
120 |                 Convert.convert_type(filed, name)
121 |         return struct
122 | 
123 |     @staticmethod
124 |     def convert_protocol(obj):
125 |         if obj.name in Convert.protocol_dict.keys():
126 |             print("Error:redifine protocol %s \n" % (obj.name))
127 |             return
128 |         if obj.tag in Convert.protocol_tags.keys():
129 |             print("Error:redifine protocol tags %d \n" % (obj.tag))
130 |             return
131 |         protocol = {}
132 |         protocol["tag"] = int(obj.tag)
133 |         protocol["name"] = obj.name
134 |         for fi in obj.fileds:
135 |             if type(fi.pro_filed) == TypeName:
136 |                 assert fi.pro_filed.is_arr == False, "syntax error at %s.%s" % (obj.name, fi.subpro_type)
137 |                 newtype_name = str(''.join(fi.pro_filed.fullname))
138 |                 protocol[fi.subpro_type] = newtype_name
139 |             elif type(fi.pro_filed) == Struct:
140 |                 newtype_name = obj.name + "." + fi.subpro_type
141 |                 Convert.type_dict[newtype_name] = Convert.convert_struct(fi.pro_filed, newtype_name)
142 |                 protocol[fi.subpro_type] = newtype_name
143 |            
144 |         Convert.protocol_dict[obj.name] = protocol
145 |         Convert.protocol_tags[obj.tag] = True
146 | 
147 |     @staticmethod
148 |     def get_typename(name):
149 |         if name in builtin_types:
150 |             return "builtin"
151 |         else:
152 |             return "UserDefine"
153 | 
154 | # ===============================================================
155 | # export functions
156 | # ===============================================================
157 | __all__ = ["parse", "parse_list", "builtin_types"]
158 | 
159 | def parse(text, name, check=True):
160 |     build = Convert.parse(text, name)
161 |     if check:
162 |         flattypename(build)
163 |     return build
164 | 
165 | def parse_list(sproto_list):
166 |     build = {"protocol":{}, "type":{}}
167 |     for v in sproto_list:
168 |         ast = Convert.parse(v[0], v[1])
169 | 
170 |         #merge type
171 |         for stname, stype in ast["type"].iteritems():
172 |             assert stname not in build["type"], "redifine type %s in %s" % (stname, v[1])
173 |             build["type"][stname] = stype
174 |         #merge protocol
175 |         for spname, sp in ast["protocol"].iteritems():
176 |             assert spname not in build["protocol"], "redifine protocol name %s in %s" %  (spname, v[1])
177 |             for proto in build["protocol"]:
178 |                 assert sp["tag"] != build["protocol"][proto]["tag"], "redifine protocol tag %d in %s with %s" % (sp["tag"], proto, spname)
179 |             build["protocol"][spname] = sp
180 | 
181 |     flattypename(build)
182 |     # checkprotocol(build)
183 |     return build
184 | 


--------------------------------------------------------------------------------
/pysproto/core.pyx:
--------------------------------------------------------------------------------
  1 | cdef extern from "sproto.h":
  2 |     enum: SPROTO_REQUEST
  3 |     enum: SPROTO_RESPONSE
  4 |     enum: SPROTO_TINTEGER
  5 |     enum: SPROTO_TBOOLEAN
  6 |     enum: SPROTO_TSTRING
  7 |     enum: SPROTO_TSTRUCT
  8 |     enum: SPROTO_CB_ERROR
  9 |     enum: SPROTO_CB_NIL
 10 |     enum: SPROTO_CB_NOARRAY
 11 | 
 12 |     struct sproto:
 13 |         pass
 14 |     struct sproto_type:
 15 |         pass
 16 |     struct sproto_arg:
 17 |         void *ud
 18 |         const char *tagname
 19 |         int tagid
 20 |         int type
 21 |         sproto_type *subtype
 22 |         void *value
 23 |         int length
 24 |         int index
 25 |         int mainindex
 26 |         int extra
 27 | 
 28 |     sproto* sproto_create(void *, size_t)
 29 |     void sproto_release(sproto*)
 30 |     sproto_type* spt "sproto_type"(sproto*, char*)
 31 |     int sproto_pack(void*, int, void*, int)
 32 |     int sproto_unpack(void*, int, void*, int)
 33 |     int sproto_prototag(const sproto *, const char * name)
 34 |     const char * sproto_protoname(const sproto *, int proto)
 35 |     sproto_type * sproto_protoquery(const sproto *, int proto, int what)
 36 |     int sproto_protoresponse(const sproto*, int)
 37 |     void sproto_dump(sproto*)
 38 |     ctypedef int (*sproto_callback)(const sproto_arg *args) except *
 39 |     int sproto_encode(const sproto_type *, void * buffer, int size, sproto_callback cb, void *ud)
 40 |     int sproto_decode(const sproto_type *, const void * data, int size, sproto_callback cb, void *ud)
 41 | 
 42 | from cpython.pycapsule cimport *
 43 | from libc.stdint cimport *
 44 | from libc.stdio cimport printf
 45 | from libc.string cimport memcpy
 46 | from cpython.mem cimport PyMem_Malloc, PyMem_Free, PyMem_Realloc
 47 | from cpython.object cimport PyObject
 48 | from cpython.exc cimport PyErr_Occurred, PyErr_Print
 49 | 
 50 | cdef extern from "compat.h":
 51 |     ctypedef void (*capsule_dest)(PyObject *)
 52 |     object make_capsule(void *, const char *, capsule_dest)
 53 |     void* get_pointer(object, const char*)
 54 | 
 55 | 
 56 | cdef enum:
 57 |     prealloc = 2050
 58 |     max_deeplevel = 64
 59 | 
 60 | cdef void *invalid_ptr = <void*>(-1)
 61 | 
 62 | cdef struct encode_ud:
 63 |     PyObject *data
 64 |     int deep
 65 | 
 66 | cdef int _encode(const sproto_arg *args) except *:
 67 |     cdef encode_ud *self = <encode_ud*>args.ud
 68 |     # todo check deep
 69 |     data = <object>self.data
 70 |     obj = None
 71 |     tn = args.tagname
 72 |     if args.index > 0:
 73 |         try:
 74 |             c = data[tn]
 75 |         except KeyError:
 76 |             return SPROTO_CB_NOARRAY
 77 |         if args.mainindex >= 0:
 78 |             # c is a dict
 79 |             assert isinstance(c, dict)
 80 |             c = c.values()
 81 |             c.sort()
 82 |         try:
 83 |             obj = c[args.index-1]
 84 |         except IndexError:
 85 |             return SPROTO_CB_NIL
 86 |     else:
 87 |         obj = data.get(tn)
 88 |         if obj == None:
 89 |             return SPROTO_CB_NIL
 90 |     cdef int64_t v, vh
 91 |     cdef double vn
 92 |     cdef char* ptr
 93 |     cdef encode_ud *sub
 94 |     if args.type == SPROTO_TINTEGER:
 95 |         if args.extra:
 96 |             vn = obj
 97 |             v = int(vn*args.extra+0.5)
 98 |         else:
 99 |             v = obj
100 |         vh = v >> 31
101 |         if vh == 0 or vh == -1:
102 |             (<int32_t *>args.value)[0] = <int32_t>v;
103 |             return 4
104 |         else:
105 |             (<int64_t *>args.value)[0] = <int64_t>v;
106 |             return 8
107 |     elif args.type == SPROTO_TBOOLEAN:
108 |         v = obj
109 |         (<int *>args.value)[0] = <int>v
110 |         return 4
111 |     elif args.type == SPROTO_TSTRING:
112 |         ptr = obj
113 |         v = len(obj)
114 |         if v > args.length:
115 |             return SPROTO_CB_ERROR
116 |         memcpy(args.value, ptr, <size_t>v)
117 |         return v
118 |     elif args.type == SPROTO_TSTRUCT:
119 |         sub = <encode_ud *>PyMem_Malloc(sizeof(encode_ud))
120 |         try:
121 |             sub.data = <PyObject *>obj
122 |             sub.deep = self.deep + 1
123 |             r = sproto_encode(args.subtype, args.value, args.length, _encode, sub)
124 |             if r < 0:
125 |                 return SPROTO_CB_ERROR
126 |             return r
127 |         finally:
128 |             PyMem_Free(sub)
129 |     raise Exception("Invalid field type %d"%args.type)
130 | 
131 | def encode(stobj, data):
132 |     assert isinstance(data, dict)
133 |     cdef encode_ud self
134 |     cdef sproto_type *st = <sproto_type*>get_pointer(stobj, NULL)
135 |     if st == invalid_ptr:
136 |         return ""
137 |     cdef char* buf = <char*>PyMem_Malloc(prealloc)
138 |     cdef int sz = prealloc
139 |     try:
140 |         while 1:
141 |             self.data = <PyObject*>data
142 |             self.deep = 0
143 |             r = sproto_encode(st, buf, sz, _encode, &self)
144 |             if PyErr_Occurred():
145 |                 PyErr_Print()
146 |                 raise Exception("encode error")
147 |             if r < 0:
148 |                 sz = sz*2
149 |                 buf = <char*>PyMem_Realloc(buf, sz)
150 |             else:
151 |                 ret = buf[:r]
152 |                 return ret
153 |     finally:
154 |         PyMem_Free(buf)
155 | 
156 | cdef struct decode_ud:
157 |     PyObject* data
158 |     PyObject* key
159 |     int deep
160 |     int mainindex_tag
161 | 
162 | cdef int _decode(const sproto_arg *args) except *:
163 |     cdef decode_ud *self = <decode_ud *>args.ud
164 |     self_d = <dict>self.data
165 |     # todo check deep
166 |     if args.index != 0:
167 |         if args.tagname not in self_d:
168 |             if args.mainindex >= 0:
169 |                 c = {}
170 |             else:
171 |                 c = []
172 |             self_d[args.tagname] = c
173 |         else:
174 |             c = self_d[args.tagname]
175 |         if args.index < 0:
176 |             return 0
177 | 
178 |     ret = None 
179 |     cdef decode_ud *sub
180 |     if args.type == SPROTO_TINTEGER:
181 |         if args.extra:
182 |             ret = (<int64_t *>args.value)[0]
183 |             ret = <double>ret/args.extra
184 |         else:
185 |             ret = (<int64_t *>args.value)[0]
186 |     elif args.type == SPROTO_TBOOLEAN:
187 |         ret = True if (<int64_t *>args.value)[0] > 0 else False
188 |     elif args.type == SPROTO_TSTRING:
189 |         ret = (<char *>args.value)[:args.length]
190 |     elif args.type == SPROTO_TSTRUCT:
191 |         sub = <decode_ud *>PyMem_Malloc(sizeof(decode_ud))
192 |         try:
193 |             sub.deep = self.deep + 1
194 |             sub_d = {}
195 |             sub.data = <PyObject *>sub_d
196 |             if args.mainindex >= 0:
197 |                 sub.mainindex_tag = args.mainindex
198 |                 r = sproto_decode(args.subtype, args.value, args.length, _decode, sub)
199 |                 if r < 0:
200 |                     return SPROTO_CB_ERROR
201 |                 if r != args.length:
202 |                     return r
203 |                 if sub.key == NULL:
204 |                     raise Exception("can't find mainindex (tag=%d) in [%s]a"%(args.mainindex, args.tagname))
205 |                 c[<object>(sub.key)] = sub_d
206 |                 return 0
207 |             else:
208 |                 sub.mainindex_tag = -1
209 |                 r = sproto_decode(args.subtype, args.value, args.length, _decode, sub)
210 |                 if r < 0:
211 |                     return SPROTO_CB_ERROR
212 |                 if r != args.length:
213 |                     return r
214 |                 ret = sub_d
215 |         finally:
216 |             PyMem_Free(sub)
217 |     else:
218 |         raise Exception("Invalid type")
219 | 
220 |     if args.index > 0:
221 |         c.append(ret)
222 |     else:
223 |         if self.mainindex_tag == args.tagid:
224 |             self.key = <PyObject *>ret
225 |         self_d[args.tagname] = ret
226 |     return 0
227 | 
228 | def decode(stobj, data):
229 |     cdef sproto_type *st = <sproto_type*>get_pointer(stobj, NULL)
230 |     if st == invalid_ptr:
231 |         return None, 0
232 |     cdef char *buf = data
233 |     cdef int size = len(data)
234 |     cdef decode_ud self
235 |     d = {}
236 |     self.data = <PyObject *>d
237 |     self.deep = 0
238 |     self.mainindex_tag = -1
239 |     r = sproto_decode(st, buf, size, _decode, &self)
240 |     if PyErr_Occurred():
241 |         PyErr_Print()
242 |         raise Exception("decode error")
243 |     if r < 0:
244 |         raise Exception("decode error")
245 |     return d, r
246 | 
247 | cdef object __wrap_st(void *st):
248 |     if st == NULL:
249 |         return None
250 |     return make_capsule(st, NULL, NULL)
251 | 
252 | cdef void del_sproto(PyObject *obj):
253 |     sp = <sproto*>get_pointer(<object>obj, NULL)
254 |     sproto_release(sp)
255 | 
256 | def newproto(pbin):
257 |     cdef int size = len(pbin)
258 |     cdef char* pb = pbin
259 |     sp = sproto_create(pb, size)
260 |     #printf("sp: %p\n", sp)
261 |     return make_capsule(sp, NULL, del_sproto)
262 | 
263 | def query_type(spobj, protoname):
264 |     sp = <sproto*>get_pointer(spobj, NULL)
265 |     #printf("sp: %p\n", <void*>sp)
266 |     st = <sproto_type*>spt(sp, protoname)
267 |     #printf("st: %p\n", <void*>st)
268 |     return make_capsule(<void*>st, NULL, NULL)
269 | 
270 | def dump(spobj):
271 |     sp = <sproto*>get_pointer(spobj, NULL)
272 |     sproto_dump(sp)
273 | 
274 | def protocol(spobj, name_or_tag):
275 |     sp = <sproto*>get_pointer(spobj, NULL)
276 |     ret = None
277 |     if isinstance(name_or_tag, int):
278 |         tag = name_or_tag
279 |         name = sproto_protoname(sp, name_or_tag)
280 |         if not name:
281 |             return None
282 |         ret = name
283 |     else:
284 |         assert isinstance(name_or_tag, str)
285 |         tag = sproto_prototag(sp, name_or_tag)
286 |         if tag < 0:
287 |             return None
288 |         ret = tag
289 |         
290 |     req = sproto_protoquery(sp, tag, SPROTO_REQUEST)
291 |     rsp = sproto_protoquery(sp, tag, SPROTO_RESPONSE)
292 |     if rsp == NULL and sproto_protoresponse(sp, tag):
293 |         rsp = invalid_ptr
294 |     return ret, __wrap_st(req), __wrap_st(rsp)
295 | 
296 | def pack(data):
297 |     cdef char* ptr = data
298 |     cdef int size = len(data)
299 |     cdef maxsz = (size + 2047) / 2048 * 2 + size + 2
300 |     cdef char* buf = <char*>PyMem_Malloc(maxsz)
301 |     try:
302 |         out_sz = sproto_pack(ptr, size, buf, maxsz)
303 |         if out_sz > maxsz:
304 |             return None
305 |         ret = buf[:out_sz]
306 |         return ret
307 |     finally:
308 |         PyMem_Free(buf)
309 | 
310 | def unpack(data):
311 |     cdef char* ptr = data
312 |     cdef int size = len(data)
313 |     cdef char* buf = <char*>PyMem_Malloc(prealloc)
314 |     cdef r = 0
315 |     try:
316 |         r = sproto_unpack(ptr, size, buf, prealloc)
317 |         if r > prealloc:
318 |             buf = <char*>PyMem_Realloc(buf, r)
319 |             r = sproto_unpack(ptr, size, buf, r)
320 |         if r < 0:
321 |             raise Exception("Invalid unpack stream")
322 |         ret = buf[:r]
323 |         return ret
324 |     finally:
325 |         PyMem_Free(buf)
326 |     
327 | 


--------------------------------------------------------------------------------
/pysproto/sproto.c:
--------------------------------------------------------------------------------
   1 | #include <stdlib.h>
   2 | #include <string.h>
   3 | #include <stdio.h>
   4 | #include <assert.h>
   5 | #include "msvcint.h"
   6 | 
   7 | #include "sproto.h"
   8 | 
   9 | #define SPROTO_TARRAY 0x80
  10 | #define CHUNK_SIZE 1000
  11 | #define SIZEOF_LENGTH 4
  12 | #define SIZEOF_HEADER 2
  13 | #define SIZEOF_FIELD 2
  14 | 
  15 | struct field {
  16 | 	int tag;
  17 | 	int type;
  18 | 	const char * name;
  19 | 	struct sproto_type * st;
  20 | 	int key;
  21 | 	int extra;
  22 | };
  23 | 
  24 | struct sproto_type {
  25 | 	const char * name;
  26 | 	int n;
  27 | 	int base;
  28 | 	int maxn;
  29 | 	struct field *f;
  30 | };
  31 | 
  32 | struct protocol {
  33 | 	const char *name;
  34 | 	int tag;
  35 | 	int confirm;	// confirm == 1 where response nil
  36 | 	struct sproto_type * p[2];
  37 | };
  38 | 
  39 | struct chunk {
  40 | 	struct chunk * next;
  41 | };
  42 | 
  43 | struct pool {
  44 | 	struct chunk * header;
  45 | 	struct chunk * current;
  46 | 	int current_used;
  47 | };
  48 | 
  49 | struct sproto {
  50 | 	struct pool memory;
  51 | 	int type_n;
  52 | 	int protocol_n;
  53 | 	struct sproto_type * type;
  54 | 	struct protocol * proto;
  55 | };
  56 | 
  57 | static void
  58 | pool_init(struct pool *p) {
  59 | 	p->header = NULL;
  60 | 	p->current = NULL;
  61 | 	p->current_used = 0;
  62 | }
  63 | 
  64 | static void
  65 | pool_release(struct pool *p) {
  66 | 	struct chunk * tmp = p->header;
  67 | 	while (tmp) {
  68 | 		struct chunk * n = tmp->next;
  69 | 		free(tmp);
  70 | 		tmp = n;
  71 | 	}
  72 | }
  73 | 
  74 | static void *
  75 | pool_newchunk(struct pool *p, size_t sz) {
  76 | 	struct chunk * t = malloc(sz + sizeof(struct chunk));
  77 | 	if (t == NULL)
  78 | 		return NULL;
  79 | 	t->next = p->header;
  80 | 	p->header = t;
  81 | 	return t+1;
  82 | }
  83 | 
  84 | static void *
  85 | pool_alloc(struct pool *p, size_t sz) {
  86 | 	// align by 8
  87 | 	sz = (sz + 7) & ~7;
  88 | 	if (sz >= CHUNK_SIZE) {
  89 | 		return pool_newchunk(p, sz);
  90 | 	}
  91 | 	if (p->current == NULL) {
  92 | 		if (pool_newchunk(p, CHUNK_SIZE) == NULL)
  93 | 			return NULL;
  94 | 		p->current = p->header;
  95 | 	}
  96 | 	if (sz + p->current_used <= CHUNK_SIZE) {
  97 | 		void * ret = (char *)(p->current+1) + p->current_used;
  98 | 		p->current_used += sz;
  99 | 		return ret;
 100 | 	}
 101 | 
 102 | 	if (sz >= p->current_used) {
 103 | 		return pool_newchunk(p, sz);
 104 | 	} else {
 105 | 		void * ret = pool_newchunk(p, CHUNK_SIZE);
 106 | 		p->current = p->header;
 107 | 		p->current_used = sz;
 108 | 		return ret;
 109 | 	}
 110 | }
 111 | 
 112 | static inline int
 113 | toword(const uint8_t * p) {
 114 | 	return p[0] | p[1]<<8;
 115 | }
 116 | 
 117 | static inline uint32_t
 118 | todword(const uint8_t *p) {
 119 | 	return p[0] | p[1]<<8 | p[2]<<16 | p[3]<<24;
 120 | }
 121 | 
 122 | static int
 123 | count_array(const uint8_t * stream) {
 124 | 	uint32_t length = todword(stream);
 125 | 	int n = 0;
 126 | 	stream += SIZEOF_LENGTH;
 127 | 	while (length > 0) {
 128 | 		uint32_t nsz;
 129 | 		if (length < SIZEOF_LENGTH)
 130 | 			return -1;
 131 | 		nsz = todword(stream);
 132 | 		nsz += SIZEOF_LENGTH;
 133 | 		if (nsz > length)
 134 | 			return -1;
 135 | 		++n;
 136 | 		stream += nsz;
 137 | 		length -= nsz;
 138 | 	}
 139 | 
 140 | 	return n;
 141 | }
 142 | 
 143 | static int
 144 | struct_field(const uint8_t * stream, size_t sz) {
 145 | 	const uint8_t * field;
 146 | 	int fn, header, i;
 147 | 	if (sz < SIZEOF_LENGTH)
 148 | 		return -1;
 149 | 	fn = toword(stream);
 150 | 	header = SIZEOF_HEADER + SIZEOF_FIELD * fn;
 151 | 	if (sz < header)
 152 | 		return -1;
 153 | 	field = stream + SIZEOF_HEADER;
 154 | 	sz -= header;
 155 | 	stream += header;
 156 | 	for (i=0;i<fn;i++) {
 157 | 		int value= toword(field + i * SIZEOF_FIELD);
 158 | 		uint32_t dsz;
 159 | 		if (value != 0)
 160 | 			continue;
 161 | 		if (sz < SIZEOF_LENGTH)
 162 | 			return -1;
 163 | 		dsz = todword(stream);
 164 | 		if (sz < SIZEOF_LENGTH + dsz)
 165 | 			return -1;
 166 | 		stream += SIZEOF_LENGTH + dsz;
 167 | 		sz -= SIZEOF_LENGTH + dsz;
 168 | 	}
 169 | 
 170 | 	return fn;
 171 | }
 172 | 
 173 | static const char *
 174 | import_string(struct sproto *s, const uint8_t * stream) {
 175 | 	uint32_t sz = todword(stream);
 176 | 	char * buffer = pool_alloc(&s->memory, sz+1);
 177 | 	memcpy(buffer, stream+SIZEOF_LENGTH, sz);
 178 | 	buffer[sz] = '\0';
 179 | 	return buffer;
 180 | }
 181 | 
 182 | static int
 183 | calc_pow(int base, int n) {
 184 | 	int r;
 185 | 	if (n == 0)
 186 | 		return 1;
 187 | 	r = calc_pow(base * base , n / 2);
 188 | 	if (n&1) {
 189 | 		r *= base;
 190 | 	}
 191 | 	return r;
 192 | }
 193 | 
 194 | static const uint8_t *
 195 | import_field(struct sproto *s, struct field *f, const uint8_t * stream) {
 196 | 	uint32_t sz;
 197 | 	const uint8_t * result;
 198 | 	int fn;
 199 | 	int i;
 200 | 	int array = 0;
 201 | 	int tag = -1;
 202 | 	f->tag = -1;
 203 | 	f->type = -1;
 204 | 	f->name = NULL;
 205 | 	f->st = NULL;
 206 | 	f->key = -1;
 207 | 	f->extra = 0;
 208 | 
 209 | 	sz = todword(stream);
 210 | 	stream += SIZEOF_LENGTH;
 211 | 	result = stream + sz;
 212 | 	fn = struct_field(stream, sz);
 213 | 	if (fn < 0)
 214 | 		return NULL;
 215 | 	stream += SIZEOF_HEADER;
 216 | 	for (i=0;i<fn;i++) {
 217 | 		int value;
 218 | 		++tag;
 219 | 		value = toword(stream + SIZEOF_FIELD * i);
 220 | 		if (value & 1) {
 221 | 			tag+= value/2;
 222 | 			continue;
 223 | 		}
 224 | 		if (tag == 0) { // name
 225 | 			if (value != 0)
 226 | 				return NULL;
 227 | 			f->name = import_string(s, stream + fn * SIZEOF_FIELD);
 228 | 			continue;
 229 | 		}
 230 | 		if (value == 0)
 231 | 			return NULL;
 232 | 		value = value/2 - 1;
 233 | 		switch(tag) {
 234 | 		case 1: // buildin
 235 | 			if (value >= SPROTO_TSTRUCT)
 236 | 				return NULL;	// invalid buildin type
 237 | 			f->type = value;
 238 | 			break;
 239 | 		case 2: // type index
 240 | 			if (f->type == SPROTO_TINTEGER) {
 241 | 				f->extra = calc_pow(10, value);
 242 | 			} else if (f->type == SPROTO_TSTRING) {
 243 | 				f->extra = value;	// string if 0 ; binary is 1
 244 | 			} else {
 245 | 				if (value >= s->type_n)
 246 | 					return NULL;	// invalid type index
 247 | 				if (f->type >= 0)
 248 | 					return NULL;
 249 | 				f->type = SPROTO_TSTRUCT;
 250 | 				f->st = &s->type[value];
 251 | 			}
 252 | 			break;
 253 | 		case 3: // tag
 254 | 			f->tag = value;
 255 | 			break;
 256 | 		case 4: // array
 257 | 			if (value)
 258 | 				array = SPROTO_TARRAY;
 259 | 			break;
 260 | 		case 5:	// key
 261 | 			f->key = value;
 262 | 			break;
 263 | 		default:
 264 | 			return NULL;
 265 | 		}
 266 | 	}
 267 | 	if (f->tag < 0 || f->type < 0 || f->name == NULL)
 268 | 		return NULL;
 269 | 	f->type |= array;
 270 | 
 271 | 	return result;
 272 | }
 273 | 
 274 | /*
 275 | .type {
 276 | 	.field {
 277 | 		name 0 : string
 278 | 		buildin 1 : integer
 279 | 		type 2 : integer
 280 | 		tag 3 : integer
 281 | 		array 4 : boolean
 282 | 	}
 283 | 	name 0 : string
 284 | 	fields 1 : *field
 285 | }
 286 | */
 287 | static const uint8_t *
 288 | import_type(struct sproto *s, struct sproto_type *t, const uint8_t * stream) {
 289 | 	const uint8_t * result;
 290 | 	uint32_t sz = todword(stream);
 291 | 	int i;
 292 | 	int fn;
 293 | 	int n;
 294 | 	int maxn;
 295 | 	int last;
 296 | 	stream += SIZEOF_LENGTH;
 297 | 	result = stream + sz;
 298 | 	fn = struct_field(stream, sz);
 299 | 	if (fn <= 0 || fn > 2)
 300 | 		return NULL;
 301 | 	for (i=0;i<fn*SIZEOF_FIELD;i+=SIZEOF_FIELD) {
 302 | 		// name and fields must encode to 0
 303 | 		int v = toword(stream + SIZEOF_HEADER + i);
 304 | 		if (v != 0)
 305 | 			return NULL;
 306 | 	}
 307 | 	memset(t, 0, sizeof(*t));
 308 | 	stream += SIZEOF_HEADER + fn * SIZEOF_FIELD;
 309 | 	t->name = import_string(s, stream);
 310 | 	if (fn == 1) {
 311 | 		return result;
 312 | 	}
 313 | 	stream += todword(stream)+SIZEOF_LENGTH;	// second data
 314 | 	n = count_array(stream);
 315 | 	if (n<0)
 316 | 		return NULL;
 317 | 	stream += SIZEOF_LENGTH;
 318 | 	maxn = n;
 319 | 	last = -1;
 320 | 	t->n = n;
 321 | 	t->f = pool_alloc(&s->memory, sizeof(struct field) * n);
 322 | 	for (i=0;i<n;i++) {
 323 | 		int tag;
 324 | 		struct field *f = &t->f[i];
 325 | 		stream = import_field(s, f, stream);
 326 | 		if (stream == NULL)
 327 | 			return NULL;
 328 | 		tag = f->tag;
 329 | 		if (tag <= last)
 330 | 			return NULL;	// tag must in ascending order
 331 | 		if (tag > last+1) {
 332 | 			++maxn;
 333 | 		}
 334 | 		last = tag;
 335 | 	}
 336 | 	t->maxn = maxn;
 337 | 	t->base = t->f[0].tag;
 338 | 	n = t->f[n-1].tag - t->base + 1;
 339 | 	if (n != t->n) {
 340 | 		t->base = -1;
 341 | 	}
 342 | 	return result;
 343 | }
 344 | 
 345 | /*
 346 | .protocol {
 347 | 	name 0 : string
 348 | 	tag 1 : integer
 349 | 	request 2 : integer
 350 | 	response 3 : integer
 351 | }
 352 | */
 353 | static const uint8_t *
 354 | import_protocol(struct sproto *s, struct protocol *p, const uint8_t * stream) {
 355 | 	const uint8_t * result;
 356 | 	uint32_t sz = todword(stream);
 357 | 	int fn;
 358 | 	int i;
 359 | 	int tag;
 360 | 	stream += SIZEOF_LENGTH;
 361 | 	result = stream + sz;
 362 | 	fn = struct_field(stream, sz);
 363 | 	stream += SIZEOF_HEADER;
 364 | 	p->name = NULL;
 365 | 	p->tag = -1;
 366 | 	p->p[SPROTO_REQUEST] = NULL;
 367 | 	p->p[SPROTO_RESPONSE] = NULL;
 368 | 	p->confirm = 0;
 369 | 	tag = 0;
 370 | 	for (i=0;i<fn;i++,tag++) {
 371 | 		int value = toword(stream + SIZEOF_FIELD * i);
 372 | 		if (value & 1) {
 373 | 			tag += (value-1)/2;
 374 | 			continue;
 375 | 		}
 376 | 		value = value/2 - 1;
 377 | 		switch (i) {
 378 | 		case 0: // name
 379 | 			if (value != -1) {
 380 | 				return NULL;
 381 | 			}
 382 | 			p->name = import_string(s, stream + SIZEOF_FIELD *fn);
 383 | 			break;
 384 | 		case 1: // tag
 385 | 			if (value < 0) {
 386 | 				return NULL;
 387 | 			}
 388 | 			p->tag = value;
 389 | 			break;
 390 | 		case 2: // request
 391 | 			if (value < 0 || value>=s->type_n)
 392 | 				return NULL;
 393 | 			p->p[SPROTO_REQUEST] = &s->type[value];
 394 | 			break;
 395 | 		case 3: // response
 396 | 			if (value < 0 || value>=s->type_n)
 397 | 				return NULL;
 398 | 			p->p[SPROTO_RESPONSE] = &s->type[value];
 399 | 			break;
 400 | 		case 4:	// confirm
 401 | 			p->confirm = value;
 402 | 			break;
 403 | 		default:
 404 | 			return NULL;
 405 | 		}
 406 | 	}
 407 | 
 408 | 	if (p->name == NULL || p->tag<0) {
 409 | 		return NULL;
 410 | 	}
 411 | 
 412 | 	return result;
 413 | }
 414 | 
 415 | static struct sproto *
 416 | create_from_bundle(struct sproto *s, const uint8_t * stream, size_t sz) {
 417 | 	const uint8_t * content;
 418 | 	const uint8_t * typedata = NULL;
 419 | 	const uint8_t * protocoldata = NULL;
 420 | 	int fn = struct_field(stream, sz);
 421 | 	int i;
 422 | 	if (fn < 0 || fn > 2)
 423 | 		return NULL;
 424 | 
 425 | 	stream += SIZEOF_HEADER;
 426 | 	content = stream + fn*SIZEOF_FIELD;
 427 | 
 428 | 	for (i=0;i<fn;i++) {
 429 | 		int value = toword(stream + i*SIZEOF_FIELD);
 430 | 		int n;
 431 | 		if (value != 0)
 432 | 			return NULL;
 433 | 		n = count_array(content);
 434 | 		if (n<0)
 435 | 			return NULL;
 436 | 		if (i == 0) {
 437 | 			typedata = content+SIZEOF_LENGTH;
 438 | 			s->type_n = n;
 439 | 			s->type = pool_alloc(&s->memory, n * sizeof(*s->type));
 440 | 		} else {
 441 | 			protocoldata = content+SIZEOF_LENGTH;
 442 | 			s->protocol_n = n;
 443 | 			s->proto = pool_alloc(&s->memory, n * sizeof(*s->proto));
 444 | 		}
 445 | 		content += todword(content) + SIZEOF_LENGTH;
 446 | 	}
 447 | 
 448 | 	for (i=0;i<s->type_n;i++) {
 449 | 		typedata = import_type(s, &s->type[i], typedata);
 450 | 		if (typedata == NULL) {
 451 | 			return NULL;
 452 | 		}
 453 | 	}
 454 | 	for (i=0;i<s->protocol_n;i++) {
 455 | 		protocoldata = import_protocol(s, &s->proto[i], protocoldata);
 456 | 		if (protocoldata == NULL) {
 457 | 			return NULL;
 458 | 		}
 459 | 	}
 460 | 
 461 | 	return s;
 462 | }
 463 | 
 464 | struct sproto *
 465 | sproto_create(const void * proto, size_t sz) {
 466 | 	struct pool mem;
 467 | 	struct sproto * s;
 468 | 	pool_init(&mem);
 469 | 	s = pool_alloc(&mem, sizeof(*s));
 470 | 	if (s == NULL)
 471 | 		return NULL;
 472 | 	memset(s, 0, sizeof(*s));
 473 | 	s->memory = mem;
 474 | 	if (create_from_bundle(s, proto, sz) == NULL) {
 475 | 		pool_release(&s->memory);
 476 | 		return NULL;
 477 | 	}
 478 | 	return s;
 479 | }
 480 | 
 481 | void
 482 | sproto_release(struct sproto * s) {
 483 | 	if (s == NULL)
 484 | 		return;
 485 | 	pool_release(&s->memory);
 486 | }
 487 | 
 488 | void
 489 | sproto_dump(struct sproto *s) {
 490 | 	int i,j;
 491 | 	printf("=== %d types ===\n", s->type_n);
 492 | 	for (i=0;i<s->type_n;i++) {
 493 | 		struct sproto_type *t = &s->type[i];
 494 | 		printf("%s\n", t->name);
 495 | 		for (j=0;j<t->n;j++) {
 496 | 			char array[2] = { 0, 0 };
 497 | 			const char * type_name = NULL;
 498 | 			struct field *f = &t->f[j];
 499 | 			int type = f->type & ~SPROTO_TARRAY;
 500 | 			if (f->type & SPROTO_TARRAY) {
 501 | 				array[0] = '*';
 502 | 			} else {
 503 | 				array[0] = 0;
 504 | 			}
 505 | 			if (type == SPROTO_TSTRUCT) {
 506 | 				type_name = f->st->name;
 507 | 			} else {
 508 | 				switch(type) {
 509 | 				case SPROTO_TINTEGER:
 510 | 					if (f->extra) {
 511 | 						type_name = "decimal";
 512 | 					} else {
 513 | 						type_name = "integer";
 514 | 					}
 515 | 					break;
 516 | 				case SPROTO_TBOOLEAN:
 517 | 					type_name = "boolean";
 518 | 					break;
 519 | 				case SPROTO_TSTRING:
 520 | 					if (f->extra == SPROTO_TSTRING_BINARY)
 521 | 						type_name = "binary";
 522 | 					else
 523 | 						type_name = "string";
 524 | 					break;
 525 | 				default:
 526 | 					type_name = "invalid";
 527 | 					break;
 528 | 				}
 529 | 			}
 530 | 			printf("\t%s (%d) %s%s", f->name, f->tag, array, type_name);
 531 | 			if (type == SPROTO_TINTEGER && f->extra > 0) {
 532 | 				printf("(%d)", f->extra);
 533 | 			}
 534 | 			if (f->key >= 0) {
 535 | 				printf("[%d]", f->key);
 536 | 			}
 537 | 			printf("\n");
 538 | 		}
 539 | 	}
 540 | 	printf("=== %d protocol ===\n", s->protocol_n);
 541 | 	for (i=0;i<s->protocol_n;i++) {
 542 | 		struct protocol *p = &s->proto[i];
 543 | 		if (p->p[SPROTO_REQUEST]) {
 544 | 			printf("\t%s (%d) request:%s", p->name, p->tag, p->p[SPROTO_REQUEST]->name);
 545 | 		} else {
 546 | 			printf("\t%s (%d) request:(null)", p->name, p->tag);
 547 | 		}
 548 | 		if (p->p[SPROTO_RESPONSE]) {
 549 | 			printf(" response:%s", p->p[SPROTO_RESPONSE]->name);
 550 | 		} else if (p->confirm) {
 551 | 			printf(" response nil");
 552 | 		}
 553 | 		printf("\n");
 554 | 	}
 555 | }
 556 | 
 557 | // query
 558 | int
 559 | sproto_prototag(const struct sproto *sp, const char * name) {
 560 | 	int i;
 561 | 	for (i=0;i<sp->protocol_n;i++) {
 562 | 		if (strcmp(name, sp->proto[i].name) == 0) {
 563 | 			return sp->proto[i].tag;
 564 | 		}
 565 | 	}
 566 | 	return -1;
 567 | }
 568 | 
 569 | static struct protocol *
 570 | query_proto(const struct sproto *sp, int tag) {
 571 | 	int begin = 0, end = sp->protocol_n;
 572 | 	while(begin<end) {
 573 | 		int mid = (begin+end)/2;
 574 | 		int t = sp->proto[mid].tag;
 575 | 		if (t==tag) {
 576 | 			return &sp->proto[mid];
 577 | 		}
 578 | 		if (tag > t) {
 579 | 			begin = mid+1;
 580 | 		} else {
 581 | 			end = mid;
 582 | 		}
 583 | 	}
 584 | 	return NULL;
 585 | }
 586 | 
 587 | struct sproto_type *
 588 | sproto_protoquery(const struct sproto *sp, int proto, int what) {
 589 | 	struct protocol * p;
 590 | 	if (what <0 || what >1) {
 591 | 		return NULL;
 592 | 	}
 593 | 	p = query_proto(sp, proto);
 594 | 	if (p) {
 595 | 		return p->p[what];
 596 | 	}
 597 | 	return NULL;
 598 | }
 599 | 
 600 | int
 601 | sproto_protoresponse(const struct sproto * sp, int proto) {
 602 | 	struct protocol * p = query_proto(sp, proto);
 603 | 	return (p!=NULL && (p->p[SPROTO_RESPONSE] || p->confirm));
 604 | }
 605 | 
 606 | const char *
 607 | sproto_protoname(const struct sproto *sp, int proto) {
 608 | 	struct protocol * p = query_proto(sp, proto);
 609 | 	if (p) {
 610 | 		return p->name;
 611 | 	}
 612 | 	return NULL;
 613 | }
 614 | 
 615 | struct sproto_type *
 616 | sproto_type(const struct sproto *sp, const char * type_name) {
 617 | 	int i;
 618 | 	for (i=0;i<sp->type_n;i++) {
 619 | 		if (strcmp(type_name, sp->type[i].name) == 0) {
 620 | 			return &sp->type[i];
 621 | 		}
 622 | 	}
 623 | 	return NULL;
 624 | }
 625 | 
 626 | const char *
 627 | sproto_name(struct sproto_type * st) {
 628 | 	return st->name;
 629 | }
 630 | 
 631 | static struct field *
 632 | findtag(const struct sproto_type *st, int tag) {
 633 | 	int begin, end;
 634 | 	if (st->base >=0 ) {
 635 | 		tag -= st->base;
 636 | 		if (tag < 0 || tag >= st->n)
 637 | 			return NULL;
 638 | 		return &st->f[tag];
 639 | 	}
 640 | 	begin = 0;
 641 | 	end = st->n;
 642 | 	while (begin < end) {
 643 | 		int mid = (begin+end)/2;
 644 | 		struct field *f = &st->f[mid];
 645 | 		int t = f->tag;
 646 | 		if (t == tag) {
 647 | 			return f;
 648 | 		}
 649 | 		if (tag > t) {
 650 | 			begin = mid + 1;
 651 | 		} else {
 652 | 			end = mid;
 653 | 		}
 654 | 	}
 655 | 	return NULL;
 656 | }
 657 | 
 658 | // encode & decode
 659 | // sproto_callback(void *ud, int tag, int type, struct sproto_type *, void *value, int length)
 660 | //	  return size, -1 means error
 661 | 
 662 | static inline int
 663 | fill_size(uint8_t * data, int sz) {
 664 | 	data[0] = sz & 0xff;
 665 | 	data[1] = (sz >> 8) & 0xff;
 666 | 	data[2] = (sz >> 16) & 0xff;
 667 | 	data[3] = (sz >> 24) & 0xff;
 668 | 	return sz + SIZEOF_LENGTH;
 669 | }
 670 | 
 671 | static int
 672 | encode_integer(uint32_t v, uint8_t * data, int size) {
 673 | 	if (size < SIZEOF_LENGTH + sizeof(v))
 674 | 		return -1;
 675 | 	data[4] = v & 0xff;
 676 | 	data[5] = (v >> 8) & 0xff;
 677 | 	data[6] = (v >> 16) & 0xff;
 678 | 	data[7] = (v >> 24) & 0xff;
 679 | 	return fill_size(data, sizeof(v));
 680 | }
 681 | 
 682 | static int
 683 | encode_uint64(uint64_t v, uint8_t * data, int size) {
 684 | 	if (size < SIZEOF_LENGTH + sizeof(v))
 685 | 		return -1;
 686 | 	data[4] = v & 0xff;
 687 | 	data[5] = (v >> 8) & 0xff;
 688 | 	data[6] = (v >> 16) & 0xff;
 689 | 	data[7] = (v >> 24) & 0xff;
 690 | 	data[8] = (v >> 32) & 0xff;
 691 | 	data[9] = (v >> 40) & 0xff;
 692 | 	data[10] = (v >> 48) & 0xff;
 693 | 	data[11] = (v >> 56) & 0xff;
 694 | 	return fill_size(data, sizeof(v));
 695 | }
 696 | 
 697 | /*
 698 | //#define CB(tagname,type,index,subtype,value,length) cb(ud, tagname,type,index,subtype,value,length)
 699 | 
 700 | static int
 701 | do_cb(sproto_callback cb, void *ud, const char *tagname, int type, int index, struct sproto_type *subtype, void *value, int length) {
 702 | 	if (subtype) {
 703 | 		if (type >= 0) {
 704 | 			printf("callback: tag=%s[%d], subtype[%s]:%d\n",tagname,index, subtype->name, type);
 705 | 		} else {
 706 | 			printf("callback: tag=%s[%d], subtype[%s]\n",tagname,index, subtype->name);
 707 | 		}
 708 | 	} else if (index > 0) {
 709 | 		printf("callback: tag=%s[%d]\n",tagname,index);
 710 | 	} else if (index == 0) {
 711 | 		printf("callback: tag=%s\n",tagname);
 712 | 	} else {
 713 | 		printf("callback: tag=%s [mainkey]\n",tagname);
 714 | 	}
 715 | 	return cb(ud, tagname,type,index,subtype,value,length);
 716 | }
 717 | #define CB(tagname,type,index,subtype,value,length) do_cb(cb,ud, tagname,type,index,subtype,value,length)
 718 | */
 719 | 
 720 | static int
 721 | encode_object(sproto_callback cb, struct sproto_arg *args, uint8_t *data, int size) {
 722 | 	int sz;
 723 | 	if (size < SIZEOF_LENGTH)
 724 | 		return -1;
 725 | 	args->value = data+SIZEOF_LENGTH;
 726 | 	args->length = size-SIZEOF_LENGTH;
 727 | 	sz = cb(args);
 728 | 	if (sz < 0) {
 729 | 		if (sz == SPROTO_CB_NIL)
 730 | 			return 0;
 731 | 		return -1;	// sz == SPROTO_CB_ERROR
 732 | 	}
 733 | 	assert(sz <= size-SIZEOF_LENGTH);	// verify buffer overflow
 734 | 	return fill_size(data, sz);
 735 | }
 736 | 
 737 | static inline void
 738 | uint32_to_uint64(int negative, uint8_t *buffer) {
 739 | 	if (negative) {
 740 | 		buffer[4] = 0xff;
 741 | 		buffer[5] = 0xff;
 742 | 		buffer[6] = 0xff;
 743 | 		buffer[7] = 0xff;
 744 | 	} else {
 745 | 		buffer[4] = 0;
 746 | 		buffer[5] = 0;
 747 | 		buffer[6] = 0;
 748 | 		buffer[7] = 0;
 749 | 	}
 750 | }
 751 | 
 752 | static uint8_t *
 753 | encode_integer_array(sproto_callback cb, struct sproto_arg *args, uint8_t *buffer, int size, int *noarray) {
 754 | 	uint8_t * header = buffer;
 755 | 	int intlen;
 756 | 	int index;
 757 | 	if (size < 1)
 758 | 		return NULL;
 759 | 	buffer++;
 760 | 	size--;
 761 | 	intlen = sizeof(uint32_t);
 762 | 	index = 1;
 763 | 	*noarray = 0;
 764 | 
 765 | 	for (;;) {
 766 | 		int sz;
 767 | 		union {
 768 | 			uint64_t u64;
 769 | 			uint32_t u32;
 770 | 		} u;
 771 | 		args->value = &u;
 772 | 		args->length = sizeof(u);
 773 | 		args->index = index;
 774 | 		sz = cb(args);
 775 | 		if (sz <= 0) {
 776 | 			if (sz == SPROTO_CB_NIL) // nil object, end of array
 777 | 				break;
 778 | 			if (sz == SPROTO_CB_NOARRAY) {	// no array, don't encode it
 779 | 				*noarray = 1;
 780 | 				break;
 781 | 			}
 782 | 			return NULL;	// sz == SPROTO_CB_ERROR
 783 | 		}
 784 | 		if (size < sizeof(uint64_t))
 785 | 			return NULL;
 786 | 		if (sz == sizeof(uint32_t)) {
 787 | 			uint32_t v = u.u32;
 788 | 			buffer[0] = v & 0xff;
 789 | 			buffer[1] = (v >> 8) & 0xff;
 790 | 			buffer[2] = (v >> 16) & 0xff;
 791 | 			buffer[3] = (v >> 24) & 0xff;
 792 | 
 793 | 			if (intlen == sizeof(uint64_t)) {
 794 | 				uint32_to_uint64(v & 0x80000000, buffer);
 795 | 			}
 796 | 		} else {
 797 | 			uint64_t v;
 798 | 			if (sz != sizeof(uint64_t))
 799 | 				return NULL;
 800 | 			if (intlen == sizeof(uint32_t)) {
 801 | 				int i;
 802 | 				// rearrange
 803 | 				size -= (index-1) * sizeof(uint32_t);
 804 | 				if (size < sizeof(uint64_t))
 805 | 					return NULL;
 806 | 				buffer += (index-1) * sizeof(uint32_t);
 807 | 				for (i=index-2;i>=0;i--) {
 808 | 					int negative;
 809 | 					memcpy(header+1+i*sizeof(uint64_t), header+1+i*sizeof(uint32_t), sizeof(uint32_t));
 810 | 					negative = header[1+i*sizeof(uint64_t)+3] & 0x80;
 811 | 					uint32_to_uint64(negative, header+1+i*sizeof(uint64_t));
 812 | 				}
 813 | 				intlen = sizeof(uint64_t);
 814 | 			}
 815 | 
 816 | 			v = u.u64;
 817 | 			buffer[0] = v & 0xff;
 818 | 			buffer[1] = (v >> 8) & 0xff;
 819 | 			buffer[2] = (v >> 16) & 0xff;
 820 | 			buffer[3] = (v >> 24) & 0xff;
 821 | 			buffer[4] = (v >> 32) & 0xff;
 822 | 			buffer[5] = (v >> 40) & 0xff;
 823 | 			buffer[6] = (v >> 48) & 0xff;
 824 | 			buffer[7] = (v >> 56) & 0xff;
 825 | 		}
 826 | 
 827 | 		size -= intlen;
 828 | 		buffer += intlen;
 829 | 		index++;
 830 | 	}
 831 | 
 832 | 	if (buffer == header + 1) {
 833 | 		return header;
 834 | 	}
 835 | 	*header = (uint8_t)intlen;
 836 | 	return buffer;
 837 | }
 838 | 
 839 | static int
 840 | encode_array(sproto_callback cb, struct sproto_arg *args, uint8_t *data, int size) {
 841 | 	uint8_t * buffer;
 842 | 	int sz;
 843 | 	if (size < SIZEOF_LENGTH)
 844 | 		return -1;
 845 | 	size -= SIZEOF_LENGTH;
 846 | 	buffer = data + SIZEOF_LENGTH;
 847 | 	switch (args->type) {
 848 | 	case SPROTO_TINTEGER: {
 849 | 		int noarray;
 850 | 		buffer = encode_integer_array(cb,args,buffer,size, &noarray);
 851 | 		if (buffer == NULL)
 852 | 			return -1;
 853 | 	
 854 | 		if (noarray) {
 855 | 			return 0;
 856 | 		}
 857 | 		break;
 858 | 	}
 859 | 	case SPROTO_TBOOLEAN:
 860 | 		args->index = 1;
 861 | 		for (;;) {
 862 | 			int v = 0;
 863 | 			args->value = &v;
 864 | 			args->length = sizeof(v);
 865 | 			sz = cb(args);
 866 | 			if (sz < 0) {
 867 | 				if (sz == SPROTO_CB_NIL)		// nil object , end of array
 868 | 					break;
 869 | 				if (sz == SPROTO_CB_NOARRAY)	// no array, don't encode it
 870 | 					return 0;
 871 | 				return -1;	// sz == SPROTO_CB_ERROR
 872 | 			}
 873 | 			if (size < 1)
 874 | 				return -1;
 875 | 			buffer[0] = v ? 1: 0;
 876 | 			size -= 1;
 877 | 			buffer += 1;
 878 | 			++args->index;
 879 | 		}
 880 | 		break;
 881 | 	default:
 882 | 		args->index = 1;
 883 | 		for (;;) {
 884 | 			if (size < SIZEOF_LENGTH)
 885 | 				return -1;
 886 | 			size -= SIZEOF_LENGTH;
 887 | 			args->value = buffer+SIZEOF_LENGTH;
 888 | 			args->length = size;
 889 | 			sz = cb(args);
 890 | 			if (sz < 0) {
 891 | 				if (sz == SPROTO_CB_NIL) {
 892 | 					break;
 893 | 				}
 894 | 				if (sz == SPROTO_CB_NOARRAY)	// no array, don't encode it
 895 | 					return 0;
 896 | 				return -1;	// sz == SPROTO_CB_ERROR
 897 | 			}
 898 | 			fill_size(buffer, sz);
 899 | 			buffer += SIZEOF_LENGTH+sz;
 900 | 			size -=sz;
 901 | 			++args->index;
 902 | 		}
 903 | 		break;
 904 | 	}
 905 | 	sz = buffer - (data + SIZEOF_LENGTH);
 906 | 	return fill_size(data, sz);
 907 | }
 908 | 
 909 | int
 910 | sproto_encode(const struct sproto_type *st, void * buffer, int size, sproto_callback cb, void *ud) {
 911 | 	struct sproto_arg args;
 912 | 	uint8_t * header = buffer;
 913 | 	uint8_t * data;
 914 | 	int header_sz = SIZEOF_HEADER + st->maxn * SIZEOF_FIELD;
 915 | 	int i;
 916 | 	int index;
 917 | 	int lasttag;
 918 | 	int datasz;
 919 | 	if (size < header_sz)
 920 | 		return -1;
 921 | 	args.ud = ud;
 922 | 	data = header + header_sz;
 923 | 	size -= header_sz;
 924 | 	index = 0;
 925 | 	lasttag = -1;
 926 | 	for (i=0;i<st->n;i++) {
 927 | 		struct field *f = &st->f[i];
 928 | 		int type = f->type;
 929 | 		int value = 0;
 930 | 		int sz = -1;
 931 | 		args.tagname = f->name;
 932 | 		args.tagid = f->tag;
 933 | 		args.subtype = f->st;
 934 | 		args.mainindex = f->key;
 935 | 		args.extra = f->extra;
 936 | 		if (type & SPROTO_TARRAY) {
 937 | 			args.type = type & ~SPROTO_TARRAY;
 938 | 			sz = encode_array(cb, &args, data, size);
 939 | 		} else {
 940 | 			args.type = type;
 941 | 			args.index = 0;
 942 | 			switch(type) {
 943 | 			case SPROTO_TINTEGER:
 944 | 			case SPROTO_TBOOLEAN: {
 945 | 				union {
 946 | 					uint64_t u64;
 947 | 					uint32_t u32;
 948 | 				} u;
 949 | 				args.value = &u;
 950 | 				args.length = sizeof(u);
 951 | 				sz = cb(&args);
 952 | 				if (sz < 0) {
 953 | 					if (sz == SPROTO_CB_NIL)
 954 | 						continue;
 955 | 					if (sz == SPROTO_CB_NOARRAY)	// no array, don't encode it
 956 | 						return 0;
 957 | 					return -1;	// sz == SPROTO_CB_ERROR
 958 | 				}
 959 | 				if (sz == sizeof(uint32_t)) {
 960 | 					if (u.u32 < 0x7fff) {
 961 | 						value = (u.u32+1) * 2;
 962 | 						sz = 2; // sz can be any number > 0
 963 | 					} else {
 964 | 						sz = encode_integer(u.u32, data, size);
 965 | 					}
 966 | 				} else if (sz == sizeof(uint64_t)) {
 967 | 					sz= encode_uint64(u.u64, data, size);
 968 | 				} else {
 969 | 					return -1;
 970 | 				}
 971 | 				break;
 972 | 			}
 973 | 			case SPROTO_TSTRUCT:
 974 | 			case SPROTO_TSTRING:
 975 | 				sz = encode_object(cb, &args, data, size);
 976 | 				break;
 977 | 			}
 978 | 		}
 979 | 		if (sz < 0)
 980 | 			return -1;
 981 | 		if (sz > 0) {
 982 | 			uint8_t * record;
 983 | 			int tag;
 984 | 			if (value == 0) {
 985 | 				data += sz;
 986 | 				size -= sz;
 987 | 			}
 988 | 			record = header+SIZEOF_HEADER+SIZEOF_FIELD*index;
 989 | 			tag = f->tag - lasttag - 1;
 990 | 			if (tag > 0) {
 991 | 				// skip tag
 992 | 				tag = (tag - 1) * 2 + 1;
 993 | 				if (tag > 0xffff)
 994 | 					return -1;
 995 | 				record[0] = tag & 0xff;
 996 | 				record[1] = (tag >> 8) & 0xff;
 997 | 				++index;
 998 | 				record += SIZEOF_FIELD;
 999 | 			}
1000 | 			++index;
1001 | 			record[0] = value & 0xff;
1002 | 			record[1] = (value >> 8) & 0xff;
1003 | 			lasttag = f->tag;
1004 | 		}
1005 | 	}
1006 | 	header[0] = index & 0xff;
1007 | 	header[1] = (index >> 8) & 0xff;
1008 | 
1009 | 	datasz = data - (header + header_sz);
1010 | 	data = header + header_sz;
1011 | 	if (index != st->maxn) {
1012 | 		memmove(header + SIZEOF_HEADER + index * SIZEOF_FIELD, data, datasz);
1013 | 	}
1014 | 	return SIZEOF_HEADER + index * SIZEOF_FIELD + datasz;
1015 | }
1016 | 
1017 | static int
1018 | decode_array_object(sproto_callback cb, struct sproto_arg *args, uint8_t * stream, int sz) {
1019 | 	uint32_t hsz;
1020 | 	int index = 1;
1021 | 	while (sz > 0) {
1022 | 		if (sz < SIZEOF_LENGTH)
1023 | 			return -1;
1024 | 		hsz = todword(stream);
1025 | 		stream += SIZEOF_LENGTH;
1026 | 		sz -= SIZEOF_LENGTH;
1027 | 		if (hsz > sz)
1028 | 			return -1;
1029 | 		args->index = index;
1030 | 		args->value = stream;
1031 | 		args->length = hsz;
1032 | 		if (cb(args))
1033 | 			return -1;
1034 | 		sz -= hsz;
1035 | 		stream += hsz;
1036 | 		++index;
1037 | 	}
1038 | 	return 0;
1039 | }
1040 | 
1041 | static inline uint64_t
1042 | expand64(uint32_t v) {
1043 | 	uint64_t value = v;
1044 | 	if (value & 0x80000000) {
1045 | 		value |= (uint64_t)~0  << 32 ;
1046 | 	}
1047 | 	return value;
1048 | }
1049 | 
1050 | static int
1051 | decode_array(sproto_callback cb, struct sproto_arg *args, uint8_t * stream) {
1052 | 	uint32_t sz = todword(stream);
1053 | 	int type = args->type;
1054 | 	int i;
1055 | 	if (sz == 0) {
1056 | 		// It's empty array, call cb with index == -1 to create the empty array.
1057 | 		args->index = -1;
1058 | 		args->value = NULL;
1059 | 		args->length = 0;
1060 | 		cb(args);
1061 | 		return 0;
1062 | 	}	
1063 | 	stream += SIZEOF_LENGTH;
1064 | 	switch (type) {
1065 | 	case SPROTO_TINTEGER: {
1066 | 		int len = *stream;
1067 | 		++stream;
1068 | 		--sz;
1069 | 		if (len == sizeof(uint32_t)) {
1070 | 			if (sz % sizeof(uint32_t) != 0)
1071 | 				return -1;
1072 | 			for (i=0;i<sz/sizeof(uint32_t);i++) {
1073 | 				uint64_t value = expand64(todword(stream + i*sizeof(uint32_t)));
1074 | 				args->index = i+1;
1075 | 				args->value = &value;
1076 | 				args->length = sizeof(value);
1077 | 				cb(args);
1078 | 			}
1079 | 		} else if (len == sizeof(uint64_t)) {
1080 | 			if (sz % sizeof(uint64_t) != 0)
1081 | 				return -1;
1082 | 			for (i=0;i<sz/sizeof(uint64_t);i++) {
1083 | 				uint64_t low = todword(stream + i*sizeof(uint64_t));
1084 | 				uint64_t hi = todword(stream + i*sizeof(uint64_t) + sizeof(uint32_t));
1085 | 				uint64_t value = low | hi << 32;
1086 | 				args->index = i+1;
1087 | 				args->value = &value;
1088 | 				args->length = sizeof(value);
1089 | 				cb(args);
1090 | 			}
1091 | 		} else {
1092 | 			return -1;
1093 | 		}
1094 | 		break;
1095 | 	}
1096 | 	case SPROTO_TBOOLEAN:
1097 | 		for (i=0;i<sz;i++) {
1098 | 			uint64_t value = stream[i];
1099 | 			args->index = i+1;
1100 | 			args->value = &value;
1101 | 			args->length = sizeof(value);
1102 | 			cb(args);
1103 | 		}
1104 | 		break;
1105 | 	case SPROTO_TSTRING:
1106 | 	case SPROTO_TSTRUCT:
1107 | 		return decode_array_object(cb, args, stream, sz);
1108 | 	default:
1109 | 		return -1;
1110 | 	}
1111 | 	return 0;
1112 | }
1113 | 
1114 | int
1115 | sproto_decode(const struct sproto_type *st, const void * data, int size, sproto_callback cb, void *ud) {
1116 | 	struct sproto_arg args;
1117 | 	int total = size;
1118 | 	uint8_t * stream;
1119 | 	uint8_t * datastream;
1120 | 	int fn;
1121 | 	int i;
1122 | 	int tag;
1123 | 	if (size < SIZEOF_HEADER)
1124 | 		return -1;
1125 | 	// debug print
1126 | 	// printf("sproto_decode[%p] (%s)\n", ud, st->name);
1127 | 	stream = (void *)data;
1128 | 	fn = toword(stream);
1129 | 	stream += SIZEOF_HEADER;
1130 | 	size -= SIZEOF_HEADER ;
1131 | 	if (size < fn * SIZEOF_FIELD)
1132 | 		return -1;
1133 | 	datastream = stream + fn * SIZEOF_FIELD;
1134 | 	size -= fn * SIZEOF_FIELD;
1135 | 	args.ud = ud;
1136 | 
1137 | 	tag = -1;
1138 | 	for (i=0;i<fn;i++) {
1139 | 		uint8_t * currentdata;
1140 | 		struct field * f;
1141 | 		int value = toword(stream + i * SIZEOF_FIELD);
1142 | 		++ tag;
1143 | 		if (value & 1) {
1144 | 			tag += value/2;
1145 | 			continue;
1146 | 		}
1147 | 		value = value/2 - 1;
1148 | 		currentdata = datastream;
1149 | 		if (value < 0) {
1150 | 			uint32_t sz;
1151 | 			if (size < SIZEOF_LENGTH)
1152 | 				return -1;
1153 | 			sz = todword(datastream);
1154 | 			if (size < sz + SIZEOF_LENGTH)
1155 | 				return -1;
1156 | 			datastream += sz+SIZEOF_LENGTH;
1157 | 			size -= sz+SIZEOF_LENGTH;
1158 | 		}
1159 | 		f = findtag(st, tag);
1160 | 		if (f == NULL)
1161 | 			continue;
1162 | 		args.tagname = f->name;
1163 | 		args.tagid = f->tag;
1164 | 		args.type = f->type & ~SPROTO_TARRAY;
1165 | 		args.subtype = f->st;
1166 | 		args.index = 0;
1167 | 		args.mainindex = f->key;
1168 | 		args.extra = f->extra;
1169 | 		if (value < 0) {
1170 | 			if (f->type & SPROTO_TARRAY) {
1171 | 				if (decode_array(cb, &args, currentdata)) {
1172 | 					return -1;
1173 | 				}
1174 | 			} else {
1175 | 				switch (f->type) {
1176 | 				case SPROTO_TINTEGER: {
1177 | 					uint32_t sz = todword(currentdata);
1178 | 					if (sz == sizeof(uint32_t)) {
1179 | 						uint64_t v = expand64(todword(currentdata + SIZEOF_LENGTH));
1180 | 						args.value = &v;
1181 | 						args.length = sizeof(v);
1182 | 						cb(&args);
1183 | 					} else if (sz != sizeof(uint64_t)) {
1184 | 						return -1;
1185 | 					} else {
1186 | 						uint32_t low = todword(currentdata + SIZEOF_LENGTH);
1187 | 						uint32_t hi = todword(currentdata + SIZEOF_LENGTH + sizeof(uint32_t));
1188 | 						uint64_t v = (uint64_t)low | (uint64_t) hi << 32;
1189 | 						args.value = &v;
1190 | 						args.length = sizeof(v);
1191 | 						cb(&args);
1192 | 					}
1193 | 					break;
1194 | 				}
1195 | 				case SPROTO_TSTRING:
1196 | 				case SPROTO_TSTRUCT: {
1197 | 					uint32_t sz = todword(currentdata);
1198 | 					args.value = currentdata+SIZEOF_LENGTH;
1199 | 					args.length = sz;
1200 | 					if (cb(&args))
1201 | 						return -1;
1202 | 					break;
1203 | 				}
1204 | 				default:
1205 | 					return -1;
1206 | 				}
1207 | 			}
1208 | 		} else if (f->type != SPROTO_TINTEGER && f->type != SPROTO_TBOOLEAN) {
1209 | 			return -1;
1210 | 		} else {
1211 | 			uint64_t v = value;
1212 | 			args.value = &v;
1213 | 			args.length = sizeof(v);
1214 | 			cb(&args);
1215 | 		}
1216 | 	}
1217 | 	return total - size;
1218 | }
1219 | 
1220 | // 0 pack
1221 | 
1222 | static int
1223 | pack_seg(const uint8_t *src, uint8_t * buffer, int sz, int n) {
1224 | 	uint8_t header = 0;
1225 | 	int notzero = 0;
1226 | 	int i;
1227 | 	uint8_t * obuffer = buffer;
1228 | 	++buffer;
1229 | 	--sz;
1230 | 	if (sz < 0)
1231 | 		obuffer = NULL;
1232 | 
1233 | 	for (i=0;i<8;i++) {
1234 | 		if (src[i] != 0) {
1235 | 			notzero++;
1236 | 			header |= 1<<i;
1237 | 			if (sz > 0) {
1238 | 				*buffer = src[i];
1239 | 				++buffer;
1240 | 				--sz;
1241 | 			}
1242 | 		}
1243 | 	}
1244 | 	if ((notzero == 7 || notzero == 6) && n > 0) {
1245 | 		notzero = 8;
1246 | 	}
1247 | 	if (notzero == 8) {
1248 | 		if (n > 0) {
1249 | 			return 8;
1250 | 		} else {
1251 | 			return 10;
1252 | 		}
1253 | 	}
1254 | 	if (obuffer) {
1255 | 		*obuffer = header;
1256 | 	}
1257 | 	return notzero + 1;
1258 | }
1259 | 
1260 | static inline void
1261 | write_ff(const uint8_t * src, uint8_t * des, int n) {
1262 | 	int i;
1263 | 	int align8_n = (n+7)&(~7);
1264 | 
1265 | 	des[0] = 0xff;
1266 | 	des[1] = align8_n/8 - 1;
1267 | 	memcpy(des+2, src, n);
1268 | 	for(i=0; i< align8_n-n; i++){
1269 | 		des[n+2+i] = 0;
1270 | 	}
1271 | }
1272 | 
1273 | int
1274 | sproto_pack(const void * srcv, int srcsz, void * bufferv, int bufsz) {
1275 | 	uint8_t tmp[8];
1276 | 	int i;
1277 | 	const uint8_t * ff_srcstart = NULL;
1278 | 	uint8_t * ff_desstart = NULL;
1279 | 	int ff_n = 0;
1280 | 	int size = 0;
1281 | 	const uint8_t * src = srcv;
1282 | 	uint8_t * buffer = bufferv;
1283 | 	for (i=0;i<srcsz;i+=8) {
1284 | 		int n;
1285 | 		int padding = i+8 - srcsz;
1286 | 		if (padding > 0) {
1287 | 			int j;
1288 | 			memcpy(tmp, src, 8-padding);
1289 | 			for (j=0;j<padding;j++) {
1290 | 				tmp[7-j] = 0;
1291 | 			}
1292 | 			src = tmp;
1293 | 		}
1294 | 		n = pack_seg(src, buffer, bufsz, ff_n);
1295 | 		bufsz -= n;
1296 | 		if (n == 10) {
1297 | 			// first FF
1298 | 			ff_srcstart = src;
1299 | 			ff_desstart = buffer;
1300 | 			ff_n = 1;
1301 | 		} else if (n==8 && ff_n>0) {
1302 | 			++ff_n;
1303 | 			if (ff_n == 256) {
1304 | 				if (bufsz >= 0) {
1305 | 					write_ff(ff_srcstart, ff_desstart, 256*8);
1306 | 				}
1307 | 				ff_n = 0;
1308 | 			}
1309 | 		} else {
1310 | 			if (ff_n > 0) {
1311 | 				if (bufsz >= 0) {
1312 | 					write_ff(ff_srcstart, ff_desstart, ff_n*8);
1313 | 				}
1314 | 				ff_n = 0;
1315 | 			}
1316 | 		}
1317 | 		src += 8;
1318 | 		buffer += n;
1319 | 		size += n;
1320 | 	}
1321 | 	if(bufsz >= 0){
1322 | 		if(ff_n == 1)
1323 | 			write_ff(ff_srcstart, ff_desstart, 8);
1324 | 		else if (ff_n > 1)
1325 | 			write_ff(ff_srcstart, ff_desstart, srcsz - (intptr_t)(ff_srcstart - (const uint8_t*)srcv));
1326 | 	}
1327 | 	return size;
1328 | }
1329 | 
1330 | int
1331 | sproto_unpack(const void * srcv, int srcsz, void * bufferv, int bufsz) {
1332 | 	const uint8_t * src = srcv;
1333 | 	uint8_t * buffer = bufferv;
1334 | 	int size = 0;
1335 | 	while (srcsz > 0) {
1336 | 		uint8_t header = src[0];
1337 | 		--srcsz;
1338 | 		++src;
1339 | 		if (header == 0xff) {
1340 | 			int n;
1341 | 			if (srcsz < 0) {
1342 | 				return -1;
1343 | 			}
1344 | 			n = (src[0] + 1) * 8;
1345 | 			if (srcsz < n + 1)
1346 | 				return -1;
1347 | 			srcsz -= n + 1;
1348 | 			++src;
1349 | 			if (bufsz >= n) {
1350 | 				memcpy(buffer, src, n);
1351 | 			}
1352 | 			bufsz -= n;
1353 | 			buffer += n;
1354 | 			src += n;
1355 | 			size += n;
1356 | 		} else {
1357 | 			int i;
1358 | 			for (i=0;i<8;i++) {
1359 | 				int nz = (header >> i) & 1;
1360 | 				if (nz) {
1361 | 					if (srcsz < 0)
1362 | 						return -1;
1363 | 					if (bufsz > 0) {
1364 | 						*buffer = *src;
1365 | 						--bufsz;
1366 | 						++buffer;
1367 | 					}
1368 | 					++src;
1369 | 					--srcsz;
1370 | 				} else {
1371 | 					if (bufsz > 0) {
1372 | 						*buffer = 0;
1373 | 						--bufsz;
1374 | 						++buffer;
1375 | 					}
1376 | 				}
1377 | 				++size;
1378 | 			}
1379 | 		}
1380 | 	}
1381 | 	return size;
1382 | }
1383 | 


--------------------------------------------------------------------------------