├── .gitignore ├── 1.xml ├── wbxml.lua ├── wbxml.h ├── main.go └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | .idea 3 | -------------------------------------------------------------------------------- /1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | MIX 2 6 | 888833336669999 7 | MIX 2 8 | Android 8.0.0 9 | +8618599999999 10 | Android/8.0.0-EAS-1.3 11 | 中国联通 (46001) 12 | 13 | 14 | 15 | 16 | MS-EAS-Provisioning-WBXML 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /wbxml.lua: -------------------------------------------------------------------------------- 1 | -- ensure the lua2go lib is on the LUA_PATH so it will load 2 | -- normally, you'd just put it on the LUA_PATH 3 | package.path = package.path .. ';../lua/?.lua' 4 | 5 | -- load lua2go 6 | local lua2go = require('lua2go') 7 | 8 | -- load my Go library 9 | local example = lua2go.Load('/data/code/golang/src/dewbxml/wbxml.so') 10 | 11 | -- copy just the extern functions from benchmark.h into ffi.cdef structure below 12 | -- (the boilerplate cgo prologue is already defined for you in lua2go) 13 | -- this registers your Go functions to the ffi library.. 14 | lua2go.Externs[[ 15 | extern char* parse(GoString data); 16 | ]] 17 | 18 | local filename = "/data/code/golang/src/dewbxml/file.bin" 19 | local file = io.open(filename,"rb") 20 | local data = file:read("*a") 21 | local goResult = example.parse(lua2go.ToGo(data)) 22 | 23 | local Result = lua2go.ToLua(goResult) 24 | 25 | print('Result: ' .. Result) 26 | -------------------------------------------------------------------------------- /wbxml.h: -------------------------------------------------------------------------------- 1 | /* Created by "go tool cgo" - DO NOT EDIT. */ 2 | 3 | /* package command-line-arguments */ 4 | 5 | /* Start of preamble from import "C" comments. */ 6 | 7 | 8 | 9 | 10 | /* End of preamble from import "C" comments. */ 11 | 12 | 13 | /* Start of boilerplate cgo prologue. */ 14 | 15 | #ifndef GO_CGO_PROLOGUE_H 16 | #define GO_CGO_PROLOGUE_H 17 | 18 | typedef signed char GoInt8; 19 | typedef unsigned char GoUint8; 20 | typedef short GoInt16; 21 | typedef unsigned short GoUint16; 22 | typedef int GoInt32; 23 | typedef unsigned int GoUint32; 24 | typedef long long GoInt64; 25 | typedef unsigned long long GoUint64; 26 | typedef GoInt64 GoInt; 27 | typedef GoUint64 GoUint; 28 | typedef __SIZE_TYPE__ GoUintptr; 29 | typedef float GoFloat32; 30 | typedef double GoFloat64; 31 | typedef float _Complex GoComplex64; 32 | typedef double _Complex GoComplex128; 33 | 34 | /* 35 | static assertion to make sure the file is being used on architecture 36 | at least with matching size of GoInt. 37 | */ 38 | typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; 39 | 40 | typedef struct { const char *p; GoInt n; } GoString; 41 | typedef void *GoMap; 42 | typedef void *GoChan; 43 | typedef struct { void *t; void *v; } GoInterface; 44 | typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; 45 | 46 | #endif 47 | 48 | /* End of boilerplate cgo prologue. */ 49 | 50 | #ifdef __cplusplus 51 | extern "C" { 52 | #endif 53 | 54 | extern char* parse(GoString data); 55 | 56 | #ifdef __cplusplus 57 | } 58 | #endif -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (c) 2018 sec.lu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THEq 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | 23 | */ 24 | 25 | package main 26 | 27 | import ( 28 | "C" 29 | "bytes" 30 | "encoding/xml" 31 | "fmt" 32 | "os" 33 | "regexp" 34 | "strings" 35 | 36 | . "github.com/magicmonty/activesync-go/activesync" 37 | . "github.com/magicmonty/activesync-go/activesync/base" 38 | . "github.com/magicmonty/wbxml-go/wbxml" 39 | ) 40 | 41 | /* 42 | ``` 43 | 44 | 45 | 46 | 47 | MIX 2 48 | 888833336669999 49 | MIX 2 50 | Android 8.0.0 51 | +8618599999999 52 | Android/8.0.0-EAS-1.3 53 | 中国联通 (46001) 54 | 55 | 56 | 57 | 58 | MS-EAS-Provisioning-WBXML 59 | 60 | 61 | 62 | 63 | ``` 64 | */ 65 | type Provision struct { 66 | XMLName xml.Name `xml:"Provision"` 67 | Text string `xml:",chardata"` 68 | O string `xml:"O,attr"` 69 | S string `xml:"S,attr"` 70 | DeviceInformation struct { 71 | Text string `xml:",chardata"` 72 | Set struct { 73 | Text string `xml:",chardata"` 74 | Model string `xml:"Model"` 75 | IMEI string `xml:"IMEI"` 76 | FriendlyName string `xml:"FriendlyName"` 77 | OS string `xml:"OS"` 78 | PhoneNumber string `xml:"PhoneNumber"` 79 | UserAgent string `xml:"UserAgent"` 80 | MobileOperator string `xml:"MobileOperator"` 81 | } `xml:"Set"` 82 | } `xml:"DeviceInformation"` 83 | Policies struct { 84 | Text string `xml:",chardata"` 85 | Policy struct { 86 | Text string `xml:",chardata"` 87 | PolicyType string `xml:"PolicyType"` 88 | } `xml:"Policy"` 89 | } `xml:"Policies"` 90 | } 91 | 92 | func removeInvalidChars(b []byte) []byte { 93 | re := regexp.MustCompile("[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]") 94 | return re.ReplaceAll(b, []byte{}) 95 | } 96 | 97 | //export encodeXML 98 | func encodeXML(xmlString []byte) { 99 | xmlString = removeInvalidChars([]byte(xmlString)) 100 | w := bytes.NewBuffer(make([]byte, 0)) 101 | e := NewEncoder( 102 | MakeCodeBook(PROTOCOL_VERSION_14_1), 103 | string(xmlString), 104 | w) 105 | err := e.Encode() 106 | if err != nil { 107 | fmt.Println(err.Error()) 108 | } else { 109 | fmt.Println(w) 110 | } 111 | } 112 | func getDecodeResult(data ...byte) string { 113 | var result string 114 | result, _ = Decode(bytes.NewBuffer(data), MakeCodeBook(PROTOCOL_VERSION_14_1)) 115 | return result 116 | } 117 | 118 | //export parse 119 | func parse(data string) (*C.char) { 120 | result := make([]string, 0) 121 | xmldata := getDecodeResult([]byte(data)...) 122 | // fmt.Println(xmldata) 123 | 124 | out := Provision{} 125 | xml.Unmarshal([]byte(xmldata), &out) 126 | 127 | //fmt.Printf("Model: %v\n", out.DeviceInformation.Set.Model) 128 | //fmt.Printf("Imie: %v\n", out.DeviceInformation.Set.IMEI) 129 | //fmt.Printf("FriendlyName: %v\n", out.DeviceInformation.Set.FriendlyName) 130 | //fmt.Printf("PhoneNumber: %v\n", out.DeviceInformation.Set.PhoneNumber) 131 | //fmt.Printf("MobileOperator: %v\n", out.DeviceInformation.Set.MobileOperator) 132 | 133 | result = append(result, out.DeviceInformation.Set.Model) 134 | result = append(result, out.DeviceInformation.Set.IMEI) 135 | result = append(result, out.DeviceInformation.Set.FriendlyName) 136 | result = append(result, out.DeviceInformation.Set.PhoneNumber) 137 | result = append(result, out.DeviceInformation.Set.MobileOperator) 138 | 139 | return C.CString(strings.Join(result, "||")) 140 | } 141 | 142 | func main() { 143 | if len(os.Args) == 3 { 144 | cmd := os.Args[1] 145 | file := os.Args[2] 146 | f, err := os.Open(file) 147 | if err == nil { 148 | data := make([]byte, 102400) 149 | f.Read(data) 150 | 151 | if strings.ToLower(cmd) == "encode" { 152 | encodeXML(data) 153 | } else { 154 | parse(string(data)) 155 | } 156 | } 157 | } else { 158 | fmt.Printf("%s [encode|decode] filename\n", os.Args[0]) 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ```toml 2 | title = "用go语言给lua写扩展" 3 | slug = "extending-lua-with-go-language" 4 | desc = "extending-lua-with-go-language" 5 | date = "2018-11-27 00:14:21" 6 | update_date = "2018-11-27 00:14:21" 7 | author = "" 8 | thumb = "" 9 | draft = false 10 | tags = ["tag"] 11 | ``` 12 | 13 | ## 背景 14 | 最近的一个lua项目中需要解析wbxml，WBXML是XML的二进制表示形式，Exchange与手机端之间的通讯采用的就是该协议，我需要解析到手机端提交过来的数据，但lua没有现成的Wbxml解析库，从头撸一个势必要花费大量造轮子的时间，在网上查找了半天，发现有一个go语言版本的[https://github.com/magicmonty/activesync-go](https://github.com/magicmonty/activesync-go)，写了几行测试代码，确认该库可以正常解析出Exchange的wbxml数据内容，如下所示： 15 | 16 | ![](http://docs.xsec.io/images/wbxml/001.png) 17 | 18 | 19 | ## 微服务 VS lua 扩展 20 | 21 | 最初的方案打算用golang实现一个微服务，供openresty调用，该方案的特点是方便，能快速实现，但缺点也是非常明显的： 22 | 23 | - 性能损耗大：openresty每接收到一个请求都需要调用golang的restful api，然后等待golang把wbxml解析完并返回，这中间有非常大的性能损耗 24 | - 增加运维成本：golang微服务奔溃后，openresty将无法拿到想到的信息了，在运维时，除了要关注openresty本身外，还要时刻关注golang微服务的业务连续性、性能等指标 25 | 26 | 最佳的方案是提供一个lua的扩展，无缝集成到openresty中，这样可以完美地规避掉上述2个缺点。 27 | 28 | ## 用GO语言扩展lua 29 | 30 | ### 编写规范 31 | 关于用go语言扩展lua，github中已有现成的辅助库[https://github.com/theganyo/lua2go](https://github.com/theganyo/lua2go)可以使用，它的工作流程如下： 32 | 33 | 1. 编写go模块，并导出需要给lua使用的函数： 34 | 35 | ```golang 36 | //export add 37 | func add(operand1 int, operand2 int) int { 38 | return operand1 + operand2 39 | } 40 | ``` 41 | 1. 将go模块编译为静态库： 42 | 43 | ```golang 44 | go build -buildmode=c-shared -o example.so example.go 45 | ``` 46 | 1. 编写lua文件，加载自己的.so文件： 47 | 48 | ```lua 49 | local lua2go = require('lua2go') 50 | local example = lua2go.Load('./example.so') 51 | ``` 52 | 1. 在lua文件与头文件模块中注册导出的函数： 53 | 54 | ```lua 55 | lua2go.Externs[[ 56 | extern GoInt add(GoInt p0, GoInt p1); 57 | ]] 58 | ``` 59 | 1. 在lua文件中调用导出的函数并将结果转化为lua格式的数据： 60 | 61 | ```lua 62 | local goAddResult = example.add(1, 1) 63 | local addResult = lua2go.ToLua(goAddResult) 64 | print('1 + 1 = ' .. addResult) 65 | ``` 66 | 67 | 详细情况可以参考该项目的[example](https://github.com/theganyo/lua2go/tree/master/example) 68 | 69 | ### 编写自己的的wbxml解析库 70 | 71 | `getDecodeResult`函数可以将wbxml的二进制数据直接解析成xml格式的string 72 | ```golang 73 | func getDecodeResult(data ...byte) string { 74 | var result string 75 | result, _ = Decode(bytes.NewBuffer(data), MakeCodeBook(PROTOCOL_VERSION_14_1)) 76 | return result 77 | } 78 | ``` 79 | 80 | 但解析出来的xml的格式如下，多层嵌套且用了命名空间，虽然能看到明文的xml了，但是还是不能直接取到我们想要的数据 81 | ```xml 82 | 83 | 84 | 85 | 86 | MIX 2 87 | 888833336669999 88 | MIX 2 89 | Android 8.0.0 90 | +8618599999999 91 | Android/8.0.0-EAS-1.3 92 | 中国联通 (46001) 93 | 94 | 95 | 96 | 97 | MS-EAS-Provisioning-WBXML 98 | 99 | 100 | 101 | ``` 102 | 103 | 我们需要再对xml进行一次解析，解析到对应的struct中，就可以方便地获取想要的数据了，但是这个xml格式比较复杂，笔者试着手工定义了几次都失败了，干脆找了个自动化工具自动生成了，自动化工具的地址为[https://github.com/miku/zek](https://github.com/miku/zek)。 104 | 105 | 作者还提供了个Web版的在线工具，使用起来非常方便，地址为：[https://www.onlinetool.io/xmltogo/](https://www.onlinetool.io/xmltogo/) 106 | 107 | 最后生成的Struct如下： 108 | ```golang 109 | type Provision struct { 110 | XMLName xml.Name `xml:"Provision"` 111 | Text string `xml:",chardata"` 112 | O string `xml:"O,attr"` 113 | S string `xml:"S,attr"` 114 | DeviceInformation struct { 115 | Text string `xml:",chardata"` 116 | Set struct { 117 | Text string `xml:",chardata"` 118 | Model string `xml:"Model"` 119 | IMEI string `xml:"IMEI"` 120 | FriendlyName string `xml:"FriendlyName"` 121 | OS string `xml:"OS"` 122 | PhoneNumber string `xml:"PhoneNumber"` 123 | UserAgent string `xml:"UserAgent"` 124 | MobileOperator string `xml:"MobileOperator"` 125 | } `xml:"Set"` 126 | } `xml:"DeviceInformation"` 127 | Policies struct { 128 | Text string `xml:",chardata"` 129 | Policy struct { 130 | Text string `xml:",chardata"` 131 | PolicyType string `xml:"PolicyType"` 132 | } `xml:"Policy"` 133 | } `xml:"Policies"` 134 | } 135 | 136 | ``` 137 | 138 | 最终我们自己导出的处理wbxml的函数如下（将需要关注的信息放到一个用`||`分割的字符串中返回）: 139 | 140 | ```golang 141 | //export parse 142 | func parse(data []byte) (*C.char) { 143 | result := make([]string, 0) 144 | xmldata := getDecodeResult(data...) 145 | fmt.Println(xmldata) 146 | 147 | out := Provision{} 148 | xml.Unmarshal([]byte(xmldata), &out) 149 | 150 | //fmt.Printf("Model: %v\n", out.DeviceInformation.Set.Model) 151 | //fmt.Printf("Imie: %v\n", out.DeviceInformation.Set.IMEI) 152 | //fmt.Printf("FriendlyName: %v\n", out.DeviceInformation.Set.FriendlyName) 153 | //fmt.Printf("PhoneNumber: %v\n", out.DeviceInformation.Set.PhoneNumber) 154 | //fmt.Printf("MobileOperator: %v\n", out.DeviceInformation.Set.MobileOperator) 155 | 156 | result = append(result, out.DeviceInformation.Set.Model) 157 | result = append(result, out.DeviceInformation.Set.IMEI) 158 | result = append(result, out.DeviceInformation.Set.FriendlyName) 159 | result = append(result, out.DeviceInformation.Set.PhoneNumber) 160 | result = append(result, out.DeviceInformation.Set.MobileOperator) 161 | 162 | return C.CString(strings.Join(result, "||")) 163 | } 164 | ``` 165 | 166 | 接下来分别在`wbxml.h`和`xbxml/lua`中导出这个函数，如下所示： 167 | 168 | `wbxml.h`的内容： 169 | ```c 170 | #ifndef GO_CGO_PROLOGUE_H 171 | #define GO_CGO_PROLOGUE_H 172 | 173 | typedef signed char GoInt8; 174 | typedef unsigned char GoUint8; 175 | typedef short GoInt16; 176 | typedef unsigned short GoUint16; 177 | typedef int GoInt32; 178 | typedef unsigned int GoUint32; 179 | typedef long long GoInt64; 180 | typedef unsigned long long GoUint64; 181 | typedef GoInt64 GoInt; 182 | typedef GoUint64 GoUint; 183 | typedef __SIZE_TYPE__ GoUintptr; 184 | typedef float GoFloat32; 185 | typedef double GoFloat64; 186 | typedef float _Complex GoComplex64; 187 | typedef double _Complex GoComplex128; 188 | 189 | /* 190 | static assertion to make sure the file is being used on architecture 191 | at least with matching size of GoInt. 192 | */ 193 | typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; 194 | 195 | typedef struct { const char *p; GoInt n; } GoString; 196 | typedef void *GoMap; 197 | typedef void *GoChan; 198 | typedef struct { void *t; void *v; } GoInterface; 199 | typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; 200 | 201 | #endif 202 | 203 | /* End of boilerplate cgo prologue. */ 204 | 205 | #ifdef __cplusplus 206 | extern "C" { 207 | #endif 208 | 209 | extern char* parse(GoString data); 210 | 211 | #ifdef __cplusplus 212 | } 213 | #endif 214 | ``` 215 | 216 | `wbxml`的内容： 217 | 218 | ```lua 219 | -- ensure the lua2go lib is on the LUA_PATH so it will load 220 | -- normally, you'd just put it on the LUA_PATH 221 | package.path = package.path .. ';../lua/?.lua' 222 | 223 | -- load lua2go 224 | local lua2go = require('lua2go') 225 | 226 | -- load my Go library 227 | local example = lua2go.Load('/data/code/golang/src/dewbxml/wbxml.so') 228 | 229 | -- copy just the extern functions from benchmark.h into ffi.cdef structure below 230 | -- (the boilerplate cgo prologue is already defined for you in lua2go) 231 | -- this registers your Go functions to the ffi library.. 232 | lua2go.Externs[[ 233 | extern char* parse(GoString data); 234 | ]] 235 | 236 | local filename = "/data/code/golang/src/dewbxml/file.bin" 237 | local file = io.open(filename,"rb") 238 | local data = file:read("*a") 239 | local goResult = example.parse(lua2go.ToGo(data)) 240 | 241 | local Result = lua2go.ToLua(goResult) 242 | 243 | print('Result: ' .. Result) 244 | ``` 245 | 最终的结果如下图所示： 246 | ![](http://docs.xsec.io/images/wbxml/002.png) 247 | 248 | 造化弄人，在不经意间，还是造了一个轮子，项目地址为：[https://github.com/netxfly/dewbxml](https://github.com/netxfly/dewbxml) 249 | --------------------------------------------------------------------------------