├── .github ├── dependabot.yml └── workflows │ └── go.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── README_BPL.md ├── array.go ├── base_array.go ├── base_type.go ├── base_type_test.go ├── binary └── format.go ├── bpl.ext ├── bpl.go ├── bpl_test.go ├── bson │ ├── bson.go │ └── bson_test.go ├── cl.go ├── cl_ext.go ├── cl_ext_test.go ├── qlang.go └── ruler.go ├── cmd ├── hexundump │ └── hex_undump.go ├── qbpl │ └── qbpl.go ├── qbplgen │ └── qbplgen.go ├── qbplproxy │ └── qbplproxy.go ├── qmockd │ └── qmockd.go └── qreplay │ └── qreplay.go ├── compose.go ├── context.go ├── formats ├── 1.gif ├── 1935.bpl ├── 27017.bpl ├── 80.bpl ├── flv.bpl ├── format-mp4.jpg ├── gif.bpl ├── http.bpl ├── lines.bpl ├── mongo.bpl ├── mp4.bpl ├── rtmp.bpl ├── rtmp_hexdump.bpl ├── ts.bpl ├── wav.bpl └── webrtc.bpl ├── go.mod ├── go.sum ├── go └── codegen │ ├── bytes.go │ └── bytes_test.go ├── hex └── undump.go ├── mockd └── mockd.go ├── repeat.go ├── replay └── replay.go ├── struct.go └── struct_test.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "daily" 12 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v3 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v3 18 | with: 19 | go-version: 1.16 20 | 21 | - name: Build 22 | run: go build -v ./... 23 | 24 | - name: Test 25 | run: go test -v -coverprofile="coverage.txt" -covermode=atomic ./... 26 | 27 | - name: Codecov 28 | uses: codecov/codecov-action@v3 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | *.log 6 | *.dmg 7 | 8 | # Folders 9 | _obj 10 | _test 11 | 12 | pkg/ 13 | bin/ 14 | 15 | qmongoproxy/ 16 | qrtmpproxy/ 17 | qgif/ 18 | 19 | # Architecture specific extensions/prefixes 20 | *.[568vq] 21 | [568vq].out 22 | 23 | *.cgo1.go 24 | *.cgo2.c 25 | _cgo_defun.c 26 | _cgo_gotypes.go 27 | _cgo_export.* 28 | 29 | _testmain.go 30 | 31 | *.exe 32 | *.test 33 | *.prof 34 | coverage.txt 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | go install -v ./... 3 | 4 | rebuild: 5 | go install -a -v ./... 6 | 7 | install: all 8 | @mkdir -p ~/.qbpl/formats 9 | cp formats/*.bpl ~/.qbpl/formats/ 10 | 11 | test: 12 | go test ./... 13 | 14 | testv: 15 | go test -v ./... 16 | 17 | clean: 18 | go clean -i ./... 19 | 20 | fmt: 21 | gofmt -w=true ./ 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | BPL - Binary Processing Language 2 | ============ 3 | 4 | [![Build Status](https://github.com/goplus/bpl/actions/workflows/go.yml/badge.svg)](https://github.com/goplus/bpl/actions/workflows/go.yml) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/goplus/bpl)](https://goreportcard.com/report/github.com/goplus/bpl) 6 | [![GitHub release](https://img.shields.io/github/v/tag/goplus/bpl.svg?label=release)](https://github.com/goplus/bpl/releases) 7 | [![Coverage Status](https://codecov.io/gh/goplus/bpl/branch/master/graph/badge.svg)](https://codecov.io/gh/goplus/bpl) 8 | [![GoDoc](https://pkg.go.dev/badge/github.com/goplus/bpl.svg)](https://pkg.go.dev/mod/github.com/goplus/bpl) 9 | 10 | ## 快速入门 11 | 12 | 了解 BPL 最快的方式是学习 qbpl 和 qbplproxy 两个实用程序： 13 | 14 | ### qbpl 15 | 16 | qbpl 可用来分析任意的文件格式。使用方法如下： 17 | 18 | ``` 19 | qbpl [-p .bpl -o .log] 20 | ``` 21 | 22 | 多数情况下，你不需要指定 `-p .bpl` 参数，我们根据文件后缀来确定应该使用何种 protocol 来解析这个文件。例如： 23 | 24 | ``` 25 | qbpl 1.gif 26 | ``` 27 | 28 | 不过为了让 qbpl 能够找到所有的 protocols，我们需要先安装： 29 | 30 | ``` 31 | make install # 这将将所有的bpl文件拷贝到 ~/.qbpl/formats/ 32 | ``` 33 | 34 | ### qbplproxy 35 | 36 | qbplproxy 可用来分析服务器和客户端之间的网络包。它通过代理要分析的服务，让客户端请求自己来分析请求包和返回包。使用方式如下： 37 | 38 | ``` 39 | qbplproxy -h -b [-p .bpl -f -o .log] 40 | ``` 41 | 42 | 其中，`` 是 qbplproxy 自身监听的IP和端口，`` 是原始的服务。`-f ` 是过滤条件，这个条件通过 BPL_FILTER 全局变量传递到 bpl 中。 43 | 44 | 多数情况下，你不需要指定 `-p .bpl` 参数，qbplproxy 程序可以根据你监听的端口来猜测网络协议。例如： 45 | 46 | ``` 47 | mongod --port 37017 48 | qbplproxy -h localhost:27017 -b localhost:37017 49 | ``` 50 | 51 | 我们会依据端口 27017 知道你要分析的是 mongodb 的网络协议。 52 | 53 | 54 | ## BPL 文法 55 | 56 | 请参见 [BPL 文法](README_BPL.md)。 57 | 58 | 59 | ## 网络协议研究 60 | 61 | ### RTMP 协议 62 | 63 | 格式描述： 64 | 65 | * [rtmp.bpl](formats/rtmp.bpl) 66 | 67 | 测试： 68 | 69 | 1) 启动一个 rtmp server，让其监听 1936 端口（而不是默认的 1935 端口）。比如我们可以用 [node-rtsp-rtmp-server](https://github.com/iizukanao/node-rtsp-rtmp-server)： 70 | 71 | ``` 72 | git clone git@github.com:iizukanao/node-rtsp-rtmp-server.git 73 | cd node-rtsp-rtmp-server 74 | 修改 config.coffee，将： 75 | * rtmpServerPort: 1935 改为 rtmpServerPort: 1936； 76 | * serverPort: 80 改为 serverPort: 8080（这样就不用 sudo 来运行了） 77 | coffee server.coffee 78 | ``` 79 | 80 | 2) 启动 qbplproxy： 81 | 82 | ``` 83 | qbplproxy -h localhost:1935 -b localhost:1936 -p formats/rtmp.bpl | tee rtmp.log 84 | ``` 85 | 86 | 3) 推流： 87 | 88 | ``` 89 | ffmpeg -re -i test.m4v -c:v copy -c:a copy -f flv rtmp://localhost/live/123 90 | ``` 91 | 92 | 4) 播流： 93 | 94 | 在 Mac 下可以考虑用 VLC Player，打开网址 rtmp://localhost/live/123 进行播放即可。 95 | 96 | 5) 选择性查看 97 | 98 | 有时候我们并不希望看到所有的信息，rtmp.bpl 支持以 flashVer 作为过滤条件。如： 99 | 100 | ``` 101 | qbplproxy -f 'flashVer=LNX 9,0,124,2' -h localhost:1935 -b localhost:1936 -p formats/rtmp.bpl | tee .log 102 | ``` 103 | 104 | 或者我们直接用 reqMode(用来区分是推流publish还是播流play) 来过滤。如： 105 | 106 | ``` 107 | qbplproxy -f 'reqMode=play' -h localhost:1935 -b localhost:1936 -p formats/rtmp.bpl | tee .log 108 | ``` 109 | 110 | 这样就可以只捕获 VLC Player 的播流过程了。 111 | 112 | 当然，其实还有一个不用过滤条件的办法：就是让推流直接推到 rtmp server，但是播流请求发到 qbplproxy。 113 | 114 | 115 | ### FLV 协议 116 | 117 | 格式描述： 118 | 119 | * [flv.bpl](formats/flv.bpl) 120 | 121 | 测试： 122 | 123 | 1) 启动一个 rtmp/flv server，让其监听 1935/8135 端口。 124 | 125 | 2) 启动 qbplproxy： 126 | 127 | ``` 128 | qbplproxy -h localhost:8888 -b localhost:8135 -p formats/flv.bpl | tee flv.log 129 | ``` 130 | 131 | 3) 推流： 132 | 133 | ``` 134 | ffmpeg -re -i test.m4v -c:v copy -c:a copy -f flv rtmp://localhost/live/123 135 | ``` 136 | 137 | 4) 播流： 138 | 139 | 在 Mac 下可以考虑用 VLC Player，打开网址 http://localhost:8888/live/123.flv 进行播放即可。 140 | 141 | 142 | ### WebRTC 协议 143 | 144 | 格式描述： 145 | 146 | * [webrtc.bpl](formats/webrtc.bpl) 147 | 148 | 149 | ### MongoDB 协议 150 | 151 | 格式描述： 152 | 153 | * [mongo.bpl](formats/mongo.bpl) 154 | 155 | 测试： 156 | 157 | 1) 启动 MongoDB，让其监听 37017 端口（而不是默认的 27017 端口）： 158 | 159 | ``` 160 | ./mongod --port 37017 --dbpath ~/data/db 161 | ``` 162 | 163 | 2) 启动 qbplproxy： 164 | 165 | ``` 166 | qbplproxy -h localhost:27017 -b localhost:37017 -p formats/mongo.bpl | tee mongo.log 167 | ``` 168 | 169 | 3) 使用 MongoDB，比如通过 mongo shell 操作： 170 | 171 | ``` 172 | ./mongo 173 | ``` 174 | 175 | ## 文件格式研究 176 | 177 | ### MongoDB binlog 格式 178 | 179 | TODO 180 | 181 | ### MySQL binlog 格式 182 | 183 | TODO 184 | 185 | ### HLS TS 格式 186 | 187 | 格式描述： 188 | 189 | * [ts.bpl](formats/ts.bpl) 190 | 191 | 测试：TODO 192 | 193 | ### FLV 格式 194 | 195 | 格式描述： 196 | 197 | * [flv.bpl](formats/flv.bpl) 198 | 199 | 测试：TODO 200 | 201 | ### MP4 格式 202 | 203 | 格式描述： 204 | 205 | * [mp4.bpl](formats/mp4.bpl) 206 | 207 | 测试： 208 | 209 | ``` 210 | qbpl -p formats/mp4.bpl .mp4 211 | ``` 212 | 213 | ### GIF 格式 214 | 215 | 格式描述： 216 | 217 | * [gif.bpl](formats/gif.bpl) 218 | 219 | 测试： 220 | 221 | ``` 222 | qbpl -p formats/gif.bpl formats/1.gif 223 | ``` 224 | -------------------------------------------------------------------------------- /README_BPL.md: -------------------------------------------------------------------------------- 1 | BPL 文法 2 | ============ 3 | 4 | ## 基础规则 5 | 6 | 其实也就是内建类型 (builtin types)，如下： 7 | 8 | * int8, char, uint8(byte), int16, uint16 9 | * uint24, int32, uint32, int64, uint64 10 | * uint16be, uint24be, uint32be, uint64be 11 | * uint16le, uint24le, uint32le, uint64le (实际上就是 uint16, uint24, uint32, uint64，这里只是强调下 LittleEndian) 12 | * float32, float64, float32le, float64le, float32be, float64be 13 | * cstring, [n]char 14 | * bson 15 | * nil 16 | 17 | 18 | ## 复合规则 19 | 20 | * `*R`: 反复匹配规则 R，直到无法成功匹配为止。 21 | * `+R`: 反复匹配规则 R，直到无法成功匹配为止。要求至少匹配成功1次。 22 | * `?R`: 匹配规则 R 1次或0次。 23 | * `R1 R2 ... Rn`: 要求要匹配的文本满足规则序列 R1 R2 ... Rn。 24 | 25 | 26 | ## 别名 27 | 28 | 如果规则太复杂并且要出现在多个地方，那么我们就可以定义下别名。例如： 29 | 30 | ``` 31 | R = R1 R2 ... Rn 32 | ``` 33 | 34 | 这里 R 就是规则序列 `R1 R2 ... Rn` 的别名。 35 | 36 | 37 | ## 结构体 38 | 39 | 结构体本质上和 `R1 R2 ... Rn` 有些类似，属于规则序列，但是它给每个子规则都有命名，如下： 40 | 41 | ``` 42 | { 43 | Var1 R1 44 | Var2 R2 45 | ... 46 | Varn Rn 47 | } 48 | ``` 49 | 50 | 以上是 Go 风格的结构体。我们也可以改成 C 风格的： 51 | 52 | ``` 53 | {/C 54 | R1 Var1 55 | R2 Var2 56 | ... 57 | Rn Varn 58 | } 59 | ``` 60 | 61 | 在结构体里面出现的规则 R，有一些特殊性，如下： 62 | 63 | 1) 规则不能太复杂，如果复杂应该定义别名。例如以下是非法的： 64 | 65 | ``` 66 | { 67 | Var (R1 R2 ... Rn) 68 | } 69 | ``` 70 | 71 | 我们不建议写这样不清晰的东西，而是先给 `R1 R2 ... Rn` 定义别名： 72 | 73 | ``` 74 | R = R1 R2 ... Rn 75 | ``` 76 | 77 | 然后再引用 R： 78 | 79 | ``` 80 | { 81 | Var R 82 | } 83 | ``` 84 | 85 | 2) 以尽可能符合常理为原则，我们设置了规则可以是 `R`, `?R`, `*R`, `+R`, `[len]R` 这样一些情况。当然在 C 风格中应该是 `R`, `R?`, `R*`, `R+`, `R[len]` 这种结构。 86 | 87 | 3) 需要注意的一个细节是，`[len]R` 这样的规则当前只能在结构体里面出现。 88 | 89 | 90 | ## 捕获 91 | 92 | 如果我们希望生成 DOM，那么我们就需要去捕获感兴趣的数据。例如： 93 | 94 | ``` 95 | doc = R1 R2 R3 ... Rn 96 | ``` 97 | 98 | 对于这样一段数据，如果我们感兴趣 R2，那么可以： 99 | 100 | ``` 101 | doc = R1 $R2 R3 ... Rn 102 | ``` 103 | 104 | 但是如果你感兴趣多个元素，你不能多处使用 `$` 来捕获。下面样例不能正常工作： 105 | 106 | ``` 107 | doc = R1 $R2 R3 ... $Rn 108 | ``` 109 | 110 | 因为这段规则的匹配过程是这样的： 111 | 112 | * 在匹配 `$R2` 成功后，我们把 ctx.dom = ``。 113 | * 在匹配 `$Rn` 成功后，我们试图把 ctx.dom 修改为 `` 但是失败，因为 ctx.dom 已经赋值。 114 | 115 | 如果我们对多个匹配的结果感兴趣，那么我们需要写成： 116 | 117 | ``` 118 | doc = R1 [R2] R3 ... [Rn] 119 | ``` 120 | 121 | 得到的结果是 `[, ]`。 122 | 123 | 如果我们希望结果不是数组而是对象(object)。那么应该用结构体。例如： 124 | 125 | ``` 126 | doc = R1 {var2 R2; var3 R3} ... {varn Rn} 127 | ``` 128 | 129 | 那么结果就是 `{"var2": , "var3": , "varn": }`。 130 | 131 | 如果我们这样去捕获： 132 | 133 | ``` 134 | Rx = [R2 R3] 135 | 136 | doc = R1 {varx Rx} ... {varn Rn} 137 | ``` 138 | 139 | 那么结果就是 `{"varx": [, ], "varn": }`。 140 | 141 | 142 | ## dump 143 | 144 | dump 规则不匹配任何内容，但是会打印当前 Context 中已经捕获的所有变量值。如： 145 | 146 | doc = *(record dump) 147 | 148 | 这样每个 record 匹配成功后会 dump 匹配结果。如果希望某个变量不进行 dump，则该变量需要以 _ 开头。 149 | 150 | 另外，dump 也可以用于结构体中。如： 151 | 152 | ``` 153 | { 154 | Var1 R1 155 | Var2 R2 156 | ... 157 | Varn Rn 158 | dump 159 | } 160 | ``` 161 | 162 | ## case 163 | 164 | ``` 165 | case { 166 | : R1 167 | : R2 168 | ... 169 | : Rn 170 | default: Rdefault // 如果没有 default 并且前面各个分支都没有匹配成功，那么整个规则匹配会失败 171 | } 172 | ``` 173 | 174 | 典型例子： 175 | 176 | ``` 177 | header = {type uint32; ...} 178 | 179 | body1 = {...} 180 | 181 | body2 = {...} 182 | 183 | // 每个记录有个 header，header里面有个记录type，不同记录有不同的body 184 | // 185 | record = {h header} case h.type {type1: body1; type2: body2; ...} 186 | 187 | doc = *record 188 | ``` 189 | 190 | 另外条件规则也可以出现在结构体中（下文大部分规则除非特殊说明，一般都可以同时出现在规则列表和结构体）。如： 191 | 192 | ``` 193 | record = { 194 | h header 195 | case h.type { 196 | type1: body1 197 | type2: body2 198 | ... 199 | } 200 | } 201 | ``` 202 | 203 | ## if..elif..else 204 | 205 | ``` 206 | if do R1 elif do R2 ... else Rn 207 | ``` 208 | 209 | 对 `` 进行求值，如果结果为 true 或非零整数则执行 R1 规则，以此类推。如果 R1 是结构体 { ... }，则 do 可以忽略。例如： 210 | 211 | ``` 212 | record = { 213 | len uint32 214 | if len { 215 | data [len]byte 216 | next record 217 | } 218 | } 219 | ``` 220 | 221 | ## assert 222 | 223 | ``` 224 | assert 225 | ``` 226 | 227 | 对 `` 进行求值，如果结果为 true 或非零整数表示成功，其他情况均失败。 228 | 229 | ## fatal 230 | 231 | ``` 232 | fatal 233 | ``` 234 | 235 | 对 `` 进行求值，要求返回 string 类型，并以此结果作为错误提示匹配失败。 236 | 237 | ## skip 238 | 239 | ``` 240 | skip 241 | ``` 242 | 243 | 这里 `` 是一个 qlang 表达式。对 `` 求值，然后跳过相应字节数的内容。 244 | 245 | ``` 246 | record = { 247 | h header 248 | skip h.len - sizeof(header) 249 | } 250 | ``` 251 | 252 | 这通常发生在这个要读取的记录太大，但是内容又不感兴趣。 253 | 254 | ## read..do 255 | 256 | ``` 257 | read do R 258 | ``` 259 | 260 | 这里 `` 是一个 qlang 表达式。对 `` 求值，读如相应字节数的内容后，再用 R 匹配这段内容。如： 261 | 262 | ``` 263 | record = { 264 | h header 265 | read h.len - sizeof(header) do case h.type { 266 | type1: body1 267 | type2: body2 268 | ... 269 | } 270 | } 271 | ``` 272 | 273 | ## eval..do 274 | 275 | ``` 276 | eval do R 277 | ``` 278 | 279 | 对 `` 进行求值（要求求值结果为[]byte类型或者io.Reader类型）后，再用 R 匹配它。如： 280 | 281 | ``` 282 | record = { 283 | h header 284 | body [h.len - sizeof(header)]byte 285 | eval body do case h.type { 286 | type1: body1 287 | type2: body2 288 | ... 289 | } 290 | } 291 | ``` 292 | 293 | ## let 294 | 295 | ``` 296 | let = 297 | let , , ..., = 298 | ``` 299 | 300 | 如果 `` 是全局变量（见后面 `global` 一节），则修改该全局变量的值为 `` 。 301 | 如果当前 Context 已经存在名为 `` 的变量，则修改其值为 ``；否则新增名为 `` 的变量。 302 | 303 | ``` 304 | record = { 305 | let a = 1 306 | let _b = 2 // 变量 _b 不会被 dump 307 | } 308 | ``` 309 | 310 | ## return 311 | 312 | return 语句只能出现在结构体中，用来改写结构体的匹配结果。如： 313 | 314 | ``` 315 | uint24be = { 316 | b3 uint8 317 | b2 uint8 318 | b1 uint8 319 | return (b3 << 16) | (b2 << 8) | b1 320 | } 321 | ``` 322 | 323 | 在正常情况下，以上 uint24be 的匹配结果应该是 `{"b1": , "b2": , "b3": }`，但是由于 return 语句的存在，其匹配结果变成返回一个整数。类似地我们可以有： 324 | 325 | ``` 326 | uint32be = { 327 | b4 uint8 328 | b3 uint8 329 | b2 uint8 330 | b1 uint8 331 | return (b4 << 24) | (b3 << 16) | (b2 << 8) | b1 332 | } 333 | ``` 334 | 335 | 当然，在 bpl 里面已经内置了 uint24be 和 uint32be。这里仅仅只是演示如何基于 bpl 本身来自己实现这样的内置规则。 336 | 337 | ## global 338 | 339 | ``` 340 | global = 341 | ``` 342 | 343 | global 语句用于定义全局变量，这些变量并不出现在 Context 的捕获结果中。如： 344 | 345 | ``` 346 | init = { 347 | global msgs = mkmap("int:var") 348 | } 349 | ``` 350 | 351 | ## do 352 | 353 | ``` 354 | do 355 | ``` 356 | 357 | do 语句执行一个 qlang 表达式。如： 358 | 359 | ``` 360 | init = { 361 | global msgs = {"a": 12, "b": 32} 362 | } 363 | 364 | record = { 365 | do set(msgs, "c", 56, "d", 78) // 现在 msgs = {"a": 12, "b": 32, "c": 56, "d": 78} 366 | do println("msgs:", msgs) 367 | let a = msgs["a"] 368 | } 369 | 370 | doc = init record 371 | ``` 372 | 373 | ## 常量 374 | 375 | ``` 376 | const ( 377 | = 378 | ... 379 | ) 380 | ``` 381 | 382 | 语法和 Go 语言类似。例如： 383 | 384 | ``` 385 | const ( 386 | N = 6 387 | ) 388 | 389 | record = { 390 | tag [N]char 391 | assert tag == "GIF87a" || tag == "GIF89a" 392 | } 393 | ``` 394 | 395 | ## qlang 表达式 396 | 397 | bpl 集成了 qlang 表达式（不包含赋值）。以上所有 ``、``、`` 这些地方，都是 bpl 引用 qlang 表达式的地方。 398 | 399 | bpl 中的 qlang 表达式支持如下这些特性： 400 | 401 | * 所有 qlang 操作符； 402 | * string、slice、map 等内置类型； 403 | * 变量、成员变量引用； 404 | * 函数、成员函数调用； 405 | * 模块（但是我们很克制地支持了非常有限的几个模块，如：builtin、bytes 等）； 406 | 407 | 408 | ## 样例：MongoDB 网络协议 409 | 410 | ``` 411 | document = bson 412 | 413 | MsgHeader = {/C 414 | int32 messageLength; // total message size, including this 415 | int32 requestID; // identifier for this message 416 | int32 responseTo; // requestID from the original request (used in responses from db) 417 | int32 opCode; // request type - see table below 418 | } 419 | 420 | OP_UPDATE = {/C 421 | int32 ZERO; // 0 - reserved for future use 422 | cstring fullCollectionName; // "dbname.collectionname" 423 | int32 flags; // bit vector. see below 424 | document selector; // the query to select the document 425 | document update; // specification of the update to perform 426 | } 427 | 428 | OP_INSERT = {/C 429 | int32 flags; // bit vector - see below 430 | cstring fullCollectionName; // "dbname.collectionname" 431 | document* documents; // one or more documents to insert into the collection 432 | } 433 | 434 | OP_QUERY = {/C 435 | int32 flags; // bit vector of query options. See below for details. 436 | cstring fullCollectionName; // "dbname.collectionname" 437 | int32 numberToSkip; // number of documents to skip 438 | int32 numberToReturn; // number of documents to return 439 | // in the first OP_REPLY batch 440 | document query; // query object. See below for details. 441 | document? returnFieldsSelector; // Optional. Selector indicating the fields 442 | // to return. See below for details. 443 | } 444 | 445 | OP_GET_MORE = {/C 446 | int32 ZERO; // 0 - reserved for future use 447 | cstring fullCollectionName; // "dbname.collectionname" 448 | int32 numberToReturn; // number of documents to return 449 | int64 cursorID; // cursorID from the OP_REPLY 450 | } 451 | 452 | OP_DELETE = {/C 453 | int32 ZERO; // 0 - reserved for future use 454 | cstring fullCollectionName; // "dbname.collectionname" 455 | int32 flags; // bit vector - see below for details. 456 | document selector; // query object. See below for details. 457 | } 458 | 459 | OP_KILL_CURSORS = {/C 460 | int32 ZERO; // 0 - reserved for future use 461 | int32 numberOfCursorIDs; // number of cursorIDs in message 462 | int64* cursorIDs; // sequence of cursorIDs to close 463 | } 464 | 465 | OP_MSG = {/C 466 | cstring message; // message for the database 467 | } 468 | 469 | OP_REPLY = {/C 470 | int32 responseFlags; // bit vector - see details below 471 | int64 cursorID; // cursor id if client needs to do get more's 472 | int32 startingFrom; // where in the cursor this reply is starting 473 | int32 numberReturned; // number of documents in the reply 474 | document* documents; // documents 475 | } 476 | 477 | Message = { 478 | header MsgHeader // standard message header 479 | body [header.messageLength - sizeof(MsgHeader)]byte 480 | eval body do case header.opCode { 481 | 1: OP_REPLY // Reply to a client request. responseTo is set. 482 | 1000: OP_MSG // Generic msg command followed by a string. 483 | 2001: OP_UPDATE 484 | 2002: OP_INSERT 485 | 2004: OP_QUERY 486 | 2005: OP_GET_MORE // Get more data from a query. See Cursors. 487 | 2006: OP_DELETE 488 | 2007: OP_KILL_CURSORS // Notify database that the client has finished with the cursor. 489 | default: nil 490 | } 491 | } 492 | 493 | doc = *Message 494 | ``` 495 | -------------------------------------------------------------------------------- /array.go: -------------------------------------------------------------------------------- 1 | package bpl 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "reflect" 7 | ) 8 | 9 | // ----------------------------------------------------------------------------- 10 | 11 | func valueOf(v interface{}, t reflect.Type) reflect.Value { 12 | 13 | if v != nil { 14 | return reflect.ValueOf(v) 15 | } 16 | return reflect.Zero(t) 17 | } 18 | 19 | func matchArray1(R Ruler, in *bufio.Reader, ctx *Context, fCheckNil bool) (v interface{}, err error) { 20 | 21 | t := R.RetType() 22 | ret := reflect.MakeSlice(reflect.SliceOf(t), 0, 4) 23 | for { 24 | _, err = in.Peek(1) 25 | if err != nil { 26 | if err == io.EOF { 27 | if fCheckNil { 28 | return 29 | } 30 | return ret.Interface(), nil 31 | } 32 | return 33 | } 34 | v, err = R.Match(in, ctx.NewSub()) 35 | if err != nil { 36 | return 37 | } 38 | ret = reflect.Append(ret, valueOf(v, t)) 39 | fCheckNil = false 40 | } 41 | } 42 | 43 | func matchArray(R Ruler, n int, in *bufio.Reader, ctx *Context) (v interface{}, err error) { 44 | 45 | if n == 0 { 46 | return 47 | } 48 | 49 | t := R.RetType() 50 | ret := reflect.MakeSlice(reflect.SliceOf(t), 0, n) 51 | for i := 0; i < n; i++ { 52 | v, err = R.Match(in, ctx.NewSub()) 53 | if err != nil { 54 | return 55 | } 56 | ret = reflect.Append(ret, valueOf(v, t)) 57 | } 58 | return ret.Interface(), nil 59 | } 60 | 61 | // ----------------------------------------------------------------------------- 62 | 63 | type array1 struct { 64 | r Ruler 65 | } 66 | 67 | func (p *array1) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 68 | 69 | return matchArray1(p.r, in, ctx, true) 70 | } 71 | 72 | func (p *array1) RetType() reflect.Type { 73 | 74 | return reflect.SliceOf(p.r.RetType()) 75 | } 76 | 77 | func (p *array1) SizeOf() int { 78 | 79 | return -1 80 | } 81 | 82 | // Array1 returns a matching unit that matches R+ 83 | // 84 | func Array1(R Ruler) Ruler { 85 | 86 | if R == Uint8 { 87 | return ByteArray1 88 | } 89 | return &array1{r: R} 90 | } 91 | 92 | // ----------------------------------------------------------------------------- 93 | 94 | type array0 struct { 95 | r Ruler 96 | } 97 | 98 | func (p *array0) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 99 | 100 | return matchArray1(p.r, in, ctx, false) 101 | } 102 | 103 | func (p *array0) RetType() reflect.Type { 104 | 105 | return reflect.SliceOf(p.r.RetType()) 106 | } 107 | 108 | func (p *array0) SizeOf() int { 109 | 110 | return -1 111 | } 112 | 113 | // Array0 returns a matching unit that matches R* 114 | // 115 | func Array0(R Ruler) Ruler { 116 | 117 | if R == Uint8 { 118 | return ByteArray0 119 | } 120 | return &array0{r: R} 121 | } 122 | 123 | // Array01 returns a matching unit that matches R? 124 | // 125 | func Array01(R Ruler) Ruler { 126 | 127 | return Repeat01(R) 128 | } 129 | 130 | // ----------------------------------------------------------------------------- 131 | 132 | type array struct { 133 | r Ruler 134 | n int 135 | } 136 | 137 | func (p *array) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 138 | 139 | n := p.n 140 | return matchArray(p.r, n, in, ctx) 141 | } 142 | 143 | func (p *array) RetType() reflect.Type { 144 | 145 | return reflect.SliceOf(p.r.RetType()) 146 | } 147 | 148 | func (p *array) SizeOf() int { 149 | 150 | size := p.r.SizeOf() 151 | if size < 0 { 152 | return -1 153 | } 154 | return p.n * size 155 | } 156 | 157 | // Array returns a matching unit that matches R n times. 158 | // 159 | func Array(r Ruler, n int) Ruler { 160 | 161 | //TODO: 162 | //if t, ok := r.(BaseType); ok { 163 | // return &baseArray{r: t, n: n} 164 | //} 165 | if r == Char { 166 | return charArray(n) 167 | } else if r == Uint8 { 168 | return byteArray(n) 169 | } 170 | return &array{r: r, n: n} 171 | } 172 | 173 | // ----------------------------------------------------------------------------- 174 | 175 | type dynarray struct { 176 | r Ruler 177 | n func(ctx *Context) int 178 | } 179 | 180 | func (p *dynarray) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 181 | 182 | n := p.n(ctx) 183 | return matchArray(p.r, n, in, ctx) 184 | } 185 | 186 | func (p *dynarray) RetType() reflect.Type { 187 | 188 | return reflect.SliceOf(p.r.RetType()) 189 | } 190 | 191 | func (p *dynarray) SizeOf() int { 192 | 193 | return -1 194 | } 195 | 196 | // Dynarray returns a matching unit that matches R n(ctx) times. 197 | // 198 | func Dynarray(r Ruler, n func(ctx *Context) int) Ruler { 199 | 200 | //TODO: 201 | //if t, ok := r.(BaseType); ok { 202 | // return &baseDynarray{r: t, n: n} 203 | //} 204 | if r == Char { 205 | return charDynarray(n) 206 | } else if r == Uint8 { 207 | return byteDynarray(n) 208 | } 209 | return &dynarray{r: r, n: n} 210 | } 211 | 212 | // ----------------------------------------------------------------------------- 213 | -------------------------------------------------------------------------------- /base_array.go: -------------------------------------------------------------------------------- 1 | package bpl 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "reflect" 7 | "unsafe" 8 | 9 | "github.com/qiniu/x/bufiox" 10 | ) 11 | 12 | // ----------------------------------------------------------------------------- 13 | 14 | func matchCharArray(n int, in *bufio.Reader, ctx *Context) (v interface{}, err error) { 15 | 16 | if n == 0 { 17 | return "", nil 18 | } 19 | 20 | b := make([]byte, n) 21 | _, err = io.ReadFull(in, b) 22 | if err != nil { 23 | return 24 | } 25 | return string(b), nil 26 | } 27 | 28 | func matchByteArray(n int, in *bufio.Reader, ctx *Context) (v interface{}, err error) { 29 | 30 | if n == 0 { 31 | return []byte(nil), nil 32 | } 33 | 34 | b := make([]byte, n) 35 | _, err = io.ReadFull(in, b) 36 | if err != nil { 37 | return 38 | } 39 | return b, nil 40 | } 41 | 42 | func matchBaseArray(R BaseType, n int, in *bufio.Reader, ctx *Context) (v interface{}, err error) { 43 | 44 | if n == 0 { 45 | return 46 | } 47 | 48 | t := baseTypes[R] 49 | v = t.newn(n) 50 | data := (*reflect.SliceHeader)(unsafe.Pointer(reflect.ValueOf(v).UnsafeAddr())).Data 51 | b := (*[1 << 30]byte)(unsafe.Pointer(data)) 52 | _, err = io.ReadFull(in, b[:n*t.sizeOf]) 53 | return 54 | } 55 | 56 | // ----------------------------------------------------------------------------- 57 | 58 | type baseArray struct { 59 | r BaseType 60 | n int 61 | } 62 | 63 | func (p *baseArray) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 64 | 65 | n := p.n 66 | return matchBaseArray(p.r, n, in, ctx) 67 | } 68 | 69 | func (p *baseArray) RetType() reflect.Type { 70 | 71 | return reflect.SliceOf(p.r.RetType()) 72 | } 73 | 74 | func (p *baseArray) SizeOf() int { 75 | 76 | return p.r.SizeOf() * p.n 77 | } 78 | 79 | // BaseArray returns a matching unit that matches R n times. 80 | // 81 | func BaseArray(r BaseType, n int) Ruler { 82 | 83 | return &baseArray{r: r, n: n} 84 | } 85 | 86 | // ----------------------------------------------------------------------------- 87 | 88 | type baseDynarray struct { 89 | r BaseType 90 | n func(ctx *Context) int 91 | } 92 | 93 | func (p *baseDynarray) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 94 | 95 | n := p.n(ctx) 96 | return matchBaseArray(p.r, n, in, ctx) 97 | } 98 | 99 | func (p *baseDynarray) RetType() reflect.Type { 100 | 101 | return reflect.SliceOf(p.r.RetType()) 102 | } 103 | 104 | func (p *baseDynarray) SizeOf() int { 105 | 106 | return -1 107 | } 108 | 109 | // BaseDynarray returns a matching unit that matches R n(ctx) times. 110 | // 111 | func BaseDynarray(r BaseType, n func(ctx *Context) int) Ruler { 112 | 113 | return &baseDynarray{r: r, n: n} 114 | } 115 | 116 | // ----------------------------------------------------------------------------- 117 | 118 | type byteArray0 int 119 | 120 | func (p byteArray0) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 121 | 122 | v, err = bufiox.ReadAll(in) 123 | return 124 | } 125 | 126 | func (p byteArray0) RetType() reflect.Type { 127 | 128 | return tyByteSlice 129 | } 130 | 131 | func (p byteArray0) SizeOf() int { 132 | 133 | return -1 134 | } 135 | 136 | // ByteArray0 is a matching unit that matches `*byte`. 137 | // 138 | var ByteArray0 Ruler = byteArray0(0) 139 | 140 | // ----------------------------------------------------------------------------- 141 | 142 | type byteArray1 int 143 | 144 | func (p byteArray1) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 145 | 146 | ret, err := bufiox.ReadAll(in) 147 | if err != nil { 148 | return 149 | } 150 | if len(ret) == 0 { 151 | panic("match +byte failed: EOF encountered") 152 | } 153 | return ret, nil 154 | } 155 | 156 | func (p byteArray1) RetType() reflect.Type { 157 | 158 | return tyByteSlice 159 | } 160 | 161 | func (p byteArray1) SizeOf() int { 162 | 163 | return -1 164 | } 165 | 166 | // ByteArray1 is a matching unit that matches `+byte`. 167 | // 168 | var ByteArray1 Ruler = byteArray1(0) 169 | 170 | // ----------------------------------------------------------------------------- 171 | 172 | type byteArray int 173 | 174 | func (p byteArray) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 175 | 176 | return matchByteArray(int(p), in, ctx) 177 | } 178 | 179 | func (p byteArray) RetType() reflect.Type { 180 | 181 | return tyByteSlice 182 | } 183 | 184 | func (p byteArray) SizeOf() int { 185 | 186 | return int(p) 187 | } 188 | 189 | // ByteArray returns a matching unit that matches `[n]byte`. 190 | // 191 | func ByteArray(n int) Ruler { 192 | 193 | return byteArray(n) 194 | } 195 | 196 | // ----------------------------------------------------------------------------- 197 | 198 | type charArray int 199 | 200 | func (p charArray) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 201 | 202 | return matchCharArray(int(p), in, ctx) 203 | } 204 | 205 | func (p charArray) RetType() reflect.Type { 206 | 207 | return tyString 208 | } 209 | 210 | func (p charArray) SizeOf() int { 211 | 212 | return int(p) 213 | } 214 | 215 | // CharArray returns a matching unit that matches `[n]char`. 216 | // 217 | func CharArray(n int) Ruler { 218 | 219 | return charArray(n) 220 | } 221 | 222 | // ----------------------------------------------------------------------------- 223 | 224 | type byteDynarray func(ctx *Context) int 225 | 226 | func (p byteDynarray) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 227 | 228 | n := p(ctx) 229 | return matchByteArray(n, in, ctx) 230 | } 231 | 232 | func (p byteDynarray) RetType() reflect.Type { 233 | 234 | return tyByteSlice 235 | } 236 | 237 | func (p byteDynarray) SizeOf() int { 238 | 239 | return -1 240 | } 241 | 242 | // ByteDynarray returns a matching unit that matches `[n(ctx)]byte`. 243 | // 244 | func ByteDynarray(n func(ctx *Context) int) Ruler { 245 | 246 | return byteDynarray(n) 247 | } 248 | 249 | // ----------------------------------------------------------------------------- 250 | 251 | type charDynarray func(ctx *Context) int 252 | 253 | func (p charDynarray) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 254 | 255 | n := p(ctx) 256 | return matchCharArray(n, in, ctx) 257 | } 258 | 259 | func (p charDynarray) RetType() reflect.Type { 260 | 261 | return tyString 262 | } 263 | 264 | func (p charDynarray) SizeOf() int { 265 | 266 | return -1 267 | } 268 | 269 | // CharDynarray returns a matching unit that matches `[n(ctx)]char`. 270 | // 271 | func CharDynarray(n func(ctx *Context) int) Ruler { 272 | 273 | return charDynarray(n) 274 | } 275 | 276 | // ----------------------------------------------------------------------------- 277 | -------------------------------------------------------------------------------- /base_type.go: -------------------------------------------------------------------------------- 1 | package bpl 2 | 3 | import ( 4 | "bufio" 5 | "encoding/binary" 6 | "io" 7 | "reflect" 8 | "unsafe" 9 | 10 | "github.com/qiniu/x/log" 11 | ) 12 | 13 | // ----------------------------------------------------------------------------- 14 | 15 | var ( 16 | tyInt = reflect.TypeOf(int(0)) 17 | tyInt8 = reflect.TypeOf(int8(0)) 18 | tyInt16 = reflect.TypeOf(int16(0)) 19 | tyInt32 = reflect.TypeOf(int32(0)) 20 | tyInt64 = reflect.TypeOf(int64(0)) 21 | tyUint = reflect.TypeOf(uint(0)) 22 | tyUint8 = reflect.TypeOf(uint8(0)) 23 | tyUint16 = reflect.TypeOf(uint16(0)) 24 | tyUint32 = reflect.TypeOf(uint32(0)) 25 | tyUint64 = reflect.TypeOf(uint64(0)) 26 | tyFloat32 = reflect.TypeOf(float32(0)) 27 | tyFloat64 = reflect.TypeOf(float64(0)) 28 | tyString = reflect.TypeOf(string("")) 29 | tyByteSlice = reflect.TypeOf([]byte(nil)) 30 | TyInterface = reflect.TypeOf((*interface{})(nil)).Elem() 31 | tyInterfaceSlice = reflect.SliceOf(TyInterface) 32 | ) 33 | 34 | // ----------------------------------------------------------------------------- 35 | 36 | // A BaseType represents a matching unit of a builtin fixed size type. 37 | // 38 | type BaseType uint 39 | 40 | type baseTypeInfo struct { 41 | read func(in *bufio.Reader) (v interface{}, err error) 42 | newn func(n int) interface{} 43 | typ reflect.Type 44 | sizeOf int 45 | } 46 | 47 | var baseTypes = [...]baseTypeInfo{ 48 | reflect.Int8: {readInt8, newInt8n, tyInt8, 1}, 49 | reflect.Int16: {readInt16, newInt16n, tyInt16, 2}, 50 | reflect.Int32: {readInt32, newInt32n, tyInt32, 4}, 51 | reflect.Int64: {readInt64, newInt64n, tyInt64, 8}, 52 | reflect.Uint8: {readUint8, newUint8n, tyUint8, 1}, 53 | reflect.Uint16: {readUint16, newUint16n, tyUint16, 2}, 54 | reflect.Uint32: {readUint32, newUint32n, tyUint32, 4}, 55 | reflect.Uint64: {readUint64, newUint64n, tyUint64, 8}, 56 | reflect.Float32: {readFloat32, newFloat32n, tyFloat32, 4}, 57 | reflect.Float64: {readFloat64, newFloat64n, tyFloat64, 8}, 58 | } 59 | 60 | func readInt8(in *bufio.Reader) (v interface{}, err error) { 61 | 62 | t, err := in.ReadByte() 63 | return int8(t), err 64 | } 65 | 66 | func readUint8(in *bufio.Reader) (v interface{}, err error) { 67 | 68 | return in.ReadByte() 69 | } 70 | 71 | func readInt16(in *bufio.Reader) (v interface{}, err error) { 72 | 73 | t1, err := in.ReadByte() 74 | if err != nil { 75 | return 76 | } 77 | t2, err := in.ReadByte() 78 | return (int16(t2) << 8) | int16(t1), err 79 | } 80 | 81 | func readUint16(in *bufio.Reader) (v interface{}, err error) { 82 | 83 | t1, err := in.ReadByte() 84 | if err != nil { 85 | return 86 | } 87 | t2, err := in.ReadByte() 88 | return (uint16(t2) << 8) | uint16(t1), err 89 | } 90 | 91 | func readInt32(in *bufio.Reader) (v interface{}, err error) { 92 | 93 | t, err := in.Peek(4) 94 | if err != nil { 95 | return 96 | } 97 | v = int32(binary.LittleEndian.Uint32(t)) 98 | in.Discard(4) 99 | return 100 | } 101 | 102 | func readUint32(in *bufio.Reader) (v interface{}, err error) { 103 | 104 | t, err := in.Peek(4) 105 | if err != nil { 106 | return 107 | } 108 | v = binary.LittleEndian.Uint32(t) 109 | in.Discard(4) 110 | return 111 | } 112 | 113 | func readInt64(in *bufio.Reader) (v interface{}, err error) { 114 | 115 | t, err := in.Peek(8) 116 | if err != nil { 117 | return 118 | } 119 | v = int64(binary.LittleEndian.Uint64(t)) 120 | in.Discard(8) 121 | return 122 | } 123 | 124 | func readUint64(in *bufio.Reader) (v interface{}, err error) { 125 | 126 | t, err := in.Peek(8) 127 | if err != nil { 128 | return 129 | } 130 | v = binary.LittleEndian.Uint64(t) 131 | in.Discard(8) 132 | return 133 | } 134 | 135 | func readFloat32(in *bufio.Reader) (v interface{}, err error) { 136 | 137 | t, err := in.Peek(4) 138 | if err != nil { 139 | return 140 | } 141 | v = *(*float32)(unsafe.Pointer(&t[0])) 142 | in.Discard(4) 143 | return 144 | } 145 | 146 | func readFloat64(in *bufio.Reader) (v interface{}, err error) { 147 | 148 | t, err := in.Peek(8) 149 | if err != nil { 150 | return 151 | } 152 | v = *(*float64)(unsafe.Pointer(&t[0])) 153 | in.Discard(8) 154 | return 155 | } 156 | 157 | func newInt8n(n int) interface{} { 158 | 159 | return make([]int8, n) 160 | } 161 | 162 | func newUint8n(n int) interface{} { 163 | 164 | return make([]uint8, n) 165 | } 166 | 167 | func newInt16n(n int) interface{} { 168 | 169 | return make([]int16, n) 170 | } 171 | 172 | func newUint16n(n int) interface{} { 173 | 174 | return make([]uint16, n) 175 | } 176 | 177 | func newInt32n(n int) interface{} { 178 | 179 | return make([]int32, n) 180 | } 181 | 182 | func newUint32n(n int) interface{} { 183 | 184 | return make([]uint32, n) 185 | } 186 | 187 | func newInt64n(n int) interface{} { 188 | 189 | return make([]int64, n) 190 | } 191 | 192 | func newUint64n(n int) interface{} { 193 | 194 | return make([]uint64, n) 195 | } 196 | 197 | func newFloat32n(n int) interface{} { 198 | 199 | return make([]float32, n) 200 | } 201 | 202 | func newFloat64n(n int) interface{} { 203 | 204 | return make([]float64, n) 205 | } 206 | 207 | // Match is required by a matching unit. see Ruler interface. 208 | // 209 | func (p BaseType) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 210 | 211 | v, err = baseTypes[p].read(in) 212 | return 213 | } 214 | 215 | // RetType returns matching result type. 216 | // 217 | func (p BaseType) RetType() reflect.Type { 218 | 219 | return baseTypes[p].typ 220 | } 221 | 222 | // SizeOf is required by a matching unit. see Ruler interface. 223 | // 224 | func (p BaseType) SizeOf() int { 225 | 226 | return baseTypes[p].sizeOf 227 | } 228 | 229 | var ( 230 | // Int8 is the matching unit for int8 231 | Int8 = BaseType(reflect.Int8) 232 | 233 | // Int16 is the matching unit for int16 234 | Int16 = BaseType(reflect.Int16) 235 | 236 | // Int32 is the matching unit for int32 237 | Int32 = BaseType(reflect.Int32) 238 | 239 | // Int64 is the matching unit for int64 240 | Int64 = BaseType(reflect.Int64) 241 | 242 | // Uint8 is the matching unit for uint8 243 | Uint8 = BaseType(reflect.Uint8) 244 | 245 | // Uint16 is the matching unit for uint16 246 | Uint16 = BaseType(reflect.Uint16) 247 | 248 | // Uint32 is the matching unit for uint32 249 | Uint32 = BaseType(reflect.Uint32) 250 | 251 | // Uint64 is the matching unit for uint64 252 | Uint64 = BaseType(reflect.Uint64) 253 | 254 | // Float32 is the matching unit for float32 255 | Float32 = BaseType(reflect.Float32) 256 | 257 | // Float64 is the matching unit for float64 258 | Float64 = BaseType(reflect.Float64) 259 | ) 260 | 261 | // ----------------------------------------------------------------------------- 262 | 263 | type cstring int 264 | 265 | func (p cstring) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 266 | 267 | b, err := in.ReadBytes(0) 268 | if err != nil { 269 | return 270 | } 271 | return string(b[:len(b)-1]), nil 272 | } 273 | 274 | func (p cstring) RetType() reflect.Type { 275 | 276 | return tyString 277 | } 278 | 279 | func (p cstring) SizeOf() int { 280 | 281 | return -1 282 | } 283 | 284 | // CString is a matching unit that matches a C style string. 285 | // 286 | var CString Ruler = cstring(0) 287 | 288 | // ----------------------------------------------------------------------------- 289 | 290 | type charType int 291 | 292 | func (p charType) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 293 | 294 | return in.ReadByte() 295 | } 296 | 297 | func (p charType) RetType() reflect.Type { 298 | 299 | return tyUint8 300 | } 301 | 302 | func (p charType) SizeOf() int { 303 | 304 | return 1 305 | } 306 | 307 | // Char is a matching unit that matches a character. 308 | // 309 | var Char Ruler = charType(0) 310 | 311 | // ----------------------------------------------------------------------------- 312 | 313 | type fixedType struct { 314 | typ reflect.Type 315 | } 316 | 317 | func (p *fixedType) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 318 | 319 | typ := p.typ 320 | size := typ.Size() 321 | val := reflect.New(typ) 322 | b := (*[1 << 30]byte)(unsafe.Pointer(val.UnsafeAddr())) 323 | _, err = io.ReadFull(in, b[:size]) 324 | if err != nil { 325 | log.Warn("fixedType.Match: io.ReadFull failed -", err) 326 | return 327 | } 328 | return val.Interface(), nil 329 | } 330 | 331 | func (p *fixedType) RetType() reflect.Type { 332 | 333 | return p.typ 334 | } 335 | 336 | func (p *fixedType) SizeOf() int { 337 | 338 | return int(p.typ.Size()) 339 | } 340 | 341 | // FixedType returns a matching unit that matches a C style fixed size struct. 342 | // 343 | func FixedType(t reflect.Type) Ruler { 344 | 345 | return &fixedType{typ: t} 346 | } 347 | 348 | // ----------------------------------------------------------------------------- 349 | 350 | type uintbe int 351 | 352 | func (p uintbe) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 353 | 354 | t, err := in.Peek(int(p)) 355 | if err != nil { 356 | return 357 | } 358 | var val uint 359 | for i := 0; i < int(p); i++ { 360 | val = (val << 8) | uint(t[i]) 361 | } 362 | in.Discard(int(p)) 363 | return val, nil 364 | } 365 | 366 | func (p uintbe) RetType() reflect.Type { 367 | 368 | return tyUint 369 | } 370 | 371 | func (p uintbe) SizeOf() int { 372 | 373 | return int(p) 374 | } 375 | 376 | // Uintbe returns a matching unit that matches a uintbe(n) type. 377 | // 378 | func Uintbe(n int) Ruler { 379 | 380 | if n < 2 || n > 8 { 381 | panic("Uintbe: invalid argument (n >= 2 && n <= 8)") 382 | } 383 | return uintbe(n) 384 | } 385 | 386 | // ----------------------------------------------------------------------------- 387 | 388 | type uintle int 389 | 390 | func (p uintle) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 391 | 392 | t, err := in.Peek(int(p)) 393 | if err != nil { 394 | return 395 | } 396 | var val uint 397 | for i := int(p); i > 0; { 398 | i-- 399 | val = (val << 8) | uint(t[i]) 400 | } 401 | in.Discard(int(p)) 402 | return val, nil 403 | } 404 | 405 | func (p uintle) RetType() reflect.Type { 406 | 407 | return tyUint 408 | } 409 | 410 | func (p uintle) SizeOf() int { 411 | 412 | return int(p) 413 | } 414 | 415 | // Uintle returns a matching unit that matches a uintle(n) type. 416 | // 417 | func Uintle(n int) Ruler { 418 | 419 | if n < 2 || n > 8 { 420 | panic("Uintle: invalid argument (n >= 2 && n <= 8)") 421 | } 422 | return uintle(n) 423 | } 424 | 425 | // Uint24 is a matching unit that matches a uintle(3) type. 426 | // 427 | var Uint24 = Uintle(3) 428 | 429 | // ----------------------------------------------------------------------------- 430 | 431 | func float32frombits(b uint32) float32 { return *(*float32)(unsafe.Pointer(&b)) } 432 | 433 | type float32be int 434 | 435 | func (p float32be) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 436 | 437 | t, err := in.Peek(4) 438 | if err != nil { 439 | return 440 | } 441 | v = float32frombits(binary.BigEndian.Uint32(t)) 442 | in.Discard(4) 443 | return 444 | } 445 | 446 | func (p float32be) RetType() reflect.Type { 447 | 448 | return tyFloat32 449 | } 450 | 451 | func (p float32be) SizeOf() int { 452 | 453 | return 4 454 | } 455 | 456 | // Float32be returns a matching unit that matches a float32be type. 457 | // 458 | var Float32be Ruler = float32be(0) 459 | 460 | // ----------------------------------------------------------------------------- 461 | 462 | func float64frombits(b uint64) float64 { return *(*float64)(unsafe.Pointer(&b)) } 463 | 464 | type float64be int 465 | 466 | func (p float64be) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 467 | 468 | t, err := in.Peek(8) 469 | if err != nil { 470 | return 471 | } 472 | v = float64frombits(binary.BigEndian.Uint64(t)) 473 | in.Discard(8) 474 | return 475 | } 476 | 477 | func (p float64be) RetType() reflect.Type { 478 | 479 | return tyFloat64 480 | } 481 | 482 | func (p float64be) SizeOf() int { 483 | 484 | return 8 485 | } 486 | 487 | // Float64be returns a matching unit that matches a float64be type. 488 | // 489 | var Float64be Ruler = float64be(0) 490 | 491 | // ----------------------------------------------------------------------------- 492 | -------------------------------------------------------------------------------- /base_type_test.go: -------------------------------------------------------------------------------- 1 | package bpl_test 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/json" 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/goplus/bpl" 10 | "github.com/qiniu/x/bufiox" 11 | ) 12 | 13 | func TestBaseType(t *testing.T) { 14 | 15 | b := make([]byte, 8) 16 | binary.LittleEndian.PutUint64(b, 123) 17 | in := bufiox.NewReaderBuffer(b) 18 | 19 | ctx := bpl.NewContext() 20 | named := &bpl.Member{Name: "foo", Type: bpl.Int64} 21 | v, err := named.Match(in, ctx) 22 | if err != nil { 23 | t.Fatal("Member.Match failed:", err) 24 | } 25 | if v != int64(123) { 26 | t.Fatal("v != 123") 27 | } 28 | if v, ok := ctx.Var("foo"); !ok || v != int64(123) { 29 | t.Fatal("v != 123 - ", reflect.TypeOf(v), v, ok) 30 | } 31 | } 32 | 33 | func TestCString(t *testing.T) { 34 | 35 | b := []byte("Hello, world!") 36 | b = append(b, 0) 37 | in := bufiox.NewReaderBuffer(b) 38 | 39 | v, err := bpl.CString.Match(in, nil) 40 | if err != nil { 41 | t.Fatal("CString.Match failed:", err) 42 | } 43 | if v != "Hello, world!" { 44 | t.Fatal("CString.Match result:", v) 45 | } 46 | 47 | b2, err := json.Marshal(v) 48 | if err != nil { 49 | t.Fatal("json.Marshal failed:", err) 50 | } 51 | if string(b2) != `"Hello, world!"` { 52 | t.Fatal("json.Marshal result:", b2) 53 | } 54 | } 55 | 56 | func TestUintbe(t *testing.T) { 57 | 58 | b := []byte{1, 2, 3, 4} 59 | in := bufiox.NewReaderBuffer(b) 60 | v, err := bpl.Uintbe(3).Match(in, nil) 61 | if err != nil { 62 | t.Fatal("Uintbe.Match failed:", err) 63 | } 64 | if v != uint(0x010203) { 65 | t.Fatal("v != 0x010203:", v) 66 | } 67 | } 68 | 69 | func TestUintle(t *testing.T) { 70 | 71 | b := []byte{1, 2, 3, 4} 72 | in := bufiox.NewReaderBuffer(b) 73 | v, err := bpl.Uintle(3).Match(in, nil) 74 | if err != nil { 75 | t.Fatal("Uintbe.Match failed:", err) 76 | } 77 | if v != uint(0x030201) { 78 | t.Fatal("v != 0x030201:", v) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /binary/format.go: -------------------------------------------------------------------------------- 1 | package binary 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "reflect" 8 | "unsafe" 9 | ) 10 | 11 | // ----------------------------------------------------------------------------- 12 | 13 | func writeCString(w *bufio.Writer, v string) (err error) { 14 | 15 | _, err = w.WriteString(v) 16 | if err != nil { 17 | return 18 | } 19 | return w.WriteByte(0) 20 | } 21 | 22 | func writeStruct(w *bufio.Writer, v reflect.Value) (err error) { 23 | 24 | n := v.NumField() 25 | for i := 0; i < n; i++ { 26 | fv := v.Field(i) 27 | err = WriteValue(w, fv) 28 | if err != nil { 29 | return 30 | } 31 | } 32 | return 33 | } 34 | 35 | // WriteValue serializes data into a writer. 36 | // 37 | func WriteValue(w *bufio.Writer, v reflect.Value) (err error) { 38 | 39 | var val uint64 40 | 41 | retry: 42 | kind := v.Kind() 43 | switch { 44 | case kind == reflect.Struct: 45 | return writeStruct(w, v) 46 | case kind >= reflect.Int8 && kind <= reflect.Int64: 47 | val = uint64(v.Int()) 48 | kind -= reflect.Int8 49 | case kind >= reflect.Uint8 && kind <= reflect.Uint64: 50 | val = v.Uint() 51 | kind -= reflect.Uint8 52 | case kind == reflect.String: 53 | return writeCString(w, v.String()) 54 | case kind == reflect.Float64: 55 | *(*float64)(unsafe.Pointer(&val)) = v.Float() 56 | kind = 3 57 | case kind == reflect.Float32: 58 | *(*float32)(unsafe.Pointer(&val)) = float32(v.Float()) 59 | kind = 2 60 | case kind == reflect.Ptr: 61 | v = v.Elem() 62 | goto retry 63 | default: 64 | return fmt.Errorf("bpl/binary.Write - unsupported type: %v", v.Type()) 65 | } 66 | b := (*[8]byte)(unsafe.Pointer(&val)) 67 | _, err = w.Write(b[:1< 0 { 64 | flag = flags[0] 65 | } 66 | Dumper = log.New(w, "", flag) 67 | } 68 | 69 | func writePrefix(b *bytes.Buffer, lvl int) { 70 | 71 | for i := 0; i < lvl; i++ { 72 | b.WriteString(" ") 73 | } 74 | } 75 | 76 | // DumpDom dumps a dom tree. 77 | // 78 | func DumpDom(b *bytes.Buffer, dom interface{}, lvl int) { 79 | 80 | dumpDomValue(b, reflect.ValueOf(dom), lvl) 81 | } 82 | 83 | type stringSlice []reflect.Value 84 | 85 | func (p stringSlice) Len() int { return len(p) } 86 | func (p stringSlice) Less(i, j int) bool { return p[i].String() < p[j].String() } 87 | func (p stringSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } 88 | 89 | var typeBytes = reflect.TypeOf([]byte(nil)) 90 | 91 | func dumpDomValue(b *bytes.Buffer, dom reflect.Value, lvl int) { 92 | 93 | retry: 94 | switch dom.Kind() { 95 | case reflect.Slice: 96 | if dom.Type() == typeBytes { 97 | b.WriteByte('\n') 98 | d := hex.Dumper((*filterWriter)(b)) 99 | d.Write(dom.Bytes()) 100 | d.Close() 101 | return 102 | } 103 | b.WriteByte('[') 104 | n := dom.Len() 105 | for i := 0; i < n; i++ { 106 | b.WriteByte('\n') 107 | writePrefix(b, lvl+1) 108 | dumpDomValue(b, dom.Index(i), lvl+1) 109 | b.WriteByte(',') 110 | } 111 | b.WriteByte('\n') 112 | writePrefix(b, lvl) 113 | b.WriteByte(']') 114 | case reflect.Map: 115 | b.WriteByte('{') 116 | keys := dom.MapKeys() 117 | fstring := dom.Type().Key().Kind() == reflect.String 118 | if fstring { 119 | n := 0 120 | for _, key := range keys { 121 | if strings.HasPrefix(key.String(), "_") { 122 | continue 123 | } 124 | keys[n] = key 125 | n++ 126 | } 127 | keys = keys[:n] 128 | sort.Sort(stringSlice(keys)) 129 | } 130 | for _, key := range keys { 131 | item := dom.MapIndex(key) 132 | b.WriteByte('\n') 133 | writePrefix(b, lvl+1) 134 | if fstring { 135 | b.WriteString(key.String()) 136 | } else { 137 | dumpDomValue(b, key, lvl+1) 138 | } 139 | b.WriteString(": ") 140 | dumpDomValue(b, item, lvl+1) 141 | } 142 | b.WriteByte('\n') 143 | writePrefix(b, lvl) 144 | b.WriteByte('}') 145 | case reflect.Interface, reflect.Ptr: 146 | if dom.IsNil() { 147 | b.WriteString("") 148 | return 149 | } 150 | dom = dom.Elem() 151 | goto retry 152 | default: 153 | ret, _ := json.Marshal(dom.Interface()) 154 | b.Write(ret) 155 | } 156 | } 157 | 158 | // ----------------------------------------------------------------------------- 159 | 160 | type dump int 161 | 162 | func (p dump) Match(in *bufio.Reader, ctx *bpl.Context) (v interface{}, err error) { 163 | 164 | dom := ctx.Dom() 165 | if dom == qlang.Undefined { 166 | return 167 | } 168 | 169 | var b bytes.Buffer 170 | if prefix, ok := ctx.Globals.Var("BPL_DUMP_PREFIX"); ok { 171 | b.WriteString(prefix.(string)) 172 | } 173 | b.WriteByte('\n') 174 | DumpDom(&b, dom, 0) 175 | Dumper.Info(b.String()) 176 | return 177 | } 178 | 179 | func (p dump) RetType() reflect.Type { 180 | 181 | return bpl.TyInterface 182 | } 183 | 184 | func (p dump) SizeOf() int { 185 | 186 | return 0 187 | } 188 | 189 | // ----------------------------------------------------------------------------- 190 | 191 | // A Ruler is a matching unit. 192 | // 193 | type Ruler struct { 194 | Impl bpl.Ruler 195 | } 196 | 197 | // Match matches input stream `in`, and returns matching result. 198 | // 199 | func (p Ruler) Match(in *bufio.Reader, ctx *bpl.Context) (v interface{}, err error) { 200 | 201 | return bpl.MatchStream(p.Impl, in, ctx) 202 | } 203 | 204 | // SafeMatch matches input stream `in`, and returns matching result. 205 | // 206 | func (p Ruler) SafeMatch(in *bufio.Reader, ctx *bpl.Context) (v interface{}, err error) { 207 | 208 | defer func() { 209 | if e := recover(); e != nil { 210 | switch val := e.(type) { 211 | case string: 212 | err = errors.New(val) 213 | case error: 214 | err = val 215 | case int: 216 | v = val 217 | default: 218 | panic(e) 219 | } 220 | } 221 | }() 222 | 223 | return bpl.MatchStream(p.Impl, in, ctx) 224 | } 225 | 226 | // MatchStream matches input stream `r`, and returns matching result. 227 | // 228 | func (p Ruler) MatchStream(r io.Reader) (v interface{}, err error) { 229 | 230 | in := bufio.NewReader(r) 231 | ctx := bpl.NewContext() 232 | return p.SafeMatch(in, ctx) 233 | } 234 | 235 | // MatchBuffer matches input buffer `b`, and returns matching result. 236 | // 237 | func (p Ruler) MatchBuffer(b []byte) (v interface{}, err error) { 238 | 239 | in := bufiox.NewReaderBuffer(b) 240 | ctx := bpl.NewContext() 241 | return p.SafeMatch(in, ctx) 242 | } 243 | 244 | // ----------------------------------------------------------------------------- 245 | 246 | // New compiles bpl source code and returns the corresponding matching unit. 247 | // 248 | func New(code []byte, fname string) (r Ruler, err error) { 249 | 250 | defer func() { 251 | if e := recover(); e != nil { 252 | switch v := e.(type) { 253 | case string: 254 | err = errors.New(v) 255 | case error: 256 | err = v 257 | default: 258 | panic(e) 259 | } 260 | } 261 | }() 262 | 263 | p := newCompiler() 264 | engine, err := interpreter.New(p, interpreter.InsertSemis) 265 | if err != nil { 266 | return 267 | } 268 | 269 | p.ipt = engine 270 | err = engine.MatchExactly(code, fname) 271 | if err != nil { 272 | return 273 | } 274 | 275 | if DumpCode != 0 { 276 | p.code.Dump() 277 | } 278 | return p.Ret() 279 | } 280 | 281 | // NewFromString compiles bpl source code and returns the corresponding matching unit. 282 | // 283 | func NewFromString(code string, fname string) (r Ruler, err error) { 284 | 285 | return New([]byte(code), fname) 286 | } 287 | 288 | // NewFromFile compiles bpl source file and returns the corresponding matching unit. 289 | // 290 | func NewFromFile(fname string) (r Ruler, err error) { 291 | 292 | b, err := ioutil.ReadFile(fname) 293 | if err != nil { 294 | return 295 | } 296 | return New(b, fname) 297 | } 298 | 299 | // NewContext returns a new matching Context. 300 | // 301 | func NewContext() *bpl.Context { 302 | 303 | return bpl.NewContext() 304 | } 305 | 306 | // ----------------------------------------------------------------------------- 307 | 308 | // SetDumpCode sets dump code mode: 309 | // "1" - dump code with rem instruction. 310 | // "2" - dump code without rem instruction. 311 | // else - don't dump code. 312 | // 313 | func SetDumpCode(dumpCode string) { 314 | 315 | switch dumpCode { 316 | case "true", "1": 317 | DumpCode = 1 318 | case "2": 319 | DumpCode = 2 320 | default: 321 | DumpCode = 0 322 | } 323 | } 324 | 325 | // ----------------------------------------------------------------------------- 326 | -------------------------------------------------------------------------------- /bpl.ext/bpl_test.go: -------------------------------------------------------------------------------- 1 | package bpl 2 | 3 | import ( 4 | "bytes" 5 | "encoding/hex" 6 | "encoding/json" 7 | "testing" 8 | 9 | "github.com/goplus/bpl/binary" 10 | qlang "github.com/xushiwei/qlang/spec" 11 | ) 12 | 13 | // ----------------------------------------------------------------------------- 14 | 15 | func TestHexdump(t *testing.T) { 16 | 17 | b := bytes.NewBuffer(nil) 18 | d := hex.Dumper((*filterWriter)(b)) 19 | d.Write([]byte{ 20 | 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 21 | 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 22 | }) 23 | d.Close() 24 | if string(b.Bytes()) != `00000000 60 60 60 60 60 60 60 60 60 60 60 60 60 60 60 60 |................| 25 | 00000010 60 60 60 60 60 60 60 60 60 60 60 60 60 |.............| 26 | ` { 27 | t.Fatal("Hexdump failed") 28 | } 29 | } 30 | 31 | // ----------------------------------------------------------------------------- 32 | 33 | const codeBasic = ` 34 | 35 | sub1 = int8 uint16 36 | 37 | subType = cstring 38 | 39 | doc = [sub1 uint32 float32 cstring subType float64] 40 | ` 41 | 42 | type subType struct { 43 | Foo string 44 | } 45 | 46 | type fooType struct { 47 | A int8 48 | B uint16 49 | C uint32 50 | D float32 51 | E string 52 | F subType 53 | G float64 54 | } 55 | 56 | func TestBasic(t *testing.T) { 57 | 58 | foo := &fooType{ 59 | A: 1, B: 2, C: 3, D: 3.14, E: "Hello", F: subType{Foo: "foo"}, G: 7.52, 60 | // 1 + 2 + 4 + 4 + 6 + 4 + 8 = 29 61 | } 62 | b, err := binary.Marshal(&foo) 63 | if err != nil { 64 | t.Fatal("binary.Marshal failed:", err) 65 | } 66 | if len(b) != 29 { 67 | t.Fatal("len(b) != 29, len:", len(b), "data:", string(b)) 68 | } 69 | 70 | r, err := NewFromString(codeBasic, "") 71 | if err != nil { 72 | t.Fatal("New failed:", err) 73 | } 74 | v, err := r.MatchBuffer(b) 75 | if err != nil { 76 | t.Fatal("Match failed:", err, "len:", len(b)) 77 | } 78 | ret, err := json.Marshal(v) 79 | if err != nil { 80 | t.Fatal("json.Marshal failed:", err) 81 | } 82 | if string(ret) != `[null,3,3.14,"Hello","foo",7.52]` { 83 | t.Fatal("ret:", string(ret)) 84 | } 85 | } 86 | 87 | // ----------------------------------------------------------------------------- 88 | 89 | const codeBasic2 = ` 90 | 91 | sub1 = [int8 uint16] 92 | 93 | subType = cstring 94 | 95 | doc = [sub1 uint32] float32 cstring [subType] float64 96 | ` 97 | 98 | func TestBasic2(t *testing.T) { 99 | 100 | foo := &fooType{ 101 | A: 1, B: 2, C: 3, D: 3.14, E: "Hello", F: subType{Foo: "foo"}, G: 7.52, 102 | // 1 + 2 + 4 + 4 + 6 + 4 + 8 = 29 103 | } 104 | b, err := binary.Marshal(&foo) 105 | if err != nil { 106 | t.Fatal("binary.Marshal failed:", err) 107 | } 108 | if len(b) != 29 { 109 | t.Fatal("len(b) != 29, len:", len(b), "data:", string(b)) 110 | } 111 | 112 | r, err := NewFromString(codeBasic2, "") 113 | if err != nil { 114 | t.Fatal("New failed:", err) 115 | } 116 | v, err := r.MatchBuffer(b) 117 | if err != nil { 118 | t.Fatal("Match failed:", err, "len:", len(b)) 119 | } 120 | ret, err := json.Marshal(v) 121 | if err != nil { 122 | t.Fatal("json.Marshal failed:", err) 123 | } 124 | if string(ret) != `[[1,2],3,"foo"]` { 125 | t.Fatal("ret:", string(ret)) 126 | } 127 | } 128 | 129 | // ----------------------------------------------------------------------------- 130 | 131 | const codeStruct = ` 132 | 133 | sub1 = {/C 134 | int8 a 135 | uint16 b 136 | } 137 | 138 | subType = { 139 | f cstring 140 | assert f == "foo" 141 | } 142 | 143 | doc = { 144 | sub1 sub1 145 | c uint32 146 | d float32 147 | e [5]char 148 | _ byte 149 | f subType 150 | _ float64 151 | assert e == "Hello" 152 | } 153 | ` 154 | 155 | func TestStruct(t *testing.T) { 156 | 157 | foo := &fooType{ 158 | A: 1, B: 2, C: 3, D: 3.14, E: "Hello", F: subType{Foo: "foo"}, G: 7.52, 159 | // 1 + 2 + 4 + 4 + 6 + 4 + 8 = 29 160 | } 161 | b, err := binary.Marshal(&foo) 162 | if err != nil { 163 | t.Fatal("binary.Marshal failed:", err) 164 | } 165 | if len(b) != 29 { 166 | t.Fatal("len(b) != 29, len:", len(b), "data:", string(b)) 167 | } 168 | 169 | r, err := NewFromString(codeStruct, "") 170 | if err != nil { 171 | t.Fatal("New failed:", err) 172 | } 173 | v, err := r.MatchBuffer(b) 174 | if err != nil { 175 | t.Fatal("Match failed:", err, "len:", len(b)) 176 | } 177 | ret, err := json.Marshal(v) 178 | if err != nil { 179 | t.Fatal("json.Marshal failed:", err) 180 | } 181 | if string(ret) != `{"c":3,"d":3.14,"e":"Hello","f":{"f":"foo"},"sub1":{"a":1,"b":2}}` { 182 | t.Fatal("ret:", string(ret)) 183 | } 184 | } 185 | 186 | // ----------------------------------------------------------------------------- 187 | 188 | const codeDump = ` 189 | 190 | doc = { 191 | let a = 1 192 | let _b = 3 193 | } 194 | ` 195 | 196 | func TestDump(t *testing.T) { 197 | 198 | r, err := NewFromString(codeDump, "") 199 | if err != nil { 200 | t.Fatal("New failed:", err) 201 | } 202 | v, err := r.MatchBuffer(nil) 203 | if err != nil { 204 | t.Fatal("Match failed:", err) 205 | } 206 | ret, err := json.Marshal(v) 207 | if err != nil { 208 | t.Fatal("json.Marshal failed:", err) 209 | } 210 | if string(ret) != `{"_b":3,"a":1}` { 211 | t.Fatal("ret:", string(ret)) 212 | } 213 | var b bytes.Buffer 214 | DumpDom(&b, v, 0) 215 | if b.String() != "{\n a: 1\n}" { 216 | t.Fatal("dump:", b.String()) 217 | } 218 | } 219 | 220 | // ----------------------------------------------------------------------------- 221 | 222 | const codeDump2 = ` 223 | 224 | doc = { 225 | return undefined 226 | } 227 | ` 228 | 229 | func TestDump2(t *testing.T) { 230 | 231 | r, err := NewFromString(codeDump2, "") 232 | if err != nil { 233 | t.Fatal("New failed:", err) 234 | } 235 | v, err := r.MatchBuffer(nil) 236 | if err != nil { 237 | t.Fatal("Match failed:", err) 238 | } 239 | if v != qlang.Undefined { 240 | t.Fatal("v:", v) 241 | } 242 | var b bytes.Buffer 243 | DumpDom(&b, v, 0) 244 | if b.String() != "\"```undefined```\"" { 245 | t.Fatal("dump:", b.String()) 246 | } 247 | } 248 | 249 | // ----------------------------------------------------------------------------- 250 | 251 | const codeRtmp1 = ` 252 | 253 | AMF0_NULL = { 254 | return nil 255 | } 256 | 257 | AMF0_NUMBER = { 258 | val float64be 259 | return val 260 | } 261 | 262 | AMF0_STRING = { 263 | len uint16be 264 | val [len]char 265 | return val 266 | } 267 | 268 | AMF0_TYPE = { 269 | marker byte 270 | case marker { 271 | 0x00: AMF0_NUMBER 272 | 0x02: AMF0_STRING 273 | 0x05: AMF0_NULL 274 | } 275 | } 276 | 277 | AMF0_CMDDATA = { 278 | cmd AMF0_TYPE 279 | transactionId AMF0_TYPE 280 | value *AMF0_TYPE 281 | } 282 | 283 | doc = { 284 | msg AMF0_CMDDATA 285 | } 286 | ` 287 | 288 | func TestRtmp1(t *testing.T) { 289 | 290 | buf := []byte{ 291 | 0x02, 0x00, 0x08, 0x6f, 0x6e, 0x42, 0x57, 0x44, 292 | 0x6f, 0x6e, 0x65, 0x00, 0x00, 0x00, 0x00, 0x00, 293 | 0x00, 0x00, 0x00, 0x00, 0x05, 294 | } 295 | r, err := NewFromString(codeRtmp1, "") 296 | if err != nil { 297 | t.Fatal("New failed:", err) 298 | } 299 | v, err := r.MatchBuffer(buf) 300 | if err != nil { 301 | t.Fatal("Match failed:", err) 302 | } 303 | ret, err := json.Marshal(v) 304 | if err != nil { 305 | t.Fatal("json.Marshal failed:", err) 306 | } 307 | if string(ret) != `{"msg":{"cmd":"onBWDone","transactionId":0,"value":[null]}}` { 308 | t.Fatal("ret:", string(ret)) 309 | } 310 | } 311 | 312 | // ----------------------------------------------------------------------------- 313 | -------------------------------------------------------------------------------- /bpl.ext/bson/bson.go: -------------------------------------------------------------------------------- 1 | package bson 2 | 3 | import ( 4 | "bufio" 5 | "encoding/binary" 6 | "encoding/json" 7 | "io" 8 | "reflect" 9 | 10 | "github.com/goplus/bpl" 11 | "gopkg.in/mgo.v2/bson" 12 | ) 13 | 14 | // ----------------------------------------------------------------------------- 15 | 16 | func peekInt32(in *bufio.Reader) (v int32, err error) { 17 | 18 | t, err := in.Peek(4) 19 | if err != nil { 20 | return 21 | } 22 | v = int32(binary.LittleEndian.Uint32(t)) 23 | return 24 | } 25 | 26 | // ----------------------------------------------------------------------------- 27 | 28 | // A Document represents a bson document. 29 | // 30 | type Document struct { 31 | data []byte 32 | cache map[string]interface{} 33 | } 34 | 35 | // MarshalJSON is required by json.Marshal. 36 | // 37 | func (p *Document) MarshalJSON() (b []byte, err error) { 38 | 39 | vars, err := p.Vars() 40 | if err != nil { 41 | return 42 | } 43 | return json.Marshal(vars) 44 | } 45 | 46 | // Vars returns all variables of this document. 47 | // 48 | func (p *Document) Vars() (vars map[string]interface{}, err error) { 49 | 50 | if p.cache == nil { 51 | err = bson.Unmarshal(p.data, &vars) 52 | if err != nil { 53 | return 54 | } 55 | p.cache = vars 56 | return 57 | } 58 | return p.cache, nil 59 | } 60 | 61 | // ----------------------------------------------------------------------------- 62 | 63 | type typeImpl int 64 | 65 | var tyDocument = reflect.TypeOf((*Document)(nil)) 66 | 67 | func (p typeImpl) Match(in *bufio.Reader, ctx *bpl.Context) (v interface{}, err error) { 68 | 69 | n, err := peekInt32(in) 70 | if err != nil { 71 | return 72 | } 73 | data := make([]byte, n) 74 | _, err = io.ReadFull(in, data) 75 | if err != nil { 76 | return 77 | } 78 | return &Document{data: data}, nil 79 | } 80 | 81 | func (p typeImpl) RetType() reflect.Type { 82 | 83 | return tyDocument 84 | } 85 | 86 | func (p typeImpl) SizeOf() int { 87 | 88 | return -1 89 | } 90 | 91 | // Type is a matching unit that matches a bson document. 92 | // 93 | var Type bpl.Ruler = typeImpl(0) 94 | 95 | // ----------------------------------------------------------------------------- 96 | -------------------------------------------------------------------------------- /bpl.ext/bson/bson_test.go: -------------------------------------------------------------------------------- 1 | package bson 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/qiniu/x/bufiox" 8 | "gopkg.in/mgo.v2/bson" 9 | ) 10 | 11 | // ----------------------------------------------------------------------------- 12 | 13 | type M bson.M 14 | 15 | func TestBson(t *testing.T) { 16 | 17 | doc := M{"a": 1, "b": true, "c": "Hello"} 18 | b, err := bson.Marshal(doc) 19 | if err != nil { 20 | t.Fatal("bson.Marshal failed:", err) 21 | } 22 | 23 | in := bufiox.NewReaderBuffer(b) 24 | v, err := Type.Match(in, nil) 25 | if err != nil { 26 | t.Fatal("Match failed:", err) 27 | } 28 | ret, err := json.Marshal(v) 29 | if err != nil { 30 | t.Fatal("json.Marshal failed:", err) 31 | } 32 | if string(ret) != `{"a":1,"b":true,"c":"Hello"}` { 33 | t.Fatal("ret:", string(ret)) 34 | } 35 | } 36 | 37 | // ----------------------------------------------------------------------------- 38 | -------------------------------------------------------------------------------- /bpl.ext/cl.go: -------------------------------------------------------------------------------- 1 | package bpl 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | "github.com/goplus/bpl" 8 | "github.com/goplus/bpl/bpl.ext/bson" 9 | "github.com/qiniu/text/tpl/interpreter.util" 10 | "github.com/xushiwei/qlang/exec" 11 | ) 12 | 13 | const grammar = ` 14 | 15 | expr = +factor/And 16 | 17 | term1 = ifactor *( 18 | '*' ifactor/mul | '/' ifactor/quo | '%' ifactor/mod | 19 | "<<" ifactor/lshr | ">>" ifactor/rshr | '&' ifactor/bitand | "&^" ifactor/andnot) 20 | 21 | term2 = term1 *('+' term1/add | '-' term1/sub | '|' term1/bitor | '^' term1/xor) 22 | 23 | term3 = term2 *('<' term2/lt | '>' term2/gt | "==" term2/eq | "<=" term2/le | ">=" term2/ge | "!=" term2/ne) 24 | 25 | term4 = term3 *("&&" term3/iand) 26 | 27 | qexpr = term4 *("||" term4/ior) 28 | 29 | iexpr = qexpr/qline 30 | 31 | index = '['/istart iexpr ']'/iend 32 | 33 | casecond = INT/casei | STRING/cases 34 | 35 | casebody = (casecond ':' expr/source) %= ';'/ARITY ?(';' "default" ':' expr)/ARITY 36 | 37 | caseexpr = "case"/istart! iexpr/source '{'/iend casebody ?';' '}' /case 38 | 39 | exprblock = true/istart! iexpr (@'{' | "do")/iend expr 40 | 41 | ifexpr = "if" exprblock *("elif" exprblock)/ARITY ?("else"! expr)/ARITY /if 42 | 43 | skipexpr = "skip"/istart! iexpr /iend /skip 44 | 45 | readexpr = "read" exprblock /read 46 | 47 | evalexpr = "eval" exprblock /eval 48 | 49 | doexpr = "do"/istart! iexpr /iend /do 50 | 51 | letexpr = "let"! IDENT/var % ','/ARITY '='/istart! iexpr /iend /let 52 | 53 | assertexpr = ("assert"/istart! iexpr /iend) /assert 54 | 55 | fatalexpr = ("fatal"/istart! iexpr /iend) /fatal 56 | 57 | gblexpr = "global"! IDENT/var '='/istart! iexpr /iend /global 58 | 59 | retexpr = "return"/istart! iexpr /iend /return 60 | 61 | dumpexpr = "dump"/dump 62 | 63 | dynexpr = caseexpr | readexpr | skipexpr | evalexpr | assertexpr | ifexpr | letexpr | doexpr | retexpr | gblexpr | fatalexpr | dumpexpr 64 | 65 | basetype = 66 | IDENT/ident | 67 | (index IDENT/ident)/array 68 | 69 | type = 70 | basetype | 71 | ('*'! basetype)/array0 | 72 | ('?'! basetype)/array01 | 73 | ('+'! basetype)/array1 74 | 75 | member = ((IDENT type)/member | dynexpr)/xline 76 | 77 | cmember = (IDENT/ident ?(index/array | '*'/array0 | '?'/array01 | '+'/array1) IDENT/member | dynexpr)/xline 78 | 79 | cstruct = cmember %= ';'/ARITY /struct 80 | 81 | struct = member %= ';'/ARITY /struct 82 | 83 | factor = 84 | IDENT/ident | 85 | '{' ('/' "C" ';' cstruct | struct) ?';' '}' | 86 | '*' factor/repeat0 | 87 | '+' factor/repeat1 | 88 | '?' factor/repeat01 | 89 | '(' expr ')' | 90 | '[' +factor/Seq ']' | 91 | dynexpr 92 | 93 | imember = IDENT | "assert" | "fatal" | "read" | "skip" | "eval" | "let" | "sizeof" | "C" | "global" | "do" | "dump" 94 | 95 | atom = 96 | '('! qexpr %= ','/ARITY ?"..."/ARITY ?',' ')'/call | 97 | '.'! imember/mref | 98 | '['! ?qexpr/ARITY ?':'/ARITY ?qexpr/ARITY ']'/index 99 | 100 | ifactor = 101 | INT/pushi | 102 | STRING/pushs | 103 | CHAR/pushc | 104 | (IDENT/ref | '('! qexpr ')' | '[' qexpr %= ','/ARITY ?',' ']'/slice) *atom | 105 | "sizeof"! '(' IDENT/sizeof ')' | 106 | '{'! (qexpr ':' qexpr) %= ','/ARITY ?',' '}'/map | 107 | '^' ifactor/bitnot | 108 | '-' ifactor/neg | 109 | '+' ifactor 110 | 111 | cexpr = INT/cpushi 112 | 113 | const = (IDENT '=' cexpr ';')/const 114 | 115 | doc = +( 116 | (IDENT '=' expr/xline ';')/assign | 117 | "const" '(' *const ')' ';') 118 | ` 119 | 120 | var ( 121 | // ErrNoDoc is returned when `doc` is undefined. 122 | ErrNoDoc = errors.New("no doc") 123 | ) 124 | 125 | // ----------------------------------------------------------------------------- 126 | 127 | // A Compiler compiles bpl source code to matching units. 128 | // 129 | type Compiler struct { 130 | code exec.Code 131 | stk []interface{} 132 | rulers map[string]bpl.Ruler 133 | vars map[string]*bpl.TypeVar 134 | consts map[string]interface{} 135 | gstk exec.Stack 136 | ipt interpreter.Engine 137 | idxStart int 138 | } 139 | 140 | func newCompiler() (p *Compiler) { 141 | 142 | rulers := make(map[string]bpl.Ruler) 143 | vars := make(map[string]*bpl.TypeVar) 144 | consts := make(map[string]interface{}) 145 | return &Compiler{rulers: rulers, vars: vars, consts: consts} 146 | } 147 | 148 | // Ret returns compiling result. 149 | // 150 | func (p *Compiler) Ret() (r Ruler, err error) { 151 | 152 | root, ok := p.rulers["doc"] 153 | if !ok { 154 | if v, ok := p.vars["doc"]; ok { 155 | root = v.Elem 156 | } else { 157 | return Ruler{}, ErrNoDoc 158 | } 159 | } 160 | for name, v := range p.vars { 161 | if v.Elem == nil { 162 | err = fmt.Errorf("variable `%s` is not assigned", name) 163 | return 164 | } 165 | } 166 | return Ruler{Impl: root}, nil 167 | } 168 | 169 | // Grammar returns the qlang compiler's grammar. It is required by tpl.Interpreter engine. 170 | // 171 | func (p *Compiler) Grammar() string { 172 | 173 | return grammar 174 | } 175 | 176 | // Stack returns nil (no stack). It is required by tpl.Interpreter engine. 177 | // 178 | func (p *Compiler) Stack() interpreter.Stack { 179 | 180 | return nil 181 | } 182 | 183 | // ----------------------------------------------------------------------------- 184 | 185 | var exports = map[string]interface{}{ 186 | "$And": (*Compiler).and, 187 | "$Seq": (*Compiler).seq, 188 | "$istart": (*Compiler).istart, 189 | "$iend": (*Compiler).iend, 190 | "$array": (*Compiler).array, 191 | "$array1": (*Compiler).array1, 192 | "$array0": (*Compiler).array0, 193 | "$array01": (*Compiler).repeat01, 194 | "$var": (*Compiler).variable, 195 | "$ident": (*Compiler).ident, 196 | "$assign": (*Compiler).assign, 197 | "$repeat0": (*Compiler).repeat0, 198 | "$repeat1": (*Compiler).repeat1, 199 | "$repeat01": (*Compiler).repeat01, 200 | 201 | "$iand": and, 202 | "$ior": or, 203 | 204 | "$sizeof": (*Compiler).sizeof, 205 | "$map": (*Compiler).fnMap, 206 | "$slice": (*Compiler).fnSlice, 207 | "$index": (*Compiler).index, 208 | "$ARITY": (*Compiler).arity, 209 | "$call": (*Compiler).call, 210 | "$ref": (*Compiler).ref, 211 | "$mref": (*Compiler).mref, 212 | "$pushi": (*Compiler).pushi, 213 | "$pushs": (*Compiler).pushs, 214 | "$pushc": (*Compiler).pushc, 215 | "$cpushi": (*Compiler).cpushi, 216 | "$let": (*Compiler).fnLet, 217 | "$global": (*Compiler).fnGlobal, 218 | "$eval": (*Compiler).fnEval, 219 | "$do": (*Compiler).fnDo, 220 | "$if": (*Compiler).fnIf, 221 | "$read": (*Compiler).fnRead, 222 | "$skip": (*Compiler).fnSkip, 223 | "$return": (*Compiler).fnReturn, 224 | "$case": (*Compiler).fnCase, 225 | "$assert": (*Compiler).fnAssert, 226 | "$fatal": (*Compiler).fnFatal, 227 | "$dump": (*Compiler).fnDump, 228 | "$const": (*Compiler).fnConst, 229 | "$casei": (*Compiler).casei, 230 | "$cases": (*Compiler).cases, 231 | "$source": (*Compiler).source, 232 | "$member": (*Compiler).member, 233 | "$struct": (*Compiler).gostruct, 234 | "$qline": (*Compiler).codeLine, 235 | "$xline": (*Compiler).xline, 236 | 237 | "exit": exit, 238 | } 239 | 240 | var builtins = map[string]bpl.Ruler{ 241 | "int8": bpl.Int8, 242 | "int16": bpl.Int16, 243 | "int32": bpl.Int32, 244 | "int64": bpl.Int64, 245 | "uint8": bpl.Uint8, 246 | "byte": bpl.Uint8, 247 | "char": bpl.Char, 248 | "uint16": bpl.Uint16, 249 | "uint24": bpl.Uint24, 250 | "uint32": bpl.Uint32, 251 | "uint64": bpl.Uint64, 252 | "uint16be": bpl.Uintbe(2), 253 | "uint24be": bpl.Uintbe(3), 254 | "uint32be": bpl.Uintbe(4), 255 | "uint64be": bpl.Uintbe(8), 256 | "uint16le": bpl.Uint16, 257 | "uint24le": bpl.Uint24, 258 | "uint32le": bpl.Uint32, 259 | "uint64le": bpl.Uint64, 260 | "float32": bpl.Float32, 261 | "float64": bpl.Float64, 262 | "float32le": bpl.Float32, 263 | "float64le": bpl.Float64, 264 | "float32be": bpl.Float32be, 265 | "float64be": bpl.Float64be, 266 | "cstring": bpl.CString, 267 | "nil": bpl.Nil, 268 | "eof": bpl.EOF, 269 | "done": bpl.Done, 270 | "bson": bson.Type, 271 | "dump": dump(0), 272 | } 273 | 274 | // ----------------------------------------------------------------------------- 275 | -------------------------------------------------------------------------------- /bpl.ext/cl_ext.go: -------------------------------------------------------------------------------- 1 | package bpl 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strconv" 7 | "strings" 8 | 9 | "github.com/goplus/bpl" 10 | "github.com/qiniu/text/tpl/interpreter.util" 11 | "github.com/xushiwei/qlang/exec" 12 | ) 13 | 14 | // ----------------------------------------------------------------------------- 15 | 16 | func (p *Compiler) eval(ctx *bpl.Context, start, end int) interface{} { 17 | 18 | vars, hasDom := ctx.Dom().(map[string]interface{}) 19 | if vars == nil { 20 | vars = make(map[string]interface{}) 21 | } 22 | code := &p.code 23 | stk := ctx.Stack 24 | parent := exec.NewSimpleContext(ctx.Globals.Impl, nil, nil, nil) 25 | ectx := exec.NewSimpleContext(vars, stk, code, parent) 26 | code.Exec(start, end, stk, ectx) 27 | if !hasDom && len(vars) > 0 { // update dom 28 | ctx.SetDom(vars) 29 | } 30 | v, _ := stk.Pop() 31 | return v 32 | } 33 | 34 | // ----------------------------------------------------------------------------- 35 | 36 | type exprBlock struct { 37 | start int 38 | end int 39 | } 40 | 41 | func (p *Compiler) istart() { 42 | 43 | p.idxStart = p.code.Len() 44 | } 45 | 46 | func (p *Compiler) iend() { 47 | 48 | end := p.code.Len() 49 | p.gstk.Push(&exprBlock{start: p.idxStart, end: end}) 50 | } 51 | 52 | func (p *Compiler) popExpr() *exprBlock { 53 | 54 | if v, ok := p.gstk.Pop(); ok { 55 | if e, ok := v.(*exprBlock); ok { 56 | return e 57 | } 58 | } 59 | panic("no index expression") 60 | } 61 | 62 | func (p *Compiler) array() { 63 | 64 | e := p.popExpr() 65 | stk := p.stk 66 | i := len(stk) - 1 67 | n := func(ctx *bpl.Context) int { 68 | v := p.eval(ctx.Parent, e.start, e.end) 69 | return toInt(v, "index isn't an integer expression") 70 | } 71 | stk[i] = bpl.Dynarray(stk[i].(bpl.Ruler), n) 72 | } 73 | 74 | func (p *Compiler) array0() { 75 | 76 | stk := p.stk 77 | i := len(stk) - 1 78 | stk[i] = bpl.Array0(stk[i].(bpl.Ruler)) 79 | } 80 | 81 | func (p *Compiler) array1() { 82 | 83 | stk := p.stk 84 | i := len(stk) - 1 85 | stk[i] = bpl.Array1(stk[i].(bpl.Ruler)) 86 | } 87 | 88 | // ----------------------------------------------------------------------------- 89 | 90 | func (p *Compiler) casei(v int) { 91 | 92 | p.gstk.Push(v) 93 | } 94 | 95 | func (p *Compiler) cases(lit string) { 96 | 97 | v, err := strconv.Unquote(lit) 98 | if err != nil { 99 | panic("invalid string `" + lit + "`: " + err.Error()) 100 | } 101 | p.gstk.Push(v) 102 | } 103 | 104 | func (p *Compiler) source(v interface{}) { 105 | 106 | p.gstk.Push(v) 107 | } 108 | 109 | func (p *Compiler) popRule() bpl.Ruler { 110 | 111 | n := len(p.stk) - 1 112 | r := p.stk[n].(bpl.Ruler) 113 | p.stk = p.stk[:n] 114 | return r 115 | } 116 | 117 | func (p *Compiler) popRules(m int) []bpl.Ruler { 118 | 119 | if m == 0 { 120 | return nil 121 | } 122 | stk := p.stk 123 | n := len(stk) 124 | rulers := clone(stk[n-m:]) 125 | p.stk = stk[:n-m] 126 | return rulers 127 | } 128 | 129 | func sourceOf(engine interpreter.Engine, src interface{}) string { 130 | 131 | b := engine.Source(src) 132 | return strings.Trim(string(b), " \t\r\n") 133 | } 134 | 135 | func (p *Compiler) fnCase(engine interpreter.Engine) { 136 | 137 | var defaultR bpl.Ruler 138 | if p.popArity() != 0 { 139 | defaultR = p.popRule() 140 | } 141 | 142 | arity := p.popArity() 143 | 144 | stk := p.stk 145 | n := len(stk) 146 | caseRs := clone(stk[n-arity:]) 147 | caseExprAndSources := p.gstk.PopNArgs(arity << 1) 148 | e := p.popExpr() 149 | srcSw, _ := p.gstk.Pop() 150 | r := func(ctx *bpl.Context) (bpl.Ruler, error) { 151 | v := p.eval(ctx, e.start, e.end) 152 | for i := 0; i < len(caseExprAndSources); i += 2 { 153 | expr := caseExprAndSources[i] 154 | if eq(v, expr) { 155 | if SetCaseType { 156 | key := sourceOf(engine, srcSw) 157 | val := sourceOf(engine, caseExprAndSources[i+1]) 158 | ctx.SetVar(key+".kind", val) 159 | } 160 | return caseRs[i>>1], nil 161 | } 162 | } 163 | if defaultR != nil { 164 | return defaultR, nil 165 | } 166 | return nil, fmt.Errorf("case `%s(=%v)` is not found", sourceOf(engine, srcSw), v) 167 | } 168 | stk[n-arity] = bpl.Dyntype(r) 169 | p.stk = stk[:n-arity+1] 170 | } 171 | 172 | // ----------------------------------------------------------------------------- 173 | 174 | func (p *Compiler) fnIf() { 175 | 176 | var elseR bpl.Ruler 177 | if p.popArity() != 0 { 178 | elseR = p.popRule() 179 | } else { 180 | elseR = bpl.Nil 181 | } 182 | 183 | arity := p.popArity() + 1 184 | 185 | stk := p.stk 186 | n := len(stk) 187 | bodyRs := clone(stk[n-arity:]) 188 | condExprs := p.gstk.PopNArgs(arity) 189 | 190 | arityOptimized := 0 191 | for i := 0; i < arity; i++ { 192 | e := condExprs[i].(*exprBlock) 193 | if e.end-e.start == 1 { 194 | if v, ok := p.code.CheckConst(e.start); ok { 195 | if toBool(v, "condition isn't a boolean expression") { // true 196 | arityOptimized = i 197 | elseR = bodyRs[i] 198 | break 199 | } 200 | continue // false 201 | } 202 | } 203 | if arityOptimized != i { 204 | bodyRs[arityOptimized] = bodyRs[i] 205 | condExprs[arityOptimized] = condExprs[i] 206 | } 207 | arityOptimized++ 208 | } 209 | 210 | dynR := elseR 211 | if arityOptimized > 0 { 212 | r := func(ctx *bpl.Context) (bpl.Ruler, error) { 213 | for i := 0; i < arityOptimized; i++ { 214 | e := condExprs[i].(*exprBlock) 215 | v := p.eval(ctx, e.start, e.end) 216 | if toBool(v, "condition isn't a boolean expression") { 217 | return bodyRs[i], nil 218 | } 219 | } 220 | return elseR, nil 221 | } 222 | dynR = bpl.Dyntype(r) 223 | } 224 | stk[n-arity] = dynR 225 | p.stk = stk[:n-arity+1] 226 | } 227 | 228 | // ----------------------------------------------------------------------------- 229 | 230 | func (p *Compiler) fnEval() { 231 | 232 | e := p.popExpr() 233 | stk := p.stk 234 | i := len(stk) - 1 235 | expr := func(ctx *bpl.Context) interface{} { 236 | return p.eval(ctx, e.start, e.end) 237 | } 238 | stk[i] = bpl.Eval(expr, stk[i].(bpl.Ruler)) 239 | } 240 | 241 | // ----------------------------------------------------------------------------- 242 | 243 | func (p *Compiler) fnDo() { 244 | 245 | e := p.popExpr() 246 | fn := func(ctx *bpl.Context) error { 247 | p.eval(ctx, e.start, e.end) 248 | return nil 249 | } 250 | p.stk = append(p.stk, bpl.Do(fn)) 251 | } 252 | 253 | // ----------------------------------------------------------------------------- 254 | 255 | func (p *Compiler) fnLet() { 256 | 257 | e := p.popExpr() 258 | arity := p.popArity() 259 | stk := p.stk 260 | n := len(stk) - arity 261 | if arity == 1 { 262 | name := stk[n].(string) 263 | fn := func(ctx *bpl.Context) error { 264 | v := p.eval(ctx, e.start, e.end) 265 | ctx.LetVar(name, v) 266 | return nil 267 | } 268 | stk[n] = bpl.Do(fn) 269 | } else { 270 | names := cloneNames(stk[n:]) 271 | fn := func(ctx *bpl.Context) error { 272 | v := p.eval(ctx, e.start, e.end) 273 | multiAssignFromSlice(names, v, ctx) 274 | return nil 275 | } 276 | stk[n] = bpl.Do(fn) 277 | p.stk = stk[:n+1] 278 | } 279 | } 280 | 281 | func multiAssignFromSlice(names []string, val interface{}, ctx *bpl.Context) { 282 | 283 | v := reflect.ValueOf(val) 284 | if v.Kind() != reflect.Slice { 285 | panic("expression of multi assignment must be a slice") 286 | } 287 | 288 | n := v.Len() 289 | arity := len(names) 290 | if arity != n { 291 | panic(fmt.Errorf("multi assignment error: require %d variables, but we got %d", n, arity)) 292 | } 293 | 294 | for i, name := range names { 295 | ctx.LetVar(name, v.Index(i).Interface()) 296 | } 297 | } 298 | 299 | // ----------------------------------------------------------------------------- 300 | 301 | func (p *Compiler) fnGlobal() { 302 | 303 | e := p.popExpr() 304 | stk := p.stk 305 | i := len(stk) - 1 306 | name := stk[i].(string) 307 | fn := func(ctx *bpl.Context) error { 308 | v := p.eval(ctx, e.start, e.end) 309 | ctx.Globals.SetVar(name, v) 310 | return nil 311 | } 312 | stk[i] = bpl.Do(fn) 313 | } 314 | 315 | // ----------------------------------------------------------------------------- 316 | 317 | func (p *Compiler) fnAssert(src interface{}) { 318 | 319 | e := p.popExpr() 320 | expr := func(ctx *bpl.Context) bool { 321 | v := p.eval(ctx, e.start, e.end) 322 | return toBool(v, "assert condition isn't a boolean expression") 323 | } 324 | msg := sourceOf(p.ipt, src) 325 | p.stk = append(p.stk, bpl.Assert(expr, msg)) 326 | } 327 | 328 | func (p *Compiler) fnFatal(src interface{}) { 329 | 330 | e := p.popExpr() 331 | r := func(ctx *bpl.Context) (bpl.Ruler, error) { 332 | val := p.eval(ctx, e.start, e.end) 333 | if v, ok := val.(string); ok { 334 | panic("fatal: " + v) 335 | } 336 | panic("fatal must return a string") 337 | } 338 | p.stk = append(p.stk, bpl.Dyntype(r)) 339 | //p.stk = append(p.stk, bpl.And(dump(0), bpl.Dyntype(r))) 340 | } 341 | 342 | func (p *Compiler) fnDump() { 343 | 344 | p.stk = append(p.stk, bpl.Ruler(dump(0))) 345 | } 346 | 347 | // ----------------------------------------------------------------------------- 348 | 349 | func (p *Compiler) fnRead() { 350 | 351 | e := p.popExpr() 352 | stk := p.stk 353 | i := len(stk) - 1 354 | n := func(ctx *bpl.Context) int { 355 | v := p.eval(ctx, e.start, e.end) 356 | return toInt(v, "read bytes isn't an integer expression") 357 | } 358 | stk[i] = bpl.Read(n, stk[i].(bpl.Ruler)) 359 | } 360 | 361 | func (p *Compiler) fnSkip() { 362 | 363 | e := p.popExpr() 364 | n := func(ctx *bpl.Context) int { 365 | v := p.eval(ctx, e.start, e.end) 366 | return toInt(v, "skip bytes isn't an integer expression") 367 | } 368 | p.stk = append(p.stk, bpl.Skip(n)) 369 | } 370 | 371 | // ----------------------------------------------------------------------------- 372 | 373 | func (p *Compiler) fnReturn() { 374 | 375 | e := p.popExpr() 376 | fnRet := func(ctx *bpl.Context) (v interface{}, err error) { 377 | v = p.eval(ctx, e.start, e.end) 378 | return 379 | } 380 | p.stk = append(p.stk, bpl.Return(fnRet)) 381 | } 382 | 383 | // ----------------------------------------------------------------------------- 384 | 385 | func (p *Compiler) member(name string) { 386 | 387 | stk := p.stk 388 | i := len(stk) - 1 389 | stk[i] = &bpl.Member{Name: name, Type: stk[i].(bpl.Ruler)} 390 | } 391 | 392 | func (p *Compiler) gostruct() { 393 | 394 | m := p.popArity() 395 | rulers := p.popRules(m) 396 | p.stk = append(p.stk, bpl.Struct(rulers)) 397 | } 398 | 399 | // ----------------------------------------------------------------------------- 400 | -------------------------------------------------------------------------------- /bpl.ext/cl_ext_test.go: -------------------------------------------------------------------------------- 1 | package bpl 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/goplus/bpl/binary" 8 | "github.com/qiniu/x/bufiox" 9 | ) 10 | 11 | // ----------------------------------------------------------------------------- 12 | 13 | const codeArray = ` 14 | 15 | sub1 = int8 uint16 16 | 17 | subType = { 18 | array [2]cstring 19 | } 20 | 21 | doc = [sub1 uint32 float32 cstring subType float64] 22 | ` 23 | 24 | type subType2 struct { 25 | Foo string 26 | Bar string 27 | } 28 | 29 | type fooType2 struct { 30 | A int8 31 | B uint16 32 | C uint32 33 | D float32 34 | E string 35 | F subType2 36 | G float64 37 | } 38 | 39 | func TestArray(t *testing.T) { 40 | 41 | foo := &fooType2{ 42 | A: 1, B: 2, C: 3, D: 3.14, E: "Hello", F: subType2{Foo: "foo", Bar: "bar"}, G: 7.52, 43 | } 44 | b, err := binary.Marshal(&foo) 45 | if err != nil { 46 | t.Fatal("binary.Marshal failed:", err) 47 | } 48 | if len(b) != 33 { 49 | t.Fatal("len(b) != 33, len:", len(b), "data:", string(b)) 50 | } 51 | 52 | r, err := NewFromString(codeArray, "") 53 | if err != nil { 54 | t.Fatal("New failed:", err) 55 | } 56 | v, err := r.MatchBuffer(b) 57 | if err != nil { 58 | t.Fatal("Match failed:", err, "len:", len(b)) 59 | } 60 | ret, err := json.Marshal(v) 61 | if err != nil { 62 | t.Fatal("json.Marshal failed:", err) 63 | } 64 | if string(ret) != `[null,3,3.14,"Hello",{"array":["foo","bar"]},7.52]` { 65 | t.Fatal("ret:", string(ret)) 66 | } 67 | } 68 | 69 | // ----------------------------------------------------------------------------- 70 | 71 | const codeArray2 = ` 72 | 73 | headerType = { 74 | type int32 75 | _ int32 76 | n int32 77 | m int32 78 | } 79 | 80 | recType = { 81 | h headerType 82 | array [h.n + h.m]cstring 83 | } 84 | 85 | doc = [int32] *[recType] 86 | ` 87 | 88 | type headerType struct { 89 | Type int32 90 | Len int32 91 | N int32 92 | M int32 93 | } 94 | 95 | type recType1 struct { 96 | H headerType 97 | A1 string 98 | A2 string 99 | A3 string 100 | } 101 | 102 | type recType2 struct { 103 | H headerType 104 | A1 string 105 | A2 string 106 | } 107 | 108 | type fooType3 struct { 109 | N int32 110 | R1 recType1 111 | R2 recType2 112 | } 113 | 114 | func TestArray2(t *testing.T) { 115 | 116 | foo := &fooType3{ 117 | N: 2, 118 | R1: recType1{ 119 | H: headerType{ 120 | Type: 1, 121 | N: 1, 122 | M: 2, 123 | }, 124 | A1: "hello", 125 | A2: "world", 126 | A3: "bpl", 127 | }, 128 | R2: recType2{ 129 | H: headerType{ 130 | Type: 2, 131 | N: 1, 132 | M: 1, 133 | }, 134 | A1: "foo", 135 | A2: "bar", 136 | }, 137 | } 138 | b, err := binary.Marshal(&foo) 139 | if err != nil { 140 | t.Fatal("binary.Marshal failed:", err) 141 | } 142 | if len(b) != 60 { 143 | t.Fatal("len(b) != 60, len:", len(b), "data:", string(b)) 144 | } 145 | 146 | r, err := NewFromString(codeArray2, "") 147 | if err != nil { 148 | t.Fatal("New failed:", err) 149 | } 150 | v, err := r.MatchBuffer(b) 151 | if err != nil { 152 | t.Fatal("Match failed:", err, "v:", v) 153 | } 154 | ret, err := json.Marshal(v) 155 | if err != nil { 156 | t.Fatal("json.Marshal failed:", err) 157 | } 158 | if string(ret) != `[2,{"array":["hello","world","bpl"],"h":{"m":2,"n":1,"type":1}},{"array":["foo","bar"],"h":{"m":1,"n":1,"type":2}}]` { 159 | t.Fatal("ret:", string(ret)) 160 | } 161 | } 162 | 163 | // ----------------------------------------------------------------------------- 164 | 165 | const codeCase = ` 166 | 167 | headerType = { 168 | type int32 169 | _ int32 170 | n int32 171 | m int32 172 | } 173 | 174 | recType = { 175 | h headerType 176 | case h.type { 177 | 1: {t1 [3]cstring} 178 | 2: {t2 [2]cstring} 179 | } 180 | } 181 | 182 | doc = [int32] *[recType] 183 | ` 184 | 185 | func TestCase(t *testing.T) { 186 | 187 | SetCaseType = false 188 | foo := &fooType3{ 189 | N: 2, 190 | R1: recType1{ 191 | H: headerType{ 192 | Type: 1, 193 | N: 1, 194 | M: 2, 195 | }, 196 | A1: "hello", 197 | A2: "world", 198 | A3: "bpl", 199 | }, 200 | R2: recType2{ 201 | H: headerType{ 202 | Type: 2, 203 | N: 1, 204 | M: 1, 205 | }, 206 | A1: "foo", 207 | A2: "bar", 208 | }, 209 | } 210 | b, err := binary.Marshal(&foo) 211 | if err != nil { 212 | t.Fatal("binary.Marshal failed:", err) 213 | } 214 | if len(b) != 60 { 215 | t.Fatal("len(b) != 60, len:", len(b), "data:", string(b)) 216 | } 217 | 218 | r, err := NewFromString(codeCase, "") 219 | if err != nil { 220 | t.Fatal("New failed:", err) 221 | } 222 | v, err := r.MatchBuffer(b) 223 | if err != nil { 224 | t.Fatal("Match failed:", err) 225 | } 226 | ret, err := json.Marshal(v) 227 | if err != nil { 228 | t.Fatal("json.Marshal failed:", err) 229 | } 230 | if string(ret) != `[2,{"h":{"m":2,"n":1,"type":1},"t1":["hello","world","bpl"]},{"h":{"m":1,"n":1,"type":2},"t2":["foo","bar"]}]` { 231 | t.Fatal("ret:", string(ret)) 232 | } 233 | } 234 | 235 | // ----------------------------------------------------------------------------- 236 | 237 | const codeCase2 = ` 238 | 239 | const ( 240 | NVAL = 1 241 | ) 242 | 243 | headerType = { 244 | type int32 245 | do println("type:", type) 246 | 247 | _ int32 248 | n int32 249 | m int32 250 | assert n == NVAL 251 | } 252 | 253 | recType = {h headerType} case h.type { 254 | 1: {t1 [3]cstring} 255 | 2: {t2 [2]cstring} 256 | } 257 | 258 | doc = [int32] *[recType] 259 | ` 260 | 261 | func TestCase2(t *testing.T) { 262 | 263 | SetCaseType = false 264 | foo := &fooType3{ 265 | N: 2, 266 | R1: recType1{ 267 | H: headerType{ 268 | Type: 1, 269 | N: 1, 270 | M: 2, 271 | }, 272 | A1: "hello", 273 | A2: "world", 274 | A3: "bpl", 275 | }, 276 | R2: recType2{ 277 | H: headerType{ 278 | Type: 2, 279 | N: 1, 280 | M: 1, 281 | }, 282 | A1: "foo", 283 | A2: "bar", 284 | }, 285 | } 286 | b, err := binary.Marshal(&foo) 287 | if err != nil { 288 | t.Fatal("binary.Marshal failed:", err) 289 | } 290 | if len(b) != 60 { 291 | t.Fatal("len(b) != 60, len:", len(b), "data:", string(b)) 292 | } 293 | 294 | r, err := NewFromString(codeCase2, "") 295 | if err != nil { 296 | t.Fatal("New failed:", err) 297 | } 298 | v, err := r.MatchBuffer(b) 299 | if err != nil { 300 | t.Fatal("Match failed:", err) 301 | } 302 | ret, err := json.Marshal(v) 303 | if err != nil { 304 | t.Fatal("json.Marshal failed:", err) 305 | } 306 | if string(ret) != `[2,{"h":{"m":2,"n":1,"type":1},"t1":["hello","world","bpl"]},{"h":{"m":1,"n":1,"type":2},"t2":["foo","bar"]}]` { 307 | t.Fatal("ret:", string(ret)) 308 | } 309 | } 310 | 311 | // ----------------------------------------------------------------------------- 312 | 313 | const codeRead = ` 314 | 315 | headerType = { 316 | type int32 317 | len int32 318 | _ int32 319 | _ int32 320 | } 321 | 322 | recType = { 323 | h headerType 324 | read h.len - sizeof(headerType) do case h.type { 325 | 1: {t1 [2]cstring} 326 | 2: {t2 [1]cstring} 327 | } 328 | } 329 | 330 | doc = [int32] *[recType] 331 | ` 332 | 333 | func TestRead(t *testing.T) { 334 | 335 | SetCaseType = false 336 | foo := &fooType3{ 337 | N: 2, 338 | R1: recType1{ 339 | H: headerType{ 340 | Type: 1, 341 | Len: 16 + 16, 342 | N: 1, 343 | M: 2, 344 | }, 345 | A1: "hello", // 6 346 | A2: "world", // 6 347 | A3: "bpl", // 4 348 | }, 349 | R2: recType2{ 350 | H: headerType{ 351 | Type: 2, 352 | Len: 16 + 8, 353 | N: 1, 354 | M: 1, 355 | }, 356 | A1: "foo", // 4 357 | A2: "bar", // 4 358 | }, 359 | } 360 | b, err := binary.Marshal(&foo) 361 | if err != nil { 362 | t.Fatal("binary.Marshal failed:", err) 363 | } 364 | if len(b) != 60 { 365 | t.Fatal("len(b) != 60, len:", len(b), "data:", string(b)) 366 | } 367 | 368 | r, err := NewFromString(codeRead, "") 369 | if err != nil { 370 | t.Fatal("New failed:", err) 371 | } 372 | v, err := r.MatchBuffer(b) 373 | if err != nil { 374 | t.Fatal("Match failed:", err) 375 | } 376 | ret, err := json.Marshal(v) 377 | if err != nil { 378 | t.Fatal("json.Marshal failed:", err) 379 | } 380 | if string(ret) != `[2,{"h":{"len":32,"type":1},"t1":["hello","world"]},{"h":{"len":24,"type":2},"t2":["foo"]}]` { 381 | t.Fatal("ret:", string(ret)) 382 | } 383 | } 384 | 385 | // ----------------------------------------------------------------------------- 386 | 387 | const codeIf = ` 388 | 389 | headerType = { 390 | type int32 391 | len int32 392 | _ int32 393 | _ int32 394 | } 395 | 396 | recType = { 397 | h headerType 398 | read h.len - sizeof(headerType) do { 399 | if h.type == 1 { 400 | t1 [2]cstring 401 | } elif h.type == 2 { 402 | t2 [1]cstring 403 | } 404 | } 405 | } 406 | 407 | doc = [int32] *[recType] 408 | ` 409 | 410 | func TestIf(t *testing.T) { 411 | 412 | foo := &fooType3{ 413 | N: 2, 414 | R1: recType1{ 415 | H: headerType{ 416 | Type: 1, 417 | Len: 16 + 16, 418 | N: 1, 419 | M: 2, 420 | }, 421 | A1: "hello", // 6 422 | A2: "world", // 6 423 | A3: "bpl", // 4 424 | }, 425 | R2: recType2{ 426 | H: headerType{ 427 | Type: 2, 428 | Len: 16 + 8, 429 | N: 1, 430 | M: 1, 431 | }, 432 | A1: "foo", // 4 433 | A2: "bar", // 4 434 | }, 435 | } 436 | b, err := binary.Marshal(&foo) 437 | if err != nil { 438 | t.Fatal("binary.Marshal failed:", err) 439 | } 440 | if len(b) != 60 { 441 | t.Fatal("len(b) != 60, len:", len(b), "data:", string(b)) 442 | } 443 | 444 | r, err := NewFromString(codeIf, "") 445 | if err != nil { 446 | t.Fatal("New failed:", err) 447 | } 448 | v, err := r.MatchBuffer(b) 449 | if err != nil { 450 | t.Fatal("Match failed:", err) 451 | } 452 | ret, err := json.Marshal(v) 453 | if err != nil { 454 | t.Fatal("json.Marshal failed:", err) 455 | } 456 | if string(ret) != `[2,{"h":{"len":32,"type":1},"t1":["hello","world"]},{"h":{"len":24,"type":2},"t2":["foo"]}]` { 457 | t.Fatal("ret:", string(ret)) 458 | } 459 | } 460 | 461 | // ----------------------------------------------------------------------------- 462 | 463 | const codeLet = ` 464 | 465 | doc = { 466 | let a = 1 467 | let a, b = [2, 3] 468 | } 469 | ` 470 | 471 | func TestLet(t *testing.T) { 472 | 473 | r, err := NewFromString(codeLet, "") 474 | if err != nil { 475 | t.Fatal("New failed:", err) 476 | } 477 | v, err := r.MatchBuffer(nil) 478 | if err != nil { 479 | t.Fatal("Match failed:", err) 480 | } 481 | ret, err := json.Marshal(v) 482 | if err != nil { 483 | t.Fatal("json.Marshal failed:", err) 484 | } 485 | if string(ret) != `{"a":2,"b":3}` { 486 | t.Fatal("ret:", string(ret)) 487 | } 488 | } 489 | 490 | // ----------------------------------------------------------------------------- 491 | 492 | const codeRet = ` 493 | 494 | uint32be = { 495 | return 1 496 | } 497 | 498 | doc = { 499 | a uint32be 500 | } 501 | ` 502 | 503 | func TestReturn(t *testing.T) { 504 | 505 | r, err := NewFromString(codeRet, "") 506 | if err != nil { 507 | t.Fatal("New failed:", err) 508 | } 509 | v, err := r.MatchBuffer(nil) 510 | if err != nil { 511 | t.Fatal("Match failed:", err) 512 | } 513 | ret, err := json.Marshal(v) 514 | if err != nil { 515 | t.Fatal("json.Marshal failed:", err) 516 | } 517 | if string(ret) != `{"a":1}` { 518 | t.Fatal("ret:", string(ret)) 519 | } 520 | } 521 | 522 | // ----------------------------------------------------------------------------- 523 | 524 | const codeRet2 = ` 525 | 526 | uint32be = { 527 | b4 uint8 528 | b3 uint8 529 | b2 uint8 530 | b1 uint8 531 | if true { 532 | return (b4 << 24) | (b3 << 16) | (b2 << 8) | b1 533 | } 534 | } 535 | 536 | doc = { 537 | a uint32be 538 | } 539 | ` 540 | 541 | func TestReturn2(t *testing.T) { 542 | 543 | b := []byte{1, 2, 3, 4} 544 | 545 | r, err := NewFromString(codeRet2, "") 546 | if err != nil { 547 | t.Fatal("New failed:", err) 548 | } 549 | v, err := r.MatchBuffer(b) 550 | if err != nil { 551 | t.Fatal("Match failed:", err) 552 | } 553 | ret, err := json.Marshal(v) 554 | if err != nil { 555 | t.Fatal("json.Marshal failed:", err) 556 | } 557 | if string(ret) != `{"a":16909060}` { 558 | t.Fatal("ret:", string(ret)) 559 | } 560 | } 561 | 562 | // ----------------------------------------------------------------------------- 563 | 564 | const codeGlobal = ` 565 | 566 | record = { 567 | do set(msgs, 1234, 35, 123, 36) 568 | let a = msgs[123] 569 | } 570 | 571 | init = { 572 | global msgs = mkmap("int:int") 573 | } 574 | 575 | doc = init record 576 | ` 577 | 578 | func TestGlobal(t *testing.T) { 579 | 580 | b := []byte{1, 2, 3, 4} 581 | 582 | r, err := NewFromString(codeGlobal, "") 583 | if err != nil { 584 | t.Fatal("New failed:", err) 585 | } 586 | v, err := r.MatchBuffer(b) 587 | if err != nil { 588 | t.Fatal("Match failed:", err) 589 | } 590 | ret, err := json.Marshal(v) 591 | if err != nil { 592 | t.Fatal("json.Marshal failed:", err) 593 | } 594 | if string(ret) != `{"a":36}` { 595 | t.Fatal("ret:", string(ret)) 596 | } 597 | } 598 | 599 | // ----------------------------------------------------------------------------- 600 | 601 | const codeGlobal2 = ` 602 | 603 | record = { 604 | let a = 3 605 | let b = 4 606 | } 607 | 608 | init = { 609 | global a = 1 610 | } 611 | 612 | doc = init record 613 | ` 614 | 615 | func TestGlobal2(t *testing.T) { 616 | 617 | b := []byte{1, 2, 3, 4} 618 | 619 | r, err := NewFromString(codeGlobal2, "") 620 | if err != nil { 621 | t.Fatal("New failed:", err) 622 | } 623 | in := bufiox.NewReaderBuffer(b) 624 | ctx := NewContext() 625 | v, err := r.SafeMatch(in, ctx) 626 | if err != nil { 627 | t.Fatal("Match failed:", err) 628 | } 629 | ret, err := json.Marshal(v) 630 | if err != nil { 631 | t.Fatal("json.Marshal failed:", err) 632 | } 633 | if v, ok := ctx.Globals.Var("a"); !ok || v != 3 { 634 | t.Fatal("v != 3: v =", v, ok) 635 | } 636 | if string(ret) != `{"b":4}` { 637 | t.Fatal("ret:", string(ret)) 638 | } 639 | } 640 | 641 | // ----------------------------------------------------------------------------- 642 | 643 | const codeMap = ` 644 | 645 | record = { 646 | do set(msgs, "foo", 35, "bar", 36) 647 | do println("msgs:", msgs) 648 | let a = get(msgs, "bar") 649 | } 650 | 651 | init = { 652 | global msgs = {} 653 | } 654 | 655 | doc = init record 656 | ` 657 | 658 | func TestMap(t *testing.T) { 659 | 660 | b := []byte{1, 2, 3, 4} 661 | 662 | r, err := NewFromString(codeMap, "") 663 | if err != nil { 664 | t.Fatal("New failed:", err) 665 | } 666 | v, err := r.MatchBuffer(b) 667 | if err != nil { 668 | t.Fatal("Match failed:", err) 669 | } 670 | ret, err := json.Marshal(v) 671 | if err != nil { 672 | t.Fatal("json.Marshal failed:", err) 673 | } 674 | if string(ret) != `{"a":36}` { 675 | t.Fatal("ret:", string(ret)) 676 | } 677 | } 678 | 679 | // ----------------------------------------------------------------------------- 680 | 681 | const codeUnset = ` 682 | 683 | record = { 684 | let a = 1 685 | let b = 2 686 | if true { 687 | do unset("a") 688 | } 689 | } 690 | 691 | doc = record 692 | ` 693 | 694 | func TestUnset(t *testing.T) { 695 | 696 | b := []byte{1, 2, 3, 4} 697 | 698 | r, err := NewFromString(codeUnset, "") 699 | if err != nil { 700 | t.Fatal("New failed:", err) 701 | } 702 | v, err := r.MatchBuffer(b) 703 | if err != nil { 704 | t.Fatal("Match failed:", err) 705 | } 706 | ret, err := json.Marshal(v) 707 | if err != nil { 708 | t.Fatal("json.Marshal failed:", err) 709 | } 710 | if string(ret) != `{"b":2}` { 711 | t.Fatal("ret:", string(ret)) 712 | } 713 | } 714 | 715 | // ----------------------------------------------------------------------------- 716 | -------------------------------------------------------------------------------- /bpl.ext/qlang.go: -------------------------------------------------------------------------------- 1 | package bpl 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "reflect" 7 | "strconv" 8 | 9 | "github.com/xushiwei/qlang/exec" 10 | "github.com/xushiwei/qlang/lib/bytes" 11 | "github.com/xushiwei/qlang/lib/crypto/hmac" 12 | "github.com/xushiwei/qlang/lib/crypto/md5" 13 | "github.com/xushiwei/qlang/lib/crypto/sha1" 14 | "github.com/xushiwei/qlang/lib/crypto/sha256" 15 | "github.com/xushiwei/qlang/lib/encoding/hex" 16 | "github.com/xushiwei/qlang/lib/encoding/json" 17 | "github.com/xushiwei/qlang/lib/errors" 18 | "github.com/xushiwei/qlang/lib/io" 19 | "github.com/xushiwei/qlang/lib/strings" 20 | 21 | // import qlang builtin 22 | _ "github.com/xushiwei/qlang/lib/builtin" 23 | qstrconv "github.com/xushiwei/qlang/lib/strconv" 24 | qlang "github.com/xushiwei/qlang/spec" 25 | ) 26 | 27 | // ----------------------------------------------------------------------------- 28 | 29 | func exit(code int) { 30 | 31 | panic(code) 32 | } 33 | 34 | func init() { 35 | 36 | osExports := map[string]interface{}{ 37 | "exit": exit, 38 | } 39 | 40 | httpExports := map[string]interface{}{ 41 | "readRequest": http.ReadRequest, 42 | "readResponse": http.ReadResponse, 43 | } 44 | 45 | var ioutilExports = map[string]interface{}{ 46 | "nopCloser": ioutil.NopCloser, 47 | "readAll": ioutil.ReadAll, 48 | "discard": ioutil.Discard, 49 | } 50 | 51 | qlang.Import("", exports) 52 | qlang.Import("bytes", bytes.Exports) 53 | qlang.Import("md5", md5.Exports) 54 | qlang.Import("sha1", sha1.Exports) 55 | qlang.Import("sha256", sha256.Exports) 56 | qlang.Import("hmac", hmac.Exports) 57 | qlang.Import("errors", errors.Exports) 58 | qlang.Import("json", json.Exports) 59 | qlang.Import("hex", hex.Exports) 60 | qlang.Import("io", io.Exports) 61 | qlang.Import("ioutil", ioutilExports) 62 | qlang.Import("os", osExports) 63 | qlang.Import("http", httpExports) 64 | qlang.Import("strconv", qstrconv.Exports) 65 | qlang.Import("strings", strings.Exports) 66 | } 67 | 68 | // Fntable returns the qlang compiler's function table. It is required by tpl.Interpreter engine. 69 | // 70 | func (p *Compiler) Fntable() map[string]interface{} { 71 | 72 | return qlang.Fntable 73 | } 74 | 75 | // ----------------------------------------------------------------------------- 76 | 77 | func castInt(a interface{}) (int, bool) { 78 | 79 | switch a1 := a.(type) { 80 | case int: 81 | return a1, true 82 | case int32: 83 | return int(a1), true 84 | case int64: 85 | return int(a1), true 86 | case int16: 87 | return int(a1), true 88 | case int8: 89 | return int(a1), true 90 | case uint: 91 | return int(a1), true 92 | case uint32: 93 | return int(a1), true 94 | case uint64: 95 | return int(a1), true 96 | case uint16: 97 | return int(a1), true 98 | case uint8: 99 | return int(a1), true 100 | } 101 | return 0, false 102 | } 103 | 104 | func toInt(a interface{}, msg string) int { 105 | 106 | if v, ok := castInt(a); ok { 107 | return v 108 | } 109 | panic(msg) 110 | } 111 | 112 | func toBool(a interface{}, msg string) bool { 113 | 114 | if v, ok := a.(bool); ok { 115 | return v 116 | } 117 | if v, ok := castInt(a); ok { 118 | return v != 0 119 | } 120 | panic(msg) 121 | } 122 | 123 | // CallFn generates a function call instruction. It is required by tpl.Interpreter engine. 124 | // 125 | func (p *Compiler) CallFn(fn interface{}) { 126 | 127 | p.code.Block(exec.Call(fn)) 128 | } 129 | 130 | func eq(a, b interface{}) bool { 131 | 132 | if a1, ok := castInt(a); ok { 133 | switch b1 := b.(type) { 134 | case int: 135 | return a1 == b1 136 | } 137 | } 138 | if a1, ok := a.(string); ok { 139 | switch b1 := b.(type) { 140 | case string: 141 | return a1 == b1 142 | } 143 | } 144 | panicUnsupportedOp2("==", a, b) 145 | return false 146 | } 147 | 148 | func and(a, b bool) bool { 149 | 150 | return a && b 151 | } 152 | 153 | func or(a, b bool) bool { 154 | 155 | return a || b 156 | } 157 | 158 | func panicUnsupportedOp2(op string, a, b interface{}) interface{} { 159 | 160 | ta := typeString(a) 161 | tb := typeString(b) 162 | panic("unsupported operator: " + ta + op + tb) 163 | } 164 | 165 | func typeString(a interface{}) string { 166 | 167 | if a == nil { 168 | return "nil" 169 | } 170 | return reflect.TypeOf(a).String() 171 | } 172 | 173 | // ----------------------------------------------------------------------------- 174 | 175 | func (p *Compiler) popArity() int { 176 | 177 | return p.popConstInt() 178 | } 179 | 180 | func (p *Compiler) popConstInt() int { 181 | 182 | if v, ok := p.gstk.Pop(); ok { 183 | if val, ok := v.(int); ok { 184 | return val 185 | } 186 | } 187 | panic("no int") 188 | } 189 | 190 | func (p *Compiler) arity(arity int) { 191 | 192 | p.gstk.Push(arity) 193 | } 194 | 195 | func (p *Compiler) call() { 196 | 197 | variadic := p.popArity() 198 | arity := p.popArity() 199 | if variadic != 0 { 200 | if arity == 0 { 201 | panic("what do you mean of `...`?") 202 | } 203 | p.code.Block(exec.CallFnv(arity)) 204 | } else { 205 | p.code.Block(exec.CallFn(arity)) 206 | } 207 | } 208 | 209 | func (p *Compiler) ref(name string) { 210 | 211 | var instr exec.Instr 212 | if v, ok := p.consts[name]; ok { 213 | instr = exec.Push(v) 214 | } else { 215 | instr = exec.Ref(name) 216 | } 217 | p.code.Block(instr) 218 | } 219 | 220 | func (p *Compiler) mref(name string) { 221 | 222 | p.code.Block(exec.MemberRef(name)) 223 | } 224 | 225 | func (p *Compiler) pushi(v int) { 226 | 227 | p.code.Block(exec.Push(v)) 228 | } 229 | 230 | func (p *Compiler) pushs(lit string) { 231 | 232 | v, err := strconv.Unquote(lit) 233 | if err != nil { 234 | panic("invalid string `" + lit + "`: " + err.Error()) 235 | } 236 | p.code.Block(exec.Push(v)) 237 | } 238 | 239 | func (p *Compiler) pushc(lit string) { 240 | 241 | v, multibyte, tail, err := strconv.UnquoteChar(lit[1:len(lit)-1], '\'') 242 | if err != nil { 243 | panic("invalid char `" + lit + "`: " + err.Error()) 244 | } 245 | if tail != "" || multibyte { 246 | panic("invalid char: " + lit) 247 | } 248 | p.code.Block(exec.Push(byte(v))) 249 | } 250 | 251 | func (p *Compiler) cpushi(v int) { 252 | 253 | p.gstk.Push(v) 254 | } 255 | 256 | func (p *Compiler) fnConst(name string) { 257 | 258 | p.consts[name] = p.popConstInt() 259 | } 260 | 261 | // ----------------------------------------------------------------------------- 262 | 263 | func (p *Compiler) fnMap() { 264 | 265 | arity := p.popArity() 266 | p.code.Block(exec.Call(qlang.MapFrom, arity*2)) 267 | } 268 | 269 | func (p *Compiler) fnSlice() { 270 | 271 | arity := p.popArity() 272 | p.code.Block(exec.SliceFrom(arity)) 273 | } 274 | 275 | func (p *Compiler) index() { 276 | 277 | arity2 := p.popArity() 278 | arityMid := p.popArity() 279 | arity1 := p.popArity() 280 | 281 | if arityMid == 0 { 282 | if arity1 == 0 { 283 | panic("call operator[] without index") 284 | } 285 | p.code.Block(exec.Get) 286 | } else { 287 | p.code.Block(exec.Op3(qlang.SubSlice, arity1 != 0, arity2 != 0)) 288 | } 289 | } 290 | 291 | // ----------------------------------------------------------------------------- 292 | 293 | // DumpCode is mode how to dump code. 294 | // 1 means to dump code with `rem` instruction; 2 means to dump clean code; 0 means don't dump code. 295 | // 296 | var DumpCode int 297 | 298 | func (p *Compiler) codeLine(src interface{}) { 299 | 300 | ipt := p.ipt 301 | if ipt == nil { 302 | return 303 | } 304 | 305 | f := ipt.FileLine(src) 306 | p.code.CodeLine(f.File, f.Line) 307 | if DumpCode == 1 { 308 | text := string(ipt.Source(src)) 309 | p.code.Block(exec.Rem(f.File, f.Line, text)) 310 | } 311 | } 312 | 313 | // ----------------------------------------------------------------------------- 314 | -------------------------------------------------------------------------------- /bpl.ext/ruler.go: -------------------------------------------------------------------------------- 1 | package bpl 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/goplus/bpl" 7 | "github.com/xushiwei/qlang/exec" 8 | ) 9 | 10 | // ----------------------------------------------------------------------------- 11 | 12 | func clone(rs []interface{}) []bpl.Ruler { 13 | 14 | dest := make([]bpl.Ruler, len(rs)) 15 | for i, v := range rs { 16 | dest[i] = v.(bpl.Ruler) 17 | } 18 | return dest 19 | } 20 | 21 | func cloneNames(names []interface{}) []string { 22 | 23 | dest := make([]string, len(names)) 24 | for i, v := range names { 25 | dest[i] = v.(string) 26 | } 27 | return dest 28 | } 29 | 30 | func (p *Compiler) and(m int) { 31 | 32 | if m == 1 { 33 | return 34 | } 35 | stk := p.stk 36 | n := len(stk) 37 | stk[n-m] = bpl.And(clone(stk[n-m:])...) 38 | p.stk = stk[:n-m+1] 39 | } 40 | 41 | func (p *Compiler) seq(m int) { 42 | 43 | stk := p.stk 44 | n := len(stk) 45 | stk[n-m] = bpl.Seq(clone(stk[n-m:])...) 46 | p.stk = stk[:n-m+1] 47 | } 48 | 49 | func (p *Compiler) variable(name string) { 50 | 51 | p.stk = append(p.stk, name) 52 | } 53 | 54 | func (p *Compiler) ruleOf(name string) (r bpl.Ruler, ok bool) { 55 | 56 | r, ok = p.rulers[name] 57 | if !ok { 58 | if r, ok = p.vars[name]; !ok { 59 | if r, ok = builtins[name]; ok { 60 | p.rulers[name] = r 61 | } 62 | } 63 | } 64 | return 65 | } 66 | 67 | func (p *Compiler) sizeof(name string) { 68 | 69 | r, ok := p.ruleOf(name) 70 | if !ok { 71 | panic(fmt.Errorf("sizeof error: type `%v` not found", name)) 72 | } 73 | n := r.SizeOf() 74 | if n < 0 { 75 | panic(fmt.Errorf("sizeof error: type `%v` isn't a fixed size type", name)) 76 | } 77 | p.code.Block(exec.Push(n)) 78 | } 79 | 80 | func (p *Compiler) ident(name string) { 81 | 82 | r, ok := p.ruleOf(name) 83 | if !ok { 84 | v := &bpl.TypeVar{Name: name} 85 | p.vars[name] = v 86 | r = v 87 | } 88 | p.stk = append(p.stk, r) 89 | } 90 | 91 | func (p *Compiler) assign(name string) { 92 | 93 | a := p.stk[0].(bpl.Ruler) 94 | if v, ok := p.vars[name]; ok { 95 | if err := v.Assign(a); err != nil { 96 | panic(err) 97 | } 98 | } else if _, ok := p.rulers[name]; ok { 99 | panic("ruler already exists: " + name) 100 | } else { 101 | p.rulers[name] = a 102 | } 103 | p.stk = p.stk[:0] 104 | } 105 | 106 | func (p *Compiler) repeat0() { 107 | 108 | stk := p.stk 109 | i := len(stk) - 1 110 | stk[i] = bpl.Repeat0(stk[i].(bpl.Ruler)) 111 | } 112 | 113 | func (p *Compiler) repeat1() { 114 | 115 | stk := p.stk 116 | i := len(stk) - 1 117 | stk[i] = bpl.Repeat1(stk[i].(bpl.Ruler)) 118 | } 119 | 120 | func (p *Compiler) repeat01() { 121 | 122 | stk := p.stk 123 | i := len(stk) - 1 124 | stk[i] = bpl.Repeat01(stk[i].(bpl.Ruler)) 125 | } 126 | 127 | func (p *Compiler) xline(src interface{}) { 128 | 129 | f := p.ipt.FileLine(src) 130 | stk := p.stk 131 | i := len(stk) - 1 132 | stk[i] = bpl.FileLine(f.File, f.Line, stk[i].(bpl.Ruler)) 133 | } 134 | 135 | // ----------------------------------------------------------------------------- 136 | -------------------------------------------------------------------------------- /cmd/hexundump/hex_undump.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | 9 | "github.com/goplus/bpl/hex" 10 | ) 11 | 12 | // Usage: hexundump 13 | // 14 | func main() { 15 | 16 | if len(os.Args) < 3 { 17 | fmt.Fprintf(os.Stderr, "Usage: hexundump \n\n") 18 | return 19 | } 20 | b, err := ioutil.ReadFile(os.Args[1]) 21 | if err != nil { 22 | fmt.Fprintln(os.Stderr, err) 23 | os.Exit(1) 24 | } 25 | w := bytes.NewBuffer(nil) 26 | hex.UndumpText(w, string(b)) 27 | err = ioutil.WriteFile(os.Args[2], w.Bytes(), 0666) 28 | if err != nil { 29 | fmt.Fprintln(os.Stderr, err) 30 | os.Exit(2) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /cmd/qbpl/qbpl.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | 10 | bpl "github.com/goplus/bpl/bpl.ext" 11 | "github.com/qiniu/x/log" 12 | ) 13 | 14 | var ( 15 | protocol = flag.String("p", "", "protocol file in BPL syntax. default is guessed by extension.") 16 | output = flag.String("o", "", "output log file, default is stderr.") 17 | logmode = flag.String("l", "", "log mode: short (default) or long.") 18 | ) 19 | 20 | // qbpl [-p .bpl -o .log -l ] 21 | // 22 | func main() { 23 | 24 | flag.Parse() 25 | bpl.SetDumpCode(os.Getenv("BPL_DUMPCODE")) 26 | 27 | var in *bufio.Reader 28 | args := flag.Args() 29 | if len(args) > 0 { 30 | file := args[0] 31 | f, err := os.Open(file) 32 | if err != nil { 33 | fmt.Fprintln(os.Stderr, "Open failed:", file) 34 | } 35 | defer f.Close() 36 | in = bufio.NewReader(f) 37 | } else { 38 | in = bufio.NewReader(os.Stdin) 39 | } 40 | 41 | if *protocol == "" { 42 | if len(args) == 0 { 43 | fmt.Fprintln(os.Stderr, "Usage: qbpl [-p .bpl -o .log -l ] ") 44 | flag.PrintDefaults() 45 | return 46 | } 47 | ext := filepath.Ext(args[0]) 48 | if ext != "" { 49 | *protocol = os.Getenv("HOME") + "/.qbpl/formats/" + ext[1:] + ".bpl" 50 | } 51 | } 52 | 53 | logflags := bpl.Ldefault 54 | flong := (*logmode == "long") 55 | if flong { 56 | logflags = bpl.Llong 57 | } 58 | 59 | if *output != "" { 60 | f, err := os.Create(*output) 61 | if err != nil { 62 | log.Fatalln("Create log file failed:", err) 63 | } 64 | defer f.Close() 65 | bpl.SetDumper(f, logflags) 66 | } 67 | log.Std = bpl.Dumper 68 | 69 | ruler, err := bpl.NewFromFile(*protocol) 70 | if err != nil { 71 | log.Fatalln("bpl.NewFromFile failed:", err) 72 | } 73 | 74 | ctx := bpl.NewContext() 75 | _, err = ruler.SafeMatch(in, ctx) 76 | if err != nil { 77 | fmt.Fprintln(os.Stderr, "Match failed:", err) 78 | return 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /cmd/qbplgen/qbplgen.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | "strings" 10 | 11 | "github.com/goplus/bpl/go/codegen" 12 | ) 13 | 14 | // qbplgen .bpl qbpl|qbplproxy 15 | // 16 | func main() { 17 | 18 | if len(os.Args) < 3 { 19 | fmt.Fprintln(os.Stderr, "Usage: qbplgen .bpl qbpl|qbplproxy") 20 | return 21 | } 22 | 23 | protocol := os.Args[1] 24 | if path.Ext(protocol) == "" { 25 | baseDir := os.Getenv("HOME") + "/.qbpl/formats/" 26 | protocol = baseDir + protocol + ".bpl" 27 | } 28 | 29 | qbpl := os.Args[2] 30 | qdestname := strings.TrimPrefix(qbpl, "qbpl") 31 | srcbase := os.Getenv("QBOXROOT") + "/bpl/src/qiniupkg.com/text/" 32 | qbpl = srcbase + qbpl + "/" + qbpl + ".go" 33 | src, err := ioutil.ReadFile(qbpl) 34 | if err != nil { 35 | fmt.Fprintln(os.Stderr, err) 36 | os.Exit(1) 37 | } 38 | 39 | src = bytes.Replace(src, []byte("flag.Parse()\n"), []byte("flag.Parse()\n\t*protocol = \"_.bpl\"\n"), 1) 40 | src = bytes.Replace(src, []byte("bpl.NewFromFile(*protocol)"), []byte("bpl.New(BPL_PROTOCOL, \"\")"), 1) 41 | src = bytes.Replace(src, []byte("-p .bpl "), []byte{}, -1) 42 | 43 | qdest := srcbase + "/q" + strings.TrimSuffix(path.Base(protocol), ".bpl") + qdestname 44 | err = os.MkdirAll(qdest, 0700) 45 | if err != nil { 46 | fmt.Fprintln(os.Stderr, err) 47 | os.Exit(2) 48 | } 49 | 50 | f, err := os.Create(qdest + "/protocol.go") 51 | if err != nil { 52 | fmt.Fprintln(os.Stderr, err) 53 | os.Exit(3) 54 | } 55 | defer f.Close() 56 | f.WriteString("package main\n\n") 57 | err = codegen.BytesFromFile(f, "BPL_PROTOCOL", protocol) 58 | if err != nil { 59 | fmt.Fprintln(os.Stderr, "codegen.BytesFromFile:", err) 60 | os.Exit(3) 61 | } 62 | 63 | err = ioutil.WriteFile(qdest+"/main.go", src, 0777) 64 | if err != nil { 65 | fmt.Fprintln(os.Stderr, err) 66 | os.Exit(4) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /cmd/qbplproxy/qbplproxy.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "net" 10 | "net/url" 11 | "os" 12 | "path" 13 | "reflect" 14 | "strings" 15 | 16 | bpl "github.com/goplus/bpl/bpl.ext" 17 | qlang "github.com/xushiwei/qlang/spec" 18 | 19 | "github.com/qiniu/x/log" 20 | ) 21 | 22 | // ----------------------------------------------------------------------------- 23 | 24 | // A Env is the environment of a callback. 25 | // 26 | type Env struct { 27 | Src *net.TCPConn 28 | Dest *net.TCPConn 29 | Direction string 30 | Conn string 31 | } 32 | 33 | // A ReverseProxier is a reverse proxier server. 34 | // 35 | type ReverseProxier struct { 36 | Addr string 37 | Backend string 38 | OnResponse func(io.Reader, *Env) (err error) 39 | OnRequest func(io.Reader, *Env) (err error) 40 | Listened chan bool 41 | } 42 | 43 | // ListenAndServe listens on `Addr` and serves to proxy requests to `Backend`. 44 | // 45 | func (p *ReverseProxier) ListenAndServe() (err error) { 46 | 47 | addr := p.Addr 48 | l, err := net.Listen("tcp", addr) 49 | if err != nil { 50 | log.Fatalf("ListenAndServe(qbplproxy) %s failed: %v\n", addr, err) 51 | return 52 | } 53 | if p.Listened != nil { 54 | p.Listened <- true 55 | } 56 | err = p.Serve(l) 57 | if err != nil { 58 | log.Fatalf("ListenAndServe(qbplproxy) %s failed: %v\n", addr, err) 59 | } 60 | return 61 | } 62 | 63 | func onNil(r io.Reader, env *Env) (err error) { 64 | 65 | _, err = io.Copy(ioutil.Discard, r) 66 | return 67 | } 68 | 69 | // Serve serves to proxy requests to `Backend`. 70 | // 71 | func (p *ReverseProxier) Serve(l net.Listener) (err error) { 72 | 73 | defer l.Close() 74 | 75 | backend, err := net.ResolveTCPAddr("tcp", p.Backend) 76 | if err != nil { 77 | return 78 | } 79 | 80 | onResponse := p.OnResponse 81 | if onResponse == nil { 82 | onResponse = onNil 83 | } 84 | 85 | onRequest := p.OnRequest 86 | if onRequest == nil { 87 | onRequest = onNil 88 | } 89 | 90 | for { 91 | c1, err1 := l.Accept() 92 | if err1 != nil { 93 | return err1 94 | } 95 | c := c1.(*net.TCPConn) 96 | go func() { 97 | c2, err2 := net.DialTCP("tcp", nil, backend) 98 | if err2 != nil { 99 | log.Error("qbplproxy: dial backend failed -", p.Backend, "error:", err2) 100 | c.Close() 101 | return 102 | } 103 | 104 | conn := c.RemoteAddr().String() 105 | go func() { 106 | r2 := io.TeeReader(c2, c) 107 | onResponse(r2, &Env{Src: c2, Dest: c, Direction: "RESP", Conn: conn}) 108 | c.CloseWrite() 109 | c2.CloseRead() 110 | }() 111 | 112 | r := io.TeeReader(c, c2) 113 | err2 = onRequest(r, &Env{Src: c, Dest: c2, Direction: "REQ", Conn: conn}) 114 | if err2 != nil { 115 | log.Info("qbplproxy (request):", err2, "type:", reflect.TypeOf(err2)) 116 | } 117 | c.CloseRead() 118 | c2.CloseWrite() 119 | }() 120 | } 121 | } 122 | 123 | // ----------------------------------------------------------------------------- 124 | 125 | var ( 126 | host = flag.String("h", "", "listen host (listenIp:port).") 127 | backend = flag.String("b", "", "backend host (backendIp:port).") 128 | filter = flag.String("f", "", "filter condition. eg. -f 'flashVer=LNX 9,0,124,2' or -f 'reqMode=play' or -f 'dir=REQ|RESP'") 129 | protocol = flag.String("p", "", "protocol file in BPL syntax, default is guessed by .") 130 | output = flag.String("o", "", "output log file, default is stderr.") 131 | logmode = flag.String("l", "", "log mode: short (default) or long.") 132 | ) 133 | 134 | var ( 135 | baseDir string // $HOME/.qbpl/formats/ 136 | ) 137 | 138 | func fileExists(file string) bool { 139 | 140 | _, err := os.Stat(file) 141 | return err == nil 142 | } 143 | 144 | func guessProtocol(host string) string { 145 | 146 | index := strings.Index(host, ":") 147 | if index >= 0 { 148 | proto := baseDir + host[index+1:] + ".bpl" 149 | if fileExists(proto) { 150 | return proto 151 | } 152 | } 153 | return "" 154 | } 155 | 156 | // qbplproxy -h -b [-p .bpl -f -o .log -l ] 157 | // 158 | func main() { 159 | 160 | flag.Parse() 161 | if *host == "" || *backend == "" { 162 | fmt.Fprintln( 163 | os.Stderr, 164 | "Usage: qbplproxy -h -b [-p .bpl -f -o .log -l ]") 165 | flag.PrintDefaults() 166 | return 167 | } 168 | bpl.SetDumpCode(os.Getenv("BPL_DUMPCODE")) 169 | qlang.DumpStack = true 170 | 171 | baseDir = os.Getenv("HOME") + "/.qbpl/formats/" 172 | if *protocol == "" { 173 | *protocol = guessProtocol(*host) 174 | if *protocol == "" { 175 | *protocol = guessProtocol(*backend) 176 | if *protocol == "" { 177 | log.Fatalln("I can't know protocol by listening port, please use -p .") 178 | } 179 | } 180 | } else { 181 | if path.Ext(*protocol) == "" { 182 | *protocol = baseDir + *protocol + ".bpl" 183 | } 184 | } 185 | 186 | filterCond := make(map[string]interface{}) 187 | if *filter != "" { 188 | m, err := url.ParseQuery(*filter) 189 | if err != nil { 190 | log.Fatalln("Error: invalid -f argument -", err) 191 | } 192 | for k, v := range m { 193 | filterCond[k] = v[0] 194 | } 195 | } 196 | 197 | logflags := bpl.Ldefault 198 | flong := (*logmode == "long") 199 | if flong { 200 | logflags = bpl.Llong 201 | } 202 | 203 | onBpl := onNil 204 | if *protocol != "nil" { 205 | if *output != "" { 206 | f, err := os.Create(*output) 207 | if err != nil { 208 | log.Fatalln("Create log file failed:", err) 209 | } 210 | defer f.Close() 211 | bpl.SetDumper(f, logflags) 212 | } 213 | ruler, err := bpl.NewFromFile(*protocol) 214 | if err != nil { 215 | log.Fatalln("bpl.NewFromFile failed:", err) 216 | } 217 | onBpl = func(r io.Reader, env *Env) (err error) { 218 | in := bufio.NewReader(r) 219 | ctx := bpl.NewContext() 220 | ctx.Globals.SetVar("BPL_FILTER", filterCond) 221 | ctx.Globals.SetVar("BPL_DIRECTION", env.Direction) 222 | if flong { 223 | ctx.Globals.SetVar("BPL_DUMP_PREFIX", "[CONN:"+env.Conn+"]["+env.Direction+"]") 224 | } else { 225 | ctx.Globals.SetVar("BPL_DUMP_PREFIX", "["+env.Direction+"]") 226 | } 227 | _, err = ruler.SafeMatch(in, ctx) 228 | if err != nil { 229 | log.Error("Match failed:", err) 230 | } 231 | in.WriteTo(ioutil.Discard) 232 | return 233 | } 234 | } 235 | log.Std = bpl.Dumper 236 | 237 | rp := &ReverseProxier{ 238 | Addr: *host, 239 | Backend: *backend, 240 | OnRequest: onBpl, 241 | OnResponse: onBpl, 242 | } 243 | rp.ListenAndServe() 244 | } 245 | 246 | // ----------------------------------------------------------------------------- 247 | -------------------------------------------------------------------------------- /cmd/qmockd/qmockd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/goplus/bpl/mockd" 9 | ) 10 | 11 | var ( 12 | host = flag.String("h", "", "bind address.") 13 | ) 14 | 15 | // qmockd -h 16 | // 17 | func main() { 18 | 19 | flag.Parse() 20 | args := flag.Args() 21 | 22 | if *host == "" || len(args) < 1 { 23 | fmt.Fprintln(os.Stderr, "Usage: qmockd -h ") 24 | flag.PrintDefaults() 25 | return 26 | } 27 | 28 | mockd.ListenAndServe(*host, args[0]) 29 | } 30 | -------------------------------------------------------------------------------- /cmd/qreplay/qreplay.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/goplus/bpl/replay" 9 | ) 10 | 11 | var ( 12 | host = flag.String("s", "", "remote address to dial.") 13 | ) 14 | 15 | // qreplay -s 16 | // 17 | func main() { 18 | 19 | flag.Parse() 20 | 21 | if *host == "" { 22 | fmt.Fprintln(os.Stderr, "Usage: qreplay -s ") 23 | flag.PrintDefaults() 24 | return 25 | } 26 | 27 | var in *os.File 28 | args := flag.Args() 29 | if len(args) > 0 { 30 | file := args[0] 31 | f, err := os.Open(file) 32 | if err != nil { 33 | fmt.Fprintln(os.Stderr, "Open failed:", err) 34 | os.Exit(1) 35 | } 36 | defer f.Close() 37 | in = f 38 | } else { 39 | in = os.Stdin 40 | } 41 | 42 | err := replay.HexRequest(*host, nil, in, "[REQ]") 43 | if err != nil { 44 | fmt.Fprintln(os.Stderr, "replay.HexRequest:", err) 45 | os.Exit(2) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /compose.go: -------------------------------------------------------------------------------- 1 | package bpl 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "io" 7 | "io/ioutil" 8 | "reflect" 9 | 10 | "github.com/qiniu/x/bufiox" 11 | ) 12 | 13 | var ( 14 | // ErrVarNotAssigned is returned when TypeVar.Elem is not assigned. 15 | ErrVarNotAssigned = errors.New("variable is not assigned") 16 | 17 | // ErrVarAssigned is returned when TypeVar.Elem is already assigned. 18 | ErrVarAssigned = errors.New("variable is already assigned") 19 | 20 | // ErrNotEOF is returned when current position is not at EOF. 21 | ErrNotEOF = errors.New("current position is not at EOF") 22 | ) 23 | 24 | // ----------------------------------------------------------------------------- 25 | 26 | type nilType int 27 | 28 | func (p nilType) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 29 | 30 | return nil, nil 31 | } 32 | 33 | func (p nilType) RetType() reflect.Type { 34 | 35 | return TyInterface 36 | } 37 | 38 | func (p nilType) SizeOf() int { 39 | 40 | return 0 41 | } 42 | 43 | // Nil is a matching unit that matches zero bytes. 44 | // 45 | var Nil Ruler = nilType(0) 46 | 47 | // ----------------------------------------------------------------------------- 48 | 49 | type eof int 50 | 51 | func (p eof) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 52 | 53 | _, err = in.Peek(1) 54 | if err == io.EOF { 55 | return nil, nil 56 | } 57 | return nil, ErrNotEOF 58 | } 59 | 60 | func (p eof) RetType() reflect.Type { 61 | 62 | return TyInterface 63 | } 64 | 65 | func (p eof) SizeOf() int { 66 | 67 | return 0 68 | } 69 | 70 | // EOF is a matching unit that matches EOF. 71 | // 72 | var EOF Ruler = eof(0) 73 | 74 | // ----------------------------------------------------------------------------- 75 | 76 | type done int 77 | 78 | func (p done) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 79 | 80 | _, err = in.WriteTo(ioutil.Discard) 81 | return 82 | } 83 | 84 | func (p done) RetType() reflect.Type { 85 | 86 | return TyInterface 87 | } 88 | 89 | func (p done) SizeOf() int { 90 | 91 | return -1 92 | } 93 | 94 | // Done is a matching unit that seeks current position to EOF. 95 | // 96 | var Done Ruler = done(0) 97 | 98 | // ----------------------------------------------------------------------------- 99 | 100 | type and struct { 101 | rs []Ruler 102 | } 103 | 104 | func (p *and) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 105 | 106 | for _, r := range p.rs { 107 | _, err = r.Match(in, ctx) 108 | if err != nil { 109 | return 110 | } 111 | } 112 | return ctx.Dom(), nil 113 | } 114 | 115 | func (p *and) RetType() reflect.Type { 116 | 117 | return TyInterface 118 | } 119 | 120 | func (p *and) SizeOf() int { 121 | 122 | return -1 123 | } 124 | 125 | // And returns a matching unit that matches R1 R2 ... RN 126 | // 127 | func And(rs ...Ruler) Ruler { 128 | 129 | if len(rs) <= 1 { 130 | if len(rs) == 1 { 131 | return rs[0] 132 | } 133 | return Nil 134 | } 135 | return &and{rs: rs} 136 | } 137 | 138 | // ----------------------------------------------------------------------------- 139 | 140 | type seq struct { 141 | rs []Ruler 142 | } 143 | 144 | func (p *seq) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 145 | 146 | ret := ctx.requireVarSlice() 147 | for _, r := range p.rs { 148 | v, err = r.Match(in, ctx.NewSub()) 149 | if err != nil { 150 | return 151 | } 152 | ret = append(ret, v) 153 | } 154 | ctx.dom = ret 155 | return ret, nil 156 | } 157 | 158 | func (p *seq) RetType() reflect.Type { 159 | 160 | return tyInterfaceSlice 161 | } 162 | 163 | func (p *seq) SizeOf() int { 164 | 165 | return -1 166 | } 167 | 168 | // Seq returns a matching unit that matches R1 R2 ... RN and returns matching result. 169 | // 170 | func Seq(rs ...Ruler) Ruler { 171 | 172 | return &seq{rs: rs} 173 | } 174 | 175 | // ----------------------------------------------------------------------------- 176 | 177 | type act struct { 178 | fn func(ctx *Context) error 179 | } 180 | 181 | func (p *act) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 182 | 183 | err = p.fn(ctx) 184 | if err != nil { 185 | return 186 | } 187 | return ctx.Dom(), nil 188 | } 189 | 190 | func (p *act) RetType() reflect.Type { 191 | 192 | return TyInterface 193 | } 194 | 195 | func (p *act) SizeOf() int { 196 | 197 | return -1 198 | } 199 | 200 | // Do returns a matching unit that executes action fn(ctx). 201 | // 202 | func Do(fn func(ctx *Context) error) Ruler { 203 | 204 | return &act{fn: fn} 205 | } 206 | 207 | // ----------------------------------------------------------------------------- 208 | 209 | type dyntype struct { 210 | r func(ctx *Context) (Ruler, error) 211 | } 212 | 213 | func (p *dyntype) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 214 | 215 | r, err := p.r(ctx) 216 | if err != nil { 217 | return 218 | } 219 | if r != nil { 220 | return r.Match(in, ctx) 221 | } 222 | return 223 | } 224 | 225 | func (p *dyntype) RetType() reflect.Type { 226 | 227 | return TyInterface 228 | } 229 | 230 | func (p *dyntype) SizeOf() int { 231 | 232 | return -1 233 | } 234 | 235 | // Dyntype returns a dynamic matching unit. 236 | // 237 | func Dyntype(r func(ctx *Context) (Ruler, error)) Ruler { 238 | 239 | return &dyntype{r: r} 240 | } 241 | 242 | // ----------------------------------------------------------------------------- 243 | 244 | type read struct { 245 | n func(ctx *Context) int 246 | r Ruler 247 | } 248 | 249 | func (p *read) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 250 | 251 | n := p.n(ctx) 252 | b := make([]byte, n) 253 | _, err = io.ReadFull(in, b) 254 | if err != nil { 255 | return 256 | } 257 | in = bufiox.NewReaderBuffer(b) 258 | return MatchStream(p.r, in, ctx) 259 | } 260 | 261 | func (p *read) RetType() reflect.Type { 262 | 263 | return p.r.RetType() 264 | } 265 | 266 | func (p *read) SizeOf() int { 267 | 268 | return -1 269 | } 270 | 271 | // Read returns a matching unit that reads n(ctx) bytes and matches R. 272 | // 273 | func Read(n func(ctx *Context) int, r Ruler) Ruler { 274 | 275 | return &read{r: r, n: n} 276 | } 277 | 278 | // ----------------------------------------------------------------------------- 279 | 280 | type skip struct { 281 | n func(ctx *Context) int 282 | } 283 | 284 | func (p *skip) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 285 | 286 | n := p.n(ctx) 287 | v, err = in.Discard(n) 288 | return 289 | } 290 | 291 | func (p *skip) RetType() reflect.Type { 292 | 293 | return tyInt 294 | } 295 | 296 | func (p *skip) SizeOf() int { 297 | 298 | return -1 299 | } 300 | 301 | // Skip returns a matching unit that skips n(ctx) bytes. 302 | // 303 | func Skip(n func(ctx *Context) int) Ruler { 304 | 305 | return &skip{n: n} 306 | } 307 | 308 | // ----------------------------------------------------------------------------- 309 | 310 | type ifType struct { 311 | cond func(ctx *Context) bool 312 | r Ruler 313 | } 314 | 315 | func (p *ifType) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 316 | 317 | if p.cond(ctx) { 318 | return p.r.Match(in, ctx) 319 | } 320 | return 321 | } 322 | 323 | func (p *ifType) RetType() reflect.Type { 324 | 325 | return TyInterface 326 | } 327 | 328 | func (p *ifType) SizeOf() int { 329 | 330 | return -1 331 | } 332 | 333 | // If returns a matching unit that if cond(ctx) then matches it with R. 334 | // 335 | func If(cond func(ctx *Context) bool, r Ruler) Ruler { 336 | 337 | return &ifType{r: r, cond: cond} 338 | } 339 | 340 | // ----------------------------------------------------------------------------- 341 | 342 | type eval struct { 343 | expr func(ctx *Context) interface{} 344 | r Ruler 345 | } 346 | 347 | func (p *eval) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 348 | 349 | fclose := false 350 | val := p.expr(ctx) 351 | switch v := val.(type) { 352 | case []byte: 353 | in = bufiox.NewReaderBuffer(v) 354 | case io.Reader: 355 | in = bufio.NewReader(v) 356 | fclose = true 357 | default: 358 | panic("eval must return []byte or io.Reader") 359 | } 360 | v, err = MatchStream(p.r, in, ctx) 361 | if fclose { 362 | if v, ok := val.(io.Closer); ok { 363 | v.Close() 364 | } 365 | } 366 | return 367 | } 368 | 369 | func (p *eval) RetType() reflect.Type { 370 | 371 | return p.r.RetType() 372 | } 373 | 374 | func (p *eval) SizeOf() int { 375 | 376 | return -1 377 | } 378 | 379 | // Eval returns a matching unit that eval expr(ctx) and matches it with R. 380 | // 381 | func Eval(expr func(ctx *Context) interface{}, r Ruler) Ruler { 382 | 383 | return &eval{r: r, expr: expr} 384 | } 385 | 386 | // ----------------------------------------------------------------------------- 387 | 388 | type assert struct { 389 | expr func(ctx *Context) bool 390 | msg string 391 | } 392 | 393 | func (p *assert) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 394 | 395 | if p.expr(ctx) { 396 | return 397 | } 398 | panic(p.msg) 399 | } 400 | 401 | func (p *assert) RetType() reflect.Type { 402 | 403 | return TyInterface 404 | } 405 | 406 | func (p *assert) SizeOf() int { 407 | 408 | return -1 409 | } 410 | 411 | // Assert returns a matching unit that assert expr(ctx). 412 | // 413 | func Assert(expr func(ctx *Context) bool, msg string) Ruler { 414 | 415 | return &assert{msg: msg, expr: expr} 416 | } 417 | 418 | // ----------------------------------------------------------------------------- 419 | 420 | // A TypeVar is typeinfo of a `Struct` member. 421 | // 422 | type TypeVar struct { 423 | Name string 424 | Elem Ruler 425 | } 426 | 427 | // Assign assigns TypeVar.Elem. 428 | // 429 | func (p *TypeVar) Assign(r Ruler) error { 430 | 431 | if p.Elem != nil { 432 | return ErrVarAssigned 433 | } 434 | p.Elem = r 435 | return nil 436 | } 437 | 438 | // Match is required by a matching unit. see Ruler interface. 439 | // 440 | func (p *TypeVar) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 441 | 442 | r := p.Elem 443 | if r == nil { 444 | return 0, ErrVarNotAssigned 445 | } 446 | return r.Match(in, ctx) 447 | } 448 | 449 | // RetType returns matching result type. 450 | // 451 | func (p *TypeVar) RetType() reflect.Type { 452 | 453 | return p.Elem.RetType() 454 | } 455 | 456 | // SizeOf is required by a matching unit. see Ruler interface. 457 | // 458 | func (p *TypeVar) SizeOf() int { 459 | 460 | return p.Elem.SizeOf() 461 | } 462 | 463 | // ----------------------------------------------------------------------------- 464 | -------------------------------------------------------------------------------- /context.go: -------------------------------------------------------------------------------- 1 | package bpl 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/hex" 7 | "errors" 8 | "fmt" 9 | "reflect" 10 | "runtime/debug" 11 | 12 | "github.com/qiniu/x/bufiox" 13 | "github.com/xushiwei/qlang/exec" 14 | ) 15 | 16 | // ----------------------------------------------------------------------------- 17 | 18 | // A Globals represents global variables. 19 | // 20 | type Globals struct { 21 | Impl map[string]interface{} 22 | } 23 | 24 | // NewGlobals returns a `Globals` instance. 25 | // 26 | func NewGlobals() Globals { 27 | 28 | return Globals{ 29 | Impl: make(map[string]interface{}), 30 | } 31 | } 32 | 33 | // GetAndSetVar gets old value of a global variable and sets new value to it. 34 | // 35 | func (p Globals) GetAndSetVar(name string, v interface{}) (old interface{}, ok bool) { 36 | 37 | old, ok = p.Impl[name] 38 | p.Impl[name] = v 39 | return 40 | } 41 | 42 | // SetVar sets a global variable to new value. 43 | // 44 | func (p Globals) SetVar(name string, v interface{}) { 45 | 46 | p.Impl[name] = v 47 | } 48 | 49 | // Var returns value of a global variable. 50 | // 51 | func (p Globals) Var(name string) (v interface{}, ok bool) { 52 | 53 | v, ok = p.Impl[name] 54 | return 55 | } 56 | 57 | // ----------------------------------------------------------------------------- 58 | 59 | // A Context represents the matching context of bpl. 60 | // 61 | type Context struct { 62 | dom interface{} 63 | Stack *exec.Stack 64 | Parent *Context 65 | Globals Globals 66 | } 67 | 68 | // NewContext returns a new matching Context. 69 | // 70 | func NewContext() *Context { 71 | 72 | gbl := NewGlobals() 73 | stk := exec.NewStack() 74 | return &Context{Globals: gbl, Stack: stk} 75 | } 76 | 77 | // NewSub returns a new sub Context. 78 | // 79 | func (p *Context) NewSub() *Context { 80 | 81 | return &Context{Parent: p, Globals: p.Globals, Stack: p.Stack} 82 | } 83 | 84 | func (p *Context) requireVarSlice() []interface{} { 85 | 86 | var vars []interface{} 87 | if p.dom == nil { 88 | vars = make([]interface{}, 0, 4) 89 | } else if domv, ok := p.dom.([]interface{}); ok { 90 | vars = domv 91 | } else { 92 | panic("dom type isn't []interface{}") 93 | } 94 | return vars 95 | } 96 | 97 | // SetVar sets a new variable to matching context. 98 | // 99 | func (p *Context) SetVar(name string, v interface{}) { 100 | 101 | if _, ok := p.Globals.Var(name); ok { 102 | panic(fmt.Errorf("variable `%s` exists globally", name)) 103 | } 104 | 105 | var vars map[string]interface{} 106 | if p.dom == nil { 107 | vars = make(map[string]interface{}) 108 | p.dom = vars 109 | } else if domv, ok := p.dom.(map[string]interface{}); ok { 110 | if _, ok = domv[name]; ok { 111 | panic(fmt.Errorf("variable `%s` exists in dom", name)) 112 | } 113 | vars = domv 114 | } else { 115 | panic("dom type isn't map[string]interface{}") 116 | } 117 | vars[name] = v 118 | } 119 | 120 | // LetVar sets a variable to matching context. 121 | // 122 | func (p *Context) LetVar(name string, v interface{}) { 123 | 124 | if _, ok := p.Globals.Var(name); ok { 125 | p.Globals.SetVar(name, v) 126 | return 127 | } 128 | 129 | var vars map[string]interface{} 130 | if p.dom == nil { 131 | vars = make(map[string]interface{}) 132 | p.dom = vars 133 | } else if domv, ok := p.dom.(map[string]interface{}); ok { 134 | vars = domv 135 | } else { 136 | panic("dom type isn't map[string]interface{}") 137 | } 138 | vars[name] = v 139 | } 140 | 141 | // Var gets a variable from matching context. 142 | // 143 | func (p *Context) Var(name string) (v interface{}, ok bool) { 144 | 145 | vars, ok := p.dom.(map[string]interface{}) 146 | if ok { 147 | v, ok = vars[name] 148 | } else { 149 | panic("dom type isn't map[string]interface{}") 150 | } 151 | return 152 | } 153 | 154 | // SetDom set matching result of matching result. 155 | // 156 | func (p *Context) SetDom(v interface{}) { 157 | 158 | if p.dom == nil { 159 | p.dom = v 160 | } else { 161 | panic("dom was assigned already") 162 | } 163 | } 164 | 165 | // Dom returns matching result. 166 | // 167 | func (p *Context) Dom() interface{} { 168 | 169 | return p.dom 170 | } 171 | 172 | // ----------------------------------------------------------------------------- 173 | 174 | // A Ruler interface is required to a matching unit. 175 | // 176 | type Ruler interface { 177 | // Match matches input stream `in`, and returns matching result. 178 | Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) 179 | 180 | // RetType returns matching result type. 181 | RetType() reflect.Type 182 | 183 | // SizeOf returns expected length of result. If length is variadic, it returns -1. 184 | SizeOf() int 185 | } 186 | 187 | // MatchStream matches a stream. 188 | // 189 | func MatchStream(r Ruler, in *bufio.Reader, ctx *Context) (v interface{}, err error) { 190 | 191 | glbs := ctx.Globals 192 | old, ok := glbs.GetAndSetVar("BPL_IN", in) 193 | v, err = r.Match(in, ctx) 194 | if ok { 195 | glbs.SetVar("BPL_IN", old) 196 | } 197 | return 198 | } 199 | 200 | // ----------------------------------------------------------------------------- 201 | 202 | type fileLine struct { 203 | r Ruler 204 | file string 205 | line int 206 | } 207 | 208 | type errorAt struct { 209 | Err error 210 | Buf []byte 211 | } 212 | 213 | func (p *errorAt) Error() string { 214 | 215 | b := make([]byte, 0, 32) 216 | b = append(b, p.Err.Error()...) 217 | b = append(b, '\n') 218 | 219 | w := bytes.NewBuffer(b) 220 | d := hex.Dumper(w) 221 | d.Write(p.Buf) 222 | d.Close() 223 | return string(w.Bytes()) 224 | } 225 | 226 | func (p *fileLine) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 227 | 228 | v, err = doMatch(p.r, in, ctx) 229 | if err != nil { 230 | if _, ok := err.(*exec.Error); !ok { 231 | err = &exec.Error{ 232 | Err: &errorAt{Err: err, Buf: bufiox.Buffer(in)}, 233 | File: p.file, 234 | Line: p.line, 235 | Stack: debug.Stack(), 236 | } 237 | } 238 | } 239 | return 240 | } 241 | 242 | func (p *fileLine) RetType() reflect.Type { 243 | 244 | return p.r.RetType() 245 | } 246 | 247 | func (p *fileLine) SizeOf() int { 248 | 249 | return p.r.SizeOf() 250 | } 251 | 252 | func doMatch(R Ruler, in *bufio.Reader, ctx *Context) (v interface{}, err error) { 253 | 254 | defer func() { 255 | if e := recover(); e != nil { 256 | switch v := e.(type) { 257 | case string: 258 | err = errors.New(v) 259 | case error: 260 | err = v 261 | default: 262 | panic(e) 263 | } 264 | } 265 | }() 266 | 267 | return R.Match(in, ctx) 268 | } 269 | 270 | // FileLine is a matching rule that reports error file line when error occurs. 271 | // 272 | func FileLine(file string, line int, R Ruler) Ruler { 273 | 274 | if _, ok := R.(*fileLine); ok { 275 | return R 276 | } 277 | return &fileLine{r: R, file: file, line: line} 278 | } 279 | 280 | // ----------------------------------------------------------------------------- 281 | -------------------------------------------------------------------------------- /formats/1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/goplus/bpl/4a8256d0dc4208408f97521f3c7a4a0b929004c4/formats/1.gif -------------------------------------------------------------------------------- /formats/1935.bpl: -------------------------------------------------------------------------------- 1 | rtmp.bpl -------------------------------------------------------------------------------- /formats/27017.bpl: -------------------------------------------------------------------------------- 1 | mongo.bpl -------------------------------------------------------------------------------- /formats/80.bpl: -------------------------------------------------------------------------------- 1 | http.bpl -------------------------------------------------------------------------------- /formats/flv.bpl: -------------------------------------------------------------------------------- 1 | Chunk = { 2 | back uint32be // 整个msg的长度（含flv头） 3 | typeid uint8 4 | chunklen uint24be 5 | ts uint24be 6 | tsExtended uint8 7 | streamid uint24be // always be 0? 8 | data [chunklen]byte 9 | } 10 | 11 | FlvHeader = { 12 | tag [9]byte 13 | assert bytes.equal(tag, bytes.from([0x46, 0x4c, 0x56, 0x01, 0x05, 0x00, 0x00, 0x00, 0x09])) 14 | } 15 | 16 | Flv = FlvHeader *(Chunk dump) 17 | 18 | Message = { 19 | if BPL_DIRECTION == "REQ" { 20 | 21 | let _req, _err = http.readRequest(BPL_IN) 22 | assert _err == nil 23 | 24 | let method = _req.method 25 | let path = _req.URL.string() 26 | let host = _req.host 27 | let header = _req.header 28 | 29 | let _b, _err = ioutil.readAll(_req.body) 30 | assert _err == nil 31 | let body = string(_b) 32 | dump 33 | 34 | } else { 35 | 36 | let _resp, _err = http.readResponse(BPL_IN, nil) 37 | assert _err == nil 38 | 39 | let status = _resp.status 40 | let statusCode = _resp.statusCode 41 | let header = _resp.header 42 | dump 43 | 44 | eval _resp.body do Flv 45 | } 46 | } 47 | 48 | doc = *Message 49 | -------------------------------------------------------------------------------- /formats/format-mp4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/goplus/bpl/4a8256d0dc4208408f97521f3c7a4a0b929004c4/formats/format-mp4.jpg -------------------------------------------------------------------------------- /formats/gif.bpl: -------------------------------------------------------------------------------- 1 | const ( 2 | fColorTable = 0x80 3 | fColorTableBitsMask = 7 4 | ) 5 | 6 | ColorTable = { 7 | colortable [(1 << (1 + (fields&fColorTableBitsMask))) * 3]byte 8 | } 9 | 10 | Header = { 11 | tag [6]char 12 | width int16 13 | height int16 14 | fields uint8 15 | backgroundIndex uint8 16 | tmp uint8 17 | assert tag == "GIF87a" || tag == "GIF89a" 18 | if fields & fColorTable do ColorTable 19 | } 20 | 21 | sTrailer = nil 22 | 23 | ImageHeader = { 24 | left int16 25 | top int16 26 | width int16 27 | height int16 28 | fields byte 29 | if fields & fColorTable do ColorTable 30 | } 31 | 32 | ExtBlocks = { 33 | len byte 34 | if len { 35 | data [len]byte 36 | next ExtBlocks 37 | } 38 | } 39 | 40 | eComment = { 41 | blocks ExtBlocks 42 | } 43 | 44 | eApplication = { 45 | len byte 46 | name [len]char 47 | blocks ExtBlocks 48 | } 49 | 50 | eText = { 51 | text [13]char 52 | blocks ExtBlocks 53 | } 54 | 55 | eGraphicControl = { 56 | unused1 byte 57 | flags byte 58 | delayTime int16 59 | transparentIndex byte 60 | unused2 byte 61 | } 62 | 63 | sExtension = { 64 | etag byte 65 | case etag { 66 | 0x01: eText 67 | 0xF9: eGraphicControl 68 | 0xFE: eComment 69 | 0xFF: eApplication 70 | } 71 | } 72 | 73 | sImage = { 74 | h ImageHeader 75 | litWidth byte 76 | blocks ExtBlocks 77 | assert litWidth >= 2 && litWidth <= 8 78 | } 79 | 80 | Record = { 81 | tag byte 82 | case tag { 83 | 0x21: sExtension 84 | 0x2C: sImage 85 | 0x3B: sTrailer 86 | } 87 | } 88 | 89 | doc = Header dump *(Record dump) 90 | -------------------------------------------------------------------------------- /formats/http.bpl: -------------------------------------------------------------------------------- 1 | message = { 2 | if BPL_DIRECTION == "REQ" { 3 | 4 | let _req, _err = http.readRequest(BPL_IN) 5 | assert _err == nil 6 | 7 | let method = _req.method 8 | let path = _req.URL.string() 9 | let host = _req.host 10 | let header = _req.header 11 | 12 | let _b, _err = ioutil.readAll(_req.body) 13 | assert _err == nil 14 | let body = string(_b) 15 | 16 | } else { 17 | 18 | let _resp, _err = http.readResponse(BPL_IN, nil) 19 | assert _err == nil 20 | 21 | let status = _resp.status 22 | let statusCode = _resp.statusCode 23 | let header = _resp.header 24 | 25 | let _b, _err = ioutil.readAll(_resp.body) 26 | assert _err == nil 27 | 28 | let bodyLength = len(_b) 29 | } 30 | } 31 | 32 | doc = *(message dump) 33 | -------------------------------------------------------------------------------- /formats/lines.bpl: -------------------------------------------------------------------------------- 1 | line = { 2 | let line, _ = BPL_IN.readString('\n') 3 | do printf("%s", line) 4 | return line 5 | } 6 | 7 | doc = *[line] dump 8 | -------------------------------------------------------------------------------- /formats/mongo.bpl: -------------------------------------------------------------------------------- 1 | // 2 | // https://docs.mongodb.com/manual/reference/mongodb-wire-protocol/ 3 | 4 | document = bson 5 | 6 | MsgHeader = {/C 7 | int32 messageLength; // total message size, including this 8 | int32 requestID; // identifier for this message 9 | int32 responseTo; // requestID from the original request (used in responses from db) 10 | int32 opCode; // request type - see table below 11 | } 12 | 13 | OP_UPDATE = {/C 14 | int32 ZERO; // 0 - reserved for future use 15 | cstring fullCollectionName; // "dbname.collectionname" 16 | int32 flags; // bit vector. see below 17 | document selector; // the query to select the document 18 | document update; // specification of the update to perform 19 | } 20 | 21 | OP_INSERT = {/C 22 | int32 flags; // bit vector - see below 23 | cstring fullCollectionName; // "dbname.collectionname" 24 | document* documents; // one or more documents to insert into the collection 25 | } 26 | 27 | OP_QUERY = {/C 28 | int32 flags; // bit vector of query options. See below for details. 29 | cstring fullCollectionName; // "dbname.collectionname" 30 | int32 numberToSkip; // number of documents to skip 31 | int32 numberToReturn; // number of documents to return 32 | // in the first OP_REPLY batch 33 | document query; // query object. See below for details. 34 | document? returnFieldsSelector; // Optional. Selector indicating the fields 35 | // to return. See below for details. 36 | } 37 | 38 | OP_GET_MORE = {/C 39 | int32 ZERO; // 0 - reserved for future use 40 | cstring fullCollectionName; // "dbname.collectionname" 41 | int32 numberToReturn; // number of documents to return 42 | int64 cursorID; // cursorID from the OP_REPLY 43 | } 44 | 45 | OP_DELETE = {/C 46 | int32 ZERO; // 0 - reserved for future use 47 | cstring fullCollectionName; // "dbname.collectionname" 48 | int32 flags; // bit vector - see below for details. 49 | document selector; // query object. See below for details. 50 | } 51 | 52 | OP_KILL_CURSORS = {/C 53 | int32 ZERO; // 0 - reserved for future use 54 | int32 numberOfCursorIDs; // number of cursorIDs in message 55 | int64* cursorIDs; // sequence of cursorIDs to close 56 | } 57 | 58 | OP_MSG = {/C 59 | cstring message; // message for the database 60 | } 61 | 62 | OP_REPLY = {/C 63 | int32 responseFlags; // bit vector - see details below 64 | int64 cursorID; // cursor id if client needs to do get more's 65 | int32 startingFrom; // where in the cursor this reply is starting 66 | int32 numberReturned; // number of documents in the reply 67 | document* documents; // documents 68 | } 69 | 70 | OP_REQ = {/C 71 | cstring dbName; 72 | cstring cmd; 73 | document param; 74 | } 75 | 76 | OP_RET = {/C 77 | document ret; 78 | } 79 | 80 | Message = { 81 | header MsgHeader // standard message header 82 | _body [header.messageLength - sizeof(MsgHeader)]byte 83 | eval _body do case header.opCode { 84 | 1: OP_REPLY // Reply to a client request. responseTo is set. 85 | 1000: OP_MSG // Generic msg command followed by a string. 86 | 2001: OP_UPDATE 87 | 2002: OP_INSERT 88 | 2004: OP_QUERY 89 | 2005: OP_GET_MORE // Get more data from a query. See Cursors. 90 | 2006: OP_DELETE 91 | 2007: OP_KILL_CURSORS // Notify database that the client has finished with the cursor. 92 | 2010: OP_REQ 93 | 2011: OP_RET 94 | default: let body = _body 95 | } 96 | } 97 | 98 | doc = *(Message dump) 99 | -------------------------------------------------------------------------------- /formats/mp4.bpl: -------------------------------------------------------------------------------- 1 | // 2 | // http://blog.sina.com.cn/s/blog_48f93b530100jz4b.html 3 | // http://www.52rd.com/Blog/wqyuwss/559/ 4 | // http://blog.csdn.net/wutong_login/article/category/567011 5 | 6 | box = { 7 | size uint32be 8 | typ [4]char 9 | if size == 1 { 10 | _largesize uint64be 11 | let size = _largesize - 8 12 | } 13 | 14 | if size == 0 { 15 | let _body = mkslice("byte", 0) 16 | } elif typ == "mdat" { 17 | skip size - 8 18 | let _body = mkslice("byte", 0) 19 | } else { 20 | _body [size - 8]byte 21 | } 22 | } 23 | 24 | boxtr = { 25 | let body = _body 26 | } 27 | 28 | fixed16be = { 29 | _v uint16be 30 | return float64(_v) / 0x100 31 | } 32 | 33 | fixed32be = { 34 | _v uint32be 35 | return float64(_v) / 0x10000 36 | } 37 | 38 | // -------------------------------------------------------------- 39 | 40 | avc1 = { 41 | body *byte 42 | } 43 | 44 | mp4a = { 45 | body *byte 46 | } 47 | 48 | uuid = { 49 | body *byte 50 | } 51 | 52 | stsdbox = box { 53 | eval _body do case typ { 54 | "avc1": avc1 55 | default: boxtr 56 | } 57 | } 58 | 59 | stsd = { 60 | let class = "Sample Description" 61 | version byte 62 | flags uint24be 63 | nvals uint32be 64 | vals [nvals]stsdbox 65 | } 66 | 67 | // -------------------------------------------------------------- 68 | 69 | stts = { 70 | body *byte 71 | } 72 | 73 | stsz = { 74 | let class = "Sample Size" 75 | let bodyLength = len(_body) 76 | } 77 | 78 | stz2 = { 79 | body *byte 80 | } 81 | 82 | stsc = { 83 | // “stss”确定media中的关键帧。对于压缩媒体数据，关键帧是一系列压缩序列的开始帧，其解压缩时不依赖以前的帧，而后续帧的解压缩将依赖于这个关键帧。 84 | // “stss”可以非常紧凑的标记媒体内的随机存取点，它包含一个sample序号表，表内的每一项严格按照sample的序号排列，说明了媒体中的哪一个sample是关键帧。 85 | // 如果此表不存在，说明每一个sample都是一个关键帧，是一个随机存取点。 86 | // 87 | let class = "Sample To Chunk Box" 88 | let bodyLength = len(_body) 89 | } 90 | 91 | stco = { 92 | // ”stco”定义了每个thunk在媒体流中的位置。位置有两种可能，32位的和64位的，后者对非常大的电影很有用。 93 | // 在一个表中只会有一种可能，这个位置是在整个文件中的，而不是在任何box中的，这样做就可以直接在文件中找到媒体数据，而不用解释box。 94 | // 需要注意的是一旦前面的box有了任何改变，这张表都要重新建立，因为位置信息已经改变了。 95 | // 96 | let class = "Chunk Offset" 97 | let bodyLength = len(_body) 98 | } 99 | 100 | co64 = { 101 | let class = "Chunk Offset64" 102 | let bodyLength = len(_body) 103 | } 104 | 105 | ctts = { 106 | let class = "Composition Time To Sample" 107 | let bodyLength = len(_body) 108 | } 109 | 110 | stss = { 111 | // “stss”确定media中的关键帧。对于压缩媒体数据，关键帧是一系列压缩序列的开始帧，其解压缩时不依赖以前的帧，而后续帧的解压缩将依赖于这个关键帧。 112 | // “stss”可以非常紧凑的标记媒体内的随机存取点，它包含一个sample序号表，表内的每一项严格按照sample的序号排列，说明了媒体中的哪一个sample是关键帧。 113 | // 如果此表不存在，说明每一个sample都是一个关键帧，是一个随机存取点。 114 | // 115 | let class = "Sync Sample" 116 | let bodyLength = len(_body) 117 | } 118 | 119 | stbbox = box { 120 | eval _body do case typ { 121 | "stsd": stsd 122 | "stts": stts 123 | "stsz": stsz 124 | "stz2": stz2 125 | "stsc": stsc 126 | "stco": stco 127 | "co64": co64 128 | "ctts": ctts 129 | "stss": stss 130 | default: boxtr 131 | } 132 | } 133 | 134 | // Sample Table Box 135 | // 136 | stbl = { 137 | vals *stbbox 138 | } 139 | 140 | // -------------------------------------------------------------- 141 | 142 | dref = { 143 | body *byte 144 | } 145 | 146 | dibox = box { 147 | eval _body do case typ { 148 | "dref": dref 149 | default: boxtr 150 | } 151 | } 152 | 153 | dinf = { 154 | vals *dibox 155 | } 156 | 157 | // -------------------------------------------------------------- 158 | 159 | vmhd = { 160 | body *byte 161 | } 162 | 163 | smhd = { 164 | body *byte 165 | } 166 | 167 | hmhd = { 168 | body *byte 169 | } 170 | 171 | nmhd = { 172 | body *byte 173 | } 174 | 175 | mibox = box { 176 | eval _body do case typ { 177 | "vmhd": vmhd 178 | "smhd": smhd 179 | "hmhd": hmhd 180 | "nmhd": nmhd 181 | "dinf": dinf 182 | "stbl": stbl 183 | default: boxtr 184 | } 185 | } 186 | 187 | // Media Information Box 188 | // 189 | minf = { 190 | vals *mibox 191 | } 192 | 193 | // -------------------------------------------------------------- 194 | 195 | mdhd = { 196 | version byte 197 | flags uint24be 198 | 199 | ctime uint32be // 创建时间（相对于UTC时间1904-01-01零点的秒数） 200 | mtime uint32be // 修改时间 201 | 202 | time_scale uint32be // 文件媒体在1秒时间内的刻度值，可以理解为1秒长度的时间单元数 203 | duration uint32be // 该track的时间长度，用duration和time_scale值可以计算track时长 204 | 205 | language uint16be 206 | predefined uint16be 207 | } 208 | 209 | // Handler Reference Box 210 | // 211 | hdlr = { 212 | version byte 213 | flags uint24be 214 | 215 | predefined uint32be 216 | 217 | // 在media box中，该值为4个字符： 218 | // “vide”— video track 219 | // “soun”— audio track 220 | // “hint”— hint track 221 | // 222 | handler_typ [4]char 223 | 224 | reserved [12]byte 225 | name cstring // track type name，以‘\0’结尾的字符串 226 | } 227 | 228 | mdbox = box { 229 | eval _body do case typ { 230 | "mdhd": mdhd 231 | "hdlr": hdlr 232 | "minf": minf 233 | default: boxtr 234 | } 235 | } 236 | 237 | // Media Box 238 | // 239 | mdia = { 240 | vals *mdbox 241 | } 242 | 243 | // -------------------------------------------------------------- 244 | 245 | elst = { 246 | body *byte 247 | } 248 | 249 | edtsbox = box { 250 | eval _body do case typ { 251 | "elst": elst 252 | default: boxtr 253 | } 254 | } 255 | 256 | edts = { 257 | vals *edtsbox 258 | } 259 | 260 | // -------------------------------------------------------------- 261 | 262 | // Track Header Box 263 | // 264 | tkhd = { 265 | version byte 266 | 267 | // 按位或操作结果值，预定义如下： 268 | // 0x000001 track_enabled，否则该track不被播放； 269 | // 0x000002 track_in_movie，表示该track在播放中被引用； 270 | // 0x000004 track_in_preview，表示该track在预览时被引用。 271 | // 一般该值为7，如果一个媒体所有track均未设置track_in_movie和track_in_preview，将被理解为所有track均设置了这两项； 272 | // 对于hint track，该值为0 273 | // 274 | flags uint24be 275 | 276 | ctime uint32be // 创建时间（相对于UTC时间1904-01-01零点的秒数） 277 | mtime uint32be // 修改时间 278 | 279 | track_id uint32be // track id号，不能重复且不能为0 280 | 281 | reserved uint32be 282 | duration uint32be 283 | reserved2 uint64be 284 | 285 | layer uint16be // 视频层，默认为0，值小的在上层 286 | alt_group uint16be // alternate group: track分组信息，默认为0表示该track未与其他track有群组关系 287 | volume fixed16be // [8.8] 格式，1.0（0x0100）表示最大音量。如果为音频track，该值有效，否则为0 288 | reserved3 uint16be 289 | 290 | matrix [36]byte // 视频变换矩阵 291 | width fixed32be // 宽 292 | height fixed32be // 高，均为 [16.16] 格式值，与sample描述中的实际画面大小比值，用于播放时的展示宽高 293 | } 294 | 295 | trkbox = box { 296 | eval _body do case typ { 297 | "tkhd": tkhd 298 | "mdia": mdia 299 | "edts": edts 300 | default: boxtr 301 | } 302 | } 303 | 304 | // Track Box 305 | // 306 | trak = { 307 | vals *trkbox 308 | } 309 | 310 | // -------------------------------------------------------------- 311 | 312 | // Movie Header Box 313 | // 314 | mvhd = { 315 | version byte 316 | flags uint24be 317 | ctime uint32be // 创建时间（相对于UTC时间1904-01-01零点的秒数） 318 | mtime uint32be // 修改时间 319 | time_scale uint32be // 文件媒体在1秒时间内的刻度值，可以理解为1秒长度的时间单元数 320 | duration uint32be // 该track的时间长度，用duration和time_scale值可以计算track时长 321 | rate fixed32be // 推荐播放速率，高16位和低16位分别为小数点整数部分和小数部分，即[16.16] 格式，该值为1.0（0x00010000）表示正常前向播放 322 | volume fixed16be // 与rate类似，[8.8] 格式，1.0（0x0100）表示最大音量 323 | reserved [10]byte 324 | matrix [36]byte // 视频变换矩阵 325 | predefined [24]byte 326 | 327 | // 下一个track使用的id号 328 | // 329 | next_track_id uint32be 330 | } 331 | 332 | iods = { 333 | let class = "Initial Object Descriptor" 334 | version byte 335 | flags uint24be 336 | unknown *byte 337 | } 338 | 339 | movbox = box { 340 | eval _body do case typ { 341 | "mvhd": mvhd 342 | "iods": iods 343 | "trak": trak 344 | default: boxtr 345 | } 346 | dump 347 | } 348 | 349 | // Movie Box 350 | // 351 | moov = dump *movbox 352 | 353 | // -------------------------------------------------------------- 354 | 355 | // File Type Box 356 | // 357 | ftyp = { 358 | major_brand [4]char 359 | minor_version uint32be 360 | compatible_brands *[4]char 361 | dump 362 | } 363 | 364 | free = { 365 | let bodyLength = len(_body) 366 | dump 367 | } 368 | 369 | mdat = { 370 | dump 371 | } 372 | 373 | gblbox = box { 374 | // 375 | // 注意：moov 可能太大了，如果等解析完再 dump 不太友好 376 | // 377 | eval _body do case typ { 378 | "ftyp": ftyp 379 | "moov": moov 380 | "free": free 381 | "mdat": mdat 382 | default: boxtr dump 383 | } 384 | } 385 | 386 | doc = *gblbox 387 | 388 | // -------------------------------------------------------------- 389 | -------------------------------------------------------------------------------- /formats/rtmp.bpl: -------------------------------------------------------------------------------- 1 | const ( 2 | VERBOSE = 0 3 | RAWDATA = 0 4 | ) 5 | 6 | init = { 7 | global filterFlashVer = BPL_FILTER["flashVer"] 8 | global filterReqMode = BPL_FILTER["reqMode"] 9 | global filterPlay = (filterReqMode == "play") 10 | global filterDir = BPL_FILTER["dir"] 11 | 12 | if filterDir != undefined { 13 | if filterDir != BPL_DIRECTION { 14 | do exit(0) 15 | } 16 | } 17 | 18 | global fmsKey = bytes.from([ 19 | 0x47, 0x65, 0x6e, 0x75, 0x69, 0x6e, 0x65, 0x20, 20 | 0x41, 0x64, 0x6f, 0x62, 0x65, 0x20, 0x46, 0x6c, 21 | 0x61, 0x73, 0x68, 0x20, 0x4d, 0x65, 0x64, 0x69, 22 | 0x61, 0x20, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 23 | 0x20, 0x30, 0x30, 0x31, // Genuine Adobe Flash Media Server 001 24 | 0xf0, 0xee, 0xc2, 0x4a, 0x80, 0x68, 0xbe, 0xe8, 25 | 0x2e, 0x00, 0xd0, 0xd1, 0x02, 0x9e, 0x7e, 0x57, 26 | 0x6e, 0xec, 0x5d, 0x2d, 0x29, 0x80, 0x6f, 0xab, 27 | 0x93, 0xb8, 0xe6, 0x36, 0xcf, 0xeb, 0x31, 0xae, 28 | ]) 29 | global fpKey = bytes.from([ 30 | 0x47, 0x65, 0x6E, 0x75, 0x69, 0x6E, 0x65, 0x20, 31 | 0x41, 0x64, 0x6F, 0x62, 0x65, 0x20, 0x46, 0x6C, 32 | 0x61, 0x73, 0x68, 0x20, 0x50, 0x6C, 0x61, 0x79, 33 | 0x65, 0x72, 0x20, 0x30, 0x30, 0x31, // Genuine Adobe Flash Player 001 34 | 0xF0, 0xEE, 0xC2, 0x4A, 0x80, 0x68, 0xBE, 0xE8, 35 | 0x2E, 0x00, 0xD0, 0xD1, 0x02, 0x9E, 0x7E, 0x57, 36 | 0x6E, 0xEC, 0x5D, 0x2D, 0x29, 0x80, 0x6F, 0xAB, 37 | 0x93, 0xB8, 0xE6, 0x36, 0xCF, 0xEB, 0x31, 0xAE, 38 | ]) 39 | 40 | if BPL_DIRECTION == "REQ" { 41 | global handshakeKey = fpKey[:30] 42 | } else { 43 | global handshakeKey = fmsKey[:36] 44 | } 45 | 46 | global lastMsgs = mkmap("int:var") 47 | global chunksize = 128 48 | global objectend = errors.new("object end") 49 | global limitTypes = { 50 | 0: "Hard", 51 | 1: "Soft", 52 | 2: "Dynamic", 53 | } 54 | 55 | global audioFormats = { 56 | 0: "Linear PCM", 57 | 1: "ADPCM", 58 | 2: "MP3", 59 | 3: "Linear PCM le", 60 | 4: "Nellymoser 16kHz", 61 | 5: "Nellymoser 8kHz", 62 | 6: "Nellymoser", 63 | 7: "G711 A-law", 64 | 8: "G711 mu-law", 65 | 9: "Reserved", 66 | 10: "AAC", 67 | 11: "Speex", 68 | 12: "MP3 8kHz", 69 | 13: "Device Specific", 70 | } 71 | global audioRates = ["5.5 kHz", "11 kHz", "22 kHz", "44 kHz"] 72 | global audioBits = ["8 bits", "16 bits"] 73 | global audioChannels = ["Mono", "Stereo"] 74 | 75 | global videoTypes = { 76 | 1: "AVC keyframe", 77 | 2: "AVC inter frame", 78 | 3: "H.263 disposable inter frame", 79 | 4: "generated keyframe", 80 | 5: "video info/command frame", 81 | } 82 | global videoCodecs = { 83 | 1: "JPEG", 84 | 2: "H.263", 85 | 3: "screen video", 86 | 4: "On2 VP6", 87 | 5: "On2 VP6 with alpha channel", 88 | 6: "screen video v2", 89 | 7: "AVC", 90 | } 91 | global avcTypes = { 92 | 0: "sequence header", 93 | 1: "NALU", 94 | 2: "end of sequence", 95 | } 96 | } 97 | 98 | // -------------------------------------------------------------- 99 | 100 | AMF0_NUMBER = { 101 | val float64be 102 | if VERBOSE == 0 { 103 | return val 104 | } 105 | } 106 | 107 | AMF0_BOOLEAN = { 108 | val byte 109 | if VERBOSE == 0 { 110 | return byte != 0 111 | } 112 | } 113 | 114 | AMF0_STRING = { 115 | len uint16be 116 | val [len]char 117 | if VERBOSE == 0 { 118 | return val 119 | } 120 | } 121 | 122 | AMF0_OBJECT_ITEMS = { 123 | _key AMF0_STRING 124 | _val AMF0_TYPE 125 | let items = mkslice("var", 2) 126 | do set(items, 0, _key, 1, _val) 127 | if _val != objectend { 128 | _next AMF0_OBJECT_ITEMS 129 | let items = append(items, _next.items...) 130 | } 131 | } 132 | 133 | AMF0_OBJECT_NORMAL = { 134 | val AMF0_OBJECT_ITEMS 135 | let n = len(val.items) 136 | return mapFrom(val.items[:n-2]...) // 去掉了最后的 objectend 137 | } 138 | 139 | AMF0_OBJECT_VERBOSE = { 140 | _key AMF0_STRING 141 | _val AMF0_TYPE 142 | let items = [{"key": _key, "val": _val}] 143 | if _val.marker != 0x09 { 144 | _next AMF0_OBJECT_VERBOSE 145 | let items = append(items, _next.items...) 146 | } 147 | } 148 | 149 | AMF0_OBJECT = if VERBOSE do AMF0_OBJECT_VERBOSE else AMF0_OBJECT_NORMAL 150 | 151 | AMF0_STRICT_ARRAY = { 152 | len uint32be 153 | objs [len]AMF0_TYPE 154 | if VERBOSE == 0 { 155 | return objs 156 | } 157 | } 158 | 159 | AMF0_MOVIECLIP = { 160 | body *byte 161 | fatal "todo - AMF0_MOVIECLIP" 162 | } 163 | 164 | AMF0_NULL = { 165 | if VERBOSE { 166 | let val = nil 167 | } else { 168 | return nil 169 | } 170 | } 171 | 172 | AMF0_UNDEFINED = { 173 | if VERBOSE { 174 | let val = undefined 175 | } else { 176 | return undefined 177 | } 178 | } 179 | 180 | AMF0_REFERENCE = { 181 | reference uint16be 182 | } 183 | 184 | AMF0_ECMA_ARRAY = { 185 | len uint32be 186 | val AMF0_OBJECT 187 | if VERBOSE == 0 { 188 | return val 189 | } 190 | } 191 | 192 | AMF0_OBJECT_END = if VERBOSE do nil else { 193 | return objectend 194 | } 195 | 196 | AMF0_DATE = { 197 | timestamp float64be 198 | tz uint16be 199 | } 200 | 201 | AMF0_LONG_STRING = { 202 | len uint32be 203 | val [len]char 204 | if VERBOSE == 0 { 205 | return val 206 | } 207 | } 208 | 209 | AMF0_UNSUPPORTED = { 210 | body *byte 211 | fatal "todo - AMF0_UNSUPPORTED" 212 | } 213 | 214 | AMF0_RECORDSET = { 215 | body *byte 216 | fatal "todo - AMF0_RECORDSET" 217 | } 218 | 219 | AMF0_XML_DOCUMENT = AMF0_LONG_STRING 220 | 221 | AMF0_TYPED_OBJECT = { 222 | type AMF0_STRING 223 | val AMF0_OBJECT 224 | } 225 | 226 | AMF0_ACMPLUS_OBJECT = { // Switch to AMF3 227 | body *byte 228 | fatal "todo - AMF0_ACMPLUS_OBJECT" 229 | } 230 | 231 | AMF0_TYPE = { 232 | marker byte 233 | case marker { 234 | 0x00: AMF0_NUMBER 235 | 0x01: AMF0_BOOLEAN 236 | 0x02: AMF0_STRING 237 | 0x03: AMF0_OBJECT 238 | 0x04: AMF0_MOVIECLIP 239 | 0x05: AMF0_NULL 240 | 0x06: AMF0_UNDEFINED 241 | 0x07: AMF0_REFERENCE 242 | 0x08: AMF0_ECMA_ARRAY 243 | 0x09: AMF0_OBJECT_END 244 | 0x0a: AMF0_STRICT_ARRAY 245 | 0x0b: AMF0_DATE 246 | 0x0c: AMF0_LONG_STRING 247 | 0x0d: AMF0_UNSUPPORTED 248 | 0x0e: AMF0_RECORDSET 249 | 0x0f: AMF0_XML_DOCUMENT 250 | 0x10: AMF0_TYPED_OBJECT 251 | 0x11: AMF0_ACMPLUS_OBJECT 252 | } 253 | } 254 | 255 | AMF0_CMDDATA = { 256 | cmd AMF0_TYPE 257 | transactionId AMF0_TYPE 258 | value *AMF0_TYPE 259 | if filterFlashVer != undefined { 260 | if cmd == "connect" { 261 | if value[0].flashVer != filterFlashVer { 262 | do exit(0) 263 | } 264 | } 265 | } 266 | if filterPlay { 267 | if cmd == "FCPublish" { 268 | do exit(0) 269 | } 270 | } 271 | } 272 | 273 | AMF0 = { 274 | msg *AMF0_TYPE 275 | } 276 | 277 | AMF0_CMD = { 278 | msg AMF0_CMDDATA 279 | } 280 | 281 | // -------------------------------------------------------------- 282 | 283 | AMF3_UNDEFINED = AMF0_UNDEFINED 284 | 285 | AMF3_NULL = AMF0_NULL 286 | 287 | AMF3_FALSE = { 288 | if VERBOSE { 289 | let val = false 290 | } else { 291 | return false 292 | } 293 | } 294 | 295 | AMF3_TRUE = { 296 | if VERBOSE { 297 | let val = true 298 | } else { 299 | return true 300 | } 301 | } 302 | 303 | AMF3_INT = { 304 | b1 byte 305 | if b1 & 0x80 { 306 | let b1 = b1 & 0x7f 307 | b2 byte 308 | if b2 & 0x80 { 309 | let b2 = b2 & 0x7f 310 | b3 byte 311 | if b3 & 0x80 { 312 | let b3 = b3 & 0x7f 313 | b4 byte 314 | return (b1 << 22) | (b2 << 15) | (b3 << 8) | b4 315 | } else { 316 | return (b1 << 14) | (b2 << 7) | b3 317 | } 318 | } else { 319 | return (b1 << 7) | b2 320 | } 321 | } else { 322 | return int(b1) 323 | } 324 | } 325 | 326 | AMF3_INTEGER_VERBOSE = { 327 | val AMF3_INT 328 | } 329 | 330 | AMF3_INTEGER = if VERBOSE do AMF3_INTEGER_VERBOSE else AMF3_INT 331 | 332 | AMF3_DOUBLE = { 333 | val float64be 334 | if VERBOSE == 0 { 335 | return val 336 | } 337 | } 338 | 339 | AMF3_STRING = { 340 | tag AMF3_INT 341 | assert (tag & 1) != 0 // reference unsupported 342 | if tag & 1 { 343 | val [tag >> 1]char 344 | } 345 | if VERBOSE == 0 { 346 | return val 347 | } 348 | } 349 | 350 | AMF3_XMLDOC = { 351 | body *byte 352 | } 353 | 354 | AMF3_DATE = { 355 | tag AMF3_INT 356 | assert (tag & 1) != 0 // reference unsupported 357 | timestamp float64be 358 | let tz = tag >> 1 359 | if VERBOSE == 0 { 360 | do unset("tag") 361 | } 362 | } 363 | 364 | AMF3_ARRAY = { 365 | tag AMF3_INT 366 | assert (tag & 1) != 0 // reference unsupported 367 | let len = tag >> 1 368 | body *byte 369 | fatal "todo - AMF3_ARRAY" 370 | } 371 | 372 | AMF3_OBJECT = { 373 | body *byte 374 | fatal "todo - AMF3_OBJECT" 375 | } 376 | 377 | AMF3_XML = { 378 | body *byte 379 | fatal "todo - AMF3_XML" 380 | } 381 | 382 | AMF3_BYTE_ARRAY = { 383 | body *byte 384 | do println(hex.dump(body)) 385 | fatal "todo - AMF3_BYTE_ARRAY" 386 | } 387 | 388 | AMF3_VECTOR_INT = { 389 | body *byte 390 | fatal "todo - AMF3_VECTOR_INT" 391 | } 392 | 393 | AMF3_VECTOR_UINT = { 394 | body *byte 395 | fatal "todo - AMF3_VECTOR_UINT" 396 | } 397 | 398 | AMF3_VECTOR_DOUBLE = { 399 | body *byte 400 | fatal "todo - AMF3_VECTOR_DOUBLE" 401 | } 402 | 403 | AMF3_VECTOR_OBJECT = { 404 | body *byte 405 | fatal "todo - AMF3_VECTOR_OBJECT" 406 | } 407 | 408 | AMF3_DICTIONARY = { 409 | body *byte 410 | fatal "todo - AMF3_DICTIONARY" 411 | } 412 | 413 | AMF3_TYPE = { 414 | marker byte 415 | case marker { 416 | 0x00: AMF3_UNDEFINED 417 | 0x01: AMF3_NULL 418 | 0x02: AMF3_FALSE 419 | 0x03: AMF3_TRUE 420 | 0x04: AMF3_INTEGER 421 | 0x05: AMF3_DOUBLE 422 | 0x06: AMF3_STRING 423 | 0x07: AMF3_XMLDOC 424 | 0x08: AMF3_DATE 425 | 0x09: AMF3_ARRAY 426 | 0x0a: AMF3_OBJECT 427 | 0x0b: AMF3_XML 428 | 0x0c: AMF3_BYTE_ARRAY 429 | 0x0d: AMF3_VECTOR_INT 430 | 0x0e: AMF3_VECTOR_UINT 431 | 0x0f: AMF3_VECTOR_DOUBLE 432 | 0x10: AMF3_VECTOR_OBJECT 433 | 0x11: AMF3_DICTIONARY 434 | } 435 | } 436 | 437 | AMF3_CMDDATA = { 438 | cmd AMF3_TYPE 439 | transactionId AMF3_TYPE 440 | value *AMF3_TYPE 441 | } 442 | 443 | AMF3 = { 444 | msg *AMF3_TYPE 445 | } 446 | 447 | AMF3_CMD = { 448 | msg AMF3_CMDDATA 449 | } 450 | 451 | // -------------------------------------------------------------- 452 | 453 | AudioData = { 454 | tag byte 455 | let format = tag >> 4 456 | let formatKind = audioFormats[format] 457 | let rate = (tag>>2) & 3 458 | let rateKind = audioRates[rate] 459 | let bits = (tag>>1) & 1 460 | let bitsKind = audioBits[bits] 461 | let channel = tag & 1 462 | let channelKind = audioChannels[channel] 463 | 464 | if formatKind == "AAC" { 465 | aacPacketType byte 466 | if aacPacketType == 0 { 467 | let aacPacketTypeKind = "sequence header" 468 | } else { 469 | let aacPacketTypeKind = "raw data" 470 | } 471 | } 472 | if RAWDATA != 0 { 473 | raw *byte 474 | } 475 | } 476 | 477 | Audio = { 478 | audio AudioData 479 | } 480 | 481 | // -------------------------------------------------------------- 482 | 483 | VideoData = { 484 | tag byte 485 | let type = tag >> 4 486 | let typeKind = videoTypes[type] 487 | let codec = tag & 0xf 488 | let codecKind = videoCodecs[codec] 489 | 490 | if codecKind == "AVC" { 491 | avctype byte 492 | compositionTime uint24be 493 | let avctypeKind = avcTypes[int(avctype)] 494 | if avctype == 0 { // sequence header 495 | configurationVersion byte 496 | avcProfileIndication byte 497 | profileCompatibility byte 498 | avcLevelIndication byte 499 | lengthSizeMinusOne byte 500 | numOfSPS byte // SPS = SequenceParameterSets 501 | bytesOfSPS uint16be 502 | dataOfSPS [bytesOfSPS]byte // SPS包含视频长、宽的信息 503 | numOfPPS byte // PPS = PictureParameterSets 504 | bytesOfPPS uint16be 505 | dataOfPPS [bytesOfPPS]byte 506 | let lengthSizeMinusOne = (lengthSizeMinusOne & 3) + 1 507 | let numOfSPS = numOfSPS & 0x1F 508 | let numOfPPS = numOfPPS & 0x1F 509 | unknown *byte 510 | } 511 | } 512 | if RAWDATA != 0 { 513 | raw *byte 514 | } 515 | } 516 | 517 | Video = { 518 | video VideoData 519 | } 520 | 521 | // -------------------------------------------------------------- 522 | 523 | SetChunkSize = { 524 | size uint32be 525 | let chunksize = size 526 | } 527 | 528 | Abort = { 529 | csid uint32be 530 | let _last = lastMsgs[csid] 531 | do set(_last, "remain", 0) 532 | } 533 | 534 | UserControl = { 535 | evType uint16be 536 | evData *byte 537 | } 538 | 539 | AckWinsize = { 540 | winsize uint32be 541 | } 542 | 543 | SetPeerBandwidth = { 544 | winsize uint32be 545 | limitType byte 546 | let limitTypeKind = limitTypes[int(limitType)] 547 | } 548 | 549 | // -------------------------------------------------------------- 550 | 551 | Handshake0 = { 552 | h0 byte 553 | assert h0 == 3 554 | } 555 | 556 | Handshake1Verify = { 557 | let i = _diggestOffset 558 | let off = (_h[i] + _h[i+1] + _h[i+2] + _h[i+3]) % (764 - 32 - 4) + i + 4 559 | let data1 = _h[:off] 560 | let data2 = _h[off+32:] 561 | let digest = _h[off:off+32] 562 | let h = hmac.new(sha256.new, handshakeKey) 563 | do h.write(data1) 564 | do h.write(data2) 565 | let ok = bytes.equal(h.sum(nil), digest) 566 | } 567 | 568 | Handshake1 = { 569 | _h1 [1536]byte 570 | 571 | eval _h1 do { 572 | time uint32be 573 | version uint32be 574 | global _h = _h1 575 | global _diggestOffset = 772 576 | _verify1 Handshake1Verify 577 | if _verify1.ok { 578 | let digest = _verify1.digest 579 | let msg = "Flash after 10.0.32.18" 580 | } else { 581 | global _diggestOffset = 8 582 | _verify2 Handshake1Verify 583 | if _verify2.ok { 584 | let digest = _verify2.digest 585 | let msg = "Flash before 10.0.32.18" 586 | } 587 | } 588 | } 589 | } 590 | 591 | Handshake2 = { 592 | _h2 [1536]byte 593 | } 594 | 595 | // -------------------------------------------------------------- 596 | 597 | AggregateItemHeader = { 598 | typeid byte 599 | length uint24be 600 | ts uint24be 601 | streamid uint32be 602 | } 603 | 604 | AggregateItem = { 605 | header AggregateItemHeader 606 | _body [header.length]byte 607 | back uint32be 608 | assert back == header.length + 11 609 | eval _body do case header.typeid { 610 | 18: AMF0 611 | 20: AMF0_CMD 612 | 15: AMF3 613 | 17: AMF3_CMD 614 | 8: Audio 615 | 9: Video 616 | default: let body = _body 617 | } 618 | } 619 | 620 | AggregateMsg = { 621 | msgs *AggregateItem 622 | } 623 | 624 | // -------------------------------------------------------------- 625 | 626 | ChunkHeader = { 627 | _tag byte 628 | 629 | let format = (_tag >> 6) & 3 630 | assert format <= 3 631 | 632 | let csid = _tag & 0x3f 633 | if csid == 0 { 634 | _v byte 635 | let csid = _v + 0x40 636 | } elif csid == 1 { 637 | _v uint16le 638 | let csid = _v + 0x40 639 | } 640 | 641 | let _last = lastMsgs[csid] 642 | 643 | if format < 3 { 644 | ts uint24be 645 | if format < 2 { 646 | length uint24be 647 | typeid byte 648 | if format < 1 { 649 | streamid uint32le 650 | } else { 651 | let ts = ts + _last["ts"] 652 | let streamid = _last["streamid"] 653 | } 654 | let remain = 0 655 | } else { 656 | let ts = ts + _last["ts"] 657 | let length = _last["length"] 658 | let typeid = _last["typeid"] 659 | let streamid = _last["streamid"] 660 | let remain = _last["remain"] 661 | } 662 | if ts == 0xffffff { 663 | tsext uint32be 664 | } 665 | } else { 666 | let ts = _last["ts"] 667 | let length = _last["length"] 668 | let typeid = _last["typeid"] 669 | let streamid = _last["streamid"] 670 | let remain = _last["remain"] 671 | if ts == 0xffffff { 672 | tsext uint32be 673 | } 674 | } 675 | 676 | if remain == 0 { 677 | let remain = length 678 | let _body = bytes.buffer() 679 | } else { 680 | let _body = _last["body"] 681 | } 682 | } 683 | 684 | Chunk = { 685 | header ChunkHeader 686 | 687 | let _length = chunksize 688 | if header.remain < _length { 689 | let _length = header.remain 690 | } 691 | 692 | let _header = { 693 | "ts": header.ts, 694 | "length": header.length, 695 | "typeid": header.typeid, 696 | "streamid": header.streamid, 697 | "remain": header.remain - _length, 698 | "body": header._body, 699 | } 700 | do set(lastMsgs, header.csid, _header) 701 | 702 | data [_length]byte 703 | do header._body.write(data) 704 | if _length > 16 { 705 | let data = data[:16] 706 | } 707 | 708 | if _header.remain == 0 { 709 | let _body = header._body.bytes() 710 | if header.csid == 2 && header.streamid == 0 { 711 | eval _body do case header.typeid { 712 | 1: SetChunkSize 713 | 2: Abort 714 | 4: UserControl 715 | 5: AckWinsize 716 | 6: SetPeerBandwidth 717 | default: let body = _body 718 | } 719 | } else { 720 | eval _body do case header.typeid { 721 | 18: AMF0 722 | 20: AMF0_CMD 723 | 15: AMF3 724 | 17: AMF3_CMD 725 | 22: AggregateMsg 726 | 8: Audio 727 | 9: Video 728 | default: let body = _body 729 | } 730 | } 731 | } elif RAWDATA == 0 { 732 | return undefined 733 | } 734 | } 735 | 736 | doc = init Handshake0 Handshake1 Handshake2 dump *(Chunk dump) 737 | 738 | // -------------------------------------------------------------- 739 | -------------------------------------------------------------------------------- /formats/rtmp_hexdump.bpl: -------------------------------------------------------------------------------- 1 | init = { 2 | global filterDir = BPL_FILTER["dir"] 3 | 4 | if filterDir != undefined { 5 | if filterDir != BPL_DIRECTION { 6 | do exit(0) 7 | } 8 | } 9 | 10 | global msgs = mkmap("int:var") 11 | global chunksize = 128 12 | global typeidKinds = { 13 | 1: "SetChunkSize", 14 | 2: "Abort", 15 | 4: "UserControl", 16 | 5: "AckWinsize", 17 | 6: "SetPeerBandwidth", 18 | 8: "Audio", 19 | 9: "Video", 20 | 18: "AMF0", 21 | 20: "AMF0_CMD", 22 | 15: "AMF3", 23 | 17: "AMF3_CMD", 24 | } 25 | } 26 | 27 | SetChunkSize = { 28 | size uint32be 29 | let chunksize = size 30 | } 31 | 32 | Abort = { 33 | csid uint32be 34 | let _last = msgs[csid] 35 | do set(_last, "remain", 0) 36 | } 37 | 38 | Handshake0 = { 39 | h0 [1]byte 40 | } 41 | 42 | Handshake1 = { 43 | h1 [1536]byte 44 | } 45 | 46 | Handshake2 = { 47 | h2 [1536]byte 48 | } 49 | 50 | ChunkHeader = { 51 | 52 | global b = bytes.buffer() 53 | 54 | _tag byte 55 | do b.writeByte(_tag) 56 | 57 | let format = (_tag >> 6) & 3 58 | assert format <= 3 59 | 60 | let csid = _tag & 0x3f 61 | if csid == 0 { 62 | _v byte 63 | do b.writeByte(_v) 64 | let csid = _v + 0x40 65 | } elif csid == 1 { 66 | _v2 [2]byte 67 | eval _v2 do { 68 | _v uint16le 69 | let csid = _v + 0x40 70 | do b.write(_v2) 71 | } 72 | } 73 | 74 | let _last = msgs[csid] 75 | 76 | if format < 3 { 77 | _v3 [3]byte 78 | eval _v3 do { 79 | ts uint24be 80 | do b.write(_v3) 81 | } 82 | if format < 2 { 83 | _v4 [4]byte 84 | eval _v4 do { 85 | length uint24be 86 | typeid byte 87 | do b.write(_v4) 88 | } 89 | if format < 1 { 90 | _v5 [4]byte 91 | eval _v5 do { 92 | streamid uint32le 93 | do b.write(_v5) 94 | } 95 | } else { 96 | let ts = ts + _last["ts"] 97 | let streamid = _last["streamid"] 98 | } 99 | let remain = 0 100 | } else { 101 | let ts = ts + _last["ts"] 102 | let length = _last["length"] 103 | let typeid = _last["typeid"] 104 | let streamid = _last["streamid"] 105 | let remain = _last["remain"] 106 | } 107 | } else { 108 | let ts = _last["ts"] 109 | let length = _last["length"] 110 | let typeid = _last["typeid"] 111 | let streamid = _last["streamid"] 112 | let remain = _last["remain"] 113 | } 114 | 115 | let typeidKind = typeidKinds[int(typeid)] 116 | if remain == 0 { 117 | let remain = length 118 | let _body = bytes.buffer() 119 | } else { 120 | let _body = _last["body"] 121 | } 122 | } 123 | 124 | Chunk = { 125 | header ChunkHeader 126 | 127 | let _length = chunksize 128 | if header.remain < _length { 129 | let _length = header.remain 130 | } 131 | 132 | let _header = { 133 | "ts": header.ts, 134 | "length": header.length, 135 | "typeid": header.typeid, 136 | "streamid": header.streamid, 137 | "remain": header.remain - _length, 138 | "body": header._body, 139 | } 140 | do set(msgs, header.csid, _header) 141 | 142 | _data [_length]byte 143 | do header._body.write(_data) 144 | do b.write(_data) 145 | let raw = b.bytes() 146 | 147 | if _header.remain == 0 { 148 | let _body = header._body.bytes() 149 | if header.csid == 2 && header.streamid == 0 { 150 | eval _body do case header.typeid { 151 | 1: SetChunkSize 152 | 2: Abort 153 | default: nil 154 | } 155 | } 156 | } 157 | } 158 | 159 | doc = init Handshake0 Handshake1 Handshake2 dump *(Chunk dump) 160 | -------------------------------------------------------------------------------- /formats/ts.bpl: -------------------------------------------------------------------------------- 1 | doc = nil 2 | -------------------------------------------------------------------------------- /formats/wav.bpl: -------------------------------------------------------------------------------- 1 | // ------------------------------------------------------------------------------------- 2 | 3 | listChunk = { 4 | ListType [4]char 5 | dump 6 | Chunks *chunk 7 | } 8 | 9 | fmtChunk = { 10 | Format uint16 11 | case Format { 12 | 0x01: let FormatDesc = "PCM" 13 | 0x03: let FormatDesc = "IEEE-Float" 14 | 0x06: let FormatDesc = "ALAW" 15 | 0x07: let FormatDesc = "MULAW" 16 | 0x11: let FormatDesc = "ADPCM" 17 | 0xfffe: let FormatDesc = "Extensible" 18 | default: let FormatDesc = "Unknown" 19 | } 20 | Channels uint16 21 | SamplesPerSec uint32 22 | BytesPerSec uint32 23 | BlockAlign uint16 24 | BitsPerSample uint16 25 | if Size > 16 { 26 | ExtraSize uint16 27 | ExtraData [ExtraSize]byte 28 | eval ExtraData { 29 | SamplesPerBlock uint16 30 | } 31 | } 32 | global nFormat = Format 33 | global nChannel = Channels 34 | global nBlockAlign = BlockAlign 35 | let Body = _body 36 | dump 37 | } 38 | 39 | factChunk = { 40 | DataFactSize uint32 // 数据转换为PCM格式后的大小 41 | let Body = _body 42 | dump 43 | } 44 | 45 | defaultChunk = { 46 | if Size < 128 { 47 | Data [Size]byte 48 | } else { 49 | Summary [128]byte 50 | skip Size-128 51 | } 52 | dump 53 | } 54 | 55 | // ------------------------------------------------------------------------------------- 56 | 57 | monoBlockHeader = { 58 | Sample0 int16 // block 中第一个未压缩的采样值 59 | Index byte 60 | Reserved byte 61 | } 62 | 63 | block = { 64 | Header [nChannel]monoBlockHeader 65 | Data [nBlockAlign-nChannel*4]byte 66 | dump 67 | } 68 | 69 | dataChunk = case nFormat { 70 | 0x11: dump *block 71 | default: defaultChunk 72 | } 73 | 74 | // ------------------------------------------------------------------------------------- 75 | 76 | chunk = { 77 | ID [4]char 78 | Size uint32 79 | _body [Size]byte 80 | eval _body do case ID { 81 | "LIST": listChunk 82 | "fmt ": fmtChunk 83 | "fact": factChunk 84 | "data": dataChunk 85 | default: defaultChunk 86 | } 87 | } 88 | 89 | riffHeader = { 90 | ID [4]char 91 | assert ID == "RIFF" 92 | 93 | Size uint32 94 | Format [4]char 95 | dump 96 | } 97 | 98 | doc = riffHeader *chunk 99 | 100 | // ------------------------------------------------------------------------------------- 101 | -------------------------------------------------------------------------------- /formats/webrtc.bpl: -------------------------------------------------------------------------------- 1 | doc = nil 2 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/goplus/bpl 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/qiniu/text v1.9.2 7 | github.com/qiniu/x v1.11.9 8 | github.com/xushiwei/qlang v1.2.2 9 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 10 | gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 11 | gopkg.in/yaml.v2 v2.4.0 // indirect 12 | ) 13 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= 2 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 3 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 4 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 5 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 6 | github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= 7 | github.com/peterh/liner v1.2.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= 8 | github.com/qiniu/text v1.9.2 h1:1+aVZrM+LaDMf6PFL+U3q9N7zyj3YBqg2EnKeaHdLxU= 9 | github.com/qiniu/text v1.9.2/go.mod h1:v9y6inngR3QcljoFfgmBN+ukkI4WBItyzxMfyHao1C4= 10 | github.com/qiniu/x v1.11.9 h1:IfQNdeNcK43Q1+b/LdrcqmWjlhxq051YVBnua8J2qN8= 11 | github.com/qiniu/x v1.11.9/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= 12 | github.com/xushiwei/qlang v1.2.2 h1:/SWpFRYndjbXp4VumHlhTj8yJ9HtjnLexPz3v3bSUGg= 13 | github.com/xushiwei/qlang v1.2.2/go.mod h1:Dgju+HSaZhamzLvhCNoFEo3ISC8Y1wId8ZlYMCdTITE= 14 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 15 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 16 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 17 | gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= 18 | gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= 19 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 20 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 21 | -------------------------------------------------------------------------------- /go/codegen/bytes.go: -------------------------------------------------------------------------------- 1 | package codegen 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | ) 8 | 9 | // ----------------------------------------------------------------------------- 10 | 11 | const hextable = "0123456789abcdef" 12 | 13 | // BytesFrom generates a byte slice variable from a byte slice. 14 | // 15 | func BytesFrom(w io.Writer, name string, b []byte) (err error) { 16 | 17 | _, err = fmt.Fprintf(w, "var %s = []byte{\n", name) 18 | if err != nil { 19 | return 20 | } 21 | buf := make([]byte, 8*6+1) 22 | for i := 0; i < len(b); i += 8 { 23 | buf[0] = '\t' 24 | base := 1 25 | max := 8 26 | if len(b)-i < max { 27 | max = len(b) - i 28 | } 29 | for j := 0; j < max; j++ { 30 | c := b[i+j] 31 | buf[base] = '0' 32 | buf[base+1] = 'x' 33 | buf[base+2] = hextable[c>>4] 34 | buf[base+3] = hextable[c&0xf] 35 | buf[base+4] = ',' 36 | buf[base+5] = ' ' 37 | base += 6 38 | } 39 | buf[base-1] = '\n' 40 | _, err = w.Write(buf[:base]) 41 | if err != nil { 42 | return 43 | } 44 | } 45 | _, err = w.Write([]byte{'}', '\n'}) 46 | return 47 | } 48 | 49 | // BytesFromFile generates a byte slice variable from a file. 50 | // 51 | func BytesFromFile(w io.Writer, name string, file string) (err error) { 52 | 53 | b, err := ioutil.ReadFile(file) 54 | if err != nil { 55 | return 56 | } 57 | return BytesFrom(w, name, b) 58 | } 59 | 60 | // ----------------------------------------------------------------------------- 61 | -------------------------------------------------------------------------------- /go/codegen/bytes_test.go: -------------------------------------------------------------------------------- 1 | package codegen 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | ) 7 | 8 | func doTestBytesFrom(t *testing.T, name string, b []byte, expected string) { 9 | 10 | w := bytes.NewBuffer(nil) 11 | err := BytesFrom(w, name, b) 12 | if err != nil { 13 | t.Fatal("BytesFrom failed:", err) 14 | } 15 | ret := w.String() 16 | if ret != expected { 17 | t.Fatal("bytes:", ret) 18 | } 19 | } 20 | 21 | const expected0 = `var foo = []byte{ 22 | } 23 | ` 24 | 25 | const expected1 = `var foo = []byte{ 26 | 0x02, 0x03, 0x03, 0x09, 0x00, 27 | } 28 | ` 29 | 30 | const expected2 = `var foo = []byte{ 31 | 0x02, 0x03, 0x03, 0x09, 0x00, 0x02, 0xe8, 0x60, 32 | 0xf8, 33 | } 34 | ` 35 | 36 | func TestBytesFrom(t *testing.T) { 37 | 38 | doTestBytesFrom(t, "foo", nil, expected0) 39 | doTestBytesFrom(t, "foo", []byte{2, 3, 3, 9, 0}, expected1) 40 | doTestBytesFrom(t, "foo", []byte{2, 3, 3, 9, 0, 2, 0xe8, 96, 0xf8}, expected2) 41 | } 42 | -------------------------------------------------------------------------------- /hex/undump.go: -------------------------------------------------------------------------------- 1 | package hex 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/hex" 7 | "io" 8 | "os" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | func undump1(w io.Writer, text string) { // bd c2 c1 24 93 55 2a 4d 14 | 15 | b, err := hex.DecodeString(strings.Replace(text, " ", "", -1)) 16 | if err != nil { 17 | panic(err) 18 | } 19 | w.Write(b) 20 | } 21 | 22 | func undumpLine(w io.Writer, line string) { 23 | 24 | parts := strings.SplitN(line, " ", 4) 25 | max := 3 26 | if len(parts) < max { 27 | max = len(parts) 28 | } 29 | addr := parts[0] 30 | if len(addr) < 8 { 31 | return 32 | } 33 | _, err := strconv.ParseInt(addr, 16, 64) 34 | if err != nil { 35 | return 36 | } 37 | for i := 1; i < max; i++ { 38 | undump1(w, parts[i]) 39 | } 40 | } 41 | 42 | // Undump reverts `hexdump -C binary` result back to a binary data. 43 | // 44 | func Undump(w io.Writer, in *bufio.Reader, filter string) { // filter = [REQ] 45 | 46 | fskip := true 47 | for { 48 | line, err := in.ReadString('\n') 49 | if filter != "" { 50 | if strings.HasPrefix(line, "[INFO]") { 51 | fskip = !strings.HasPrefix(line[6:], filter) 52 | continue 53 | } 54 | if fskip { 55 | continue 56 | } 57 | } 58 | undumpLine(w, line) 59 | if err != nil { 60 | if err != io.EOF { 61 | panic(err) 62 | } 63 | return 64 | } 65 | } 66 | } 67 | 68 | // UndumpText reverts `hexdump -C binary` result back to a binary data. 69 | // 70 | func UndumpText(w io.Writer, text string) { 71 | 72 | lines := strings.Split(text, "\n") 73 | for _, line := range lines { 74 | undumpLine(w, line) 75 | } 76 | } 77 | 78 | // TextReader returns a reader that reverts `hexdump -C binary` result back to a binary data. 79 | // 80 | func TextReader(text string) *bytes.Reader { 81 | 82 | var w bytes.Buffer 83 | UndumpText(&w, text) 84 | return bytes.NewReader(w.Bytes()) 85 | } 86 | 87 | // Reader returns a reader that reverts `hexdump -C binary` result back to a binary data. 88 | // 89 | func Reader(f io.Reader, filter string) (ret io.ReadCloser, err error) { 90 | 91 | pr, pw := io.Pipe() 92 | go func() { 93 | in := bufio.NewReader(f) 94 | Undump(pw, in, filter) 95 | pw.Close() 96 | }() 97 | 98 | return pr, nil 99 | } 100 | 101 | // Open returns a reader that reverts `hexdump -C binary` result back to a binary data. 102 | // 103 | func Open(fname string, filter string) (ret io.ReadCloser, err error) { 104 | 105 | f, err := os.Open(fname) 106 | if err != nil { 107 | return 108 | } 109 | 110 | pr, pw := io.Pipe() 111 | go func() { 112 | in := bufio.NewReader(f) 113 | Undump(pw, in, filter) 114 | pw.Close() 115 | f.Close() 116 | }() 117 | 118 | return pr, nil 119 | } 120 | -------------------------------------------------------------------------------- /mockd/mockd.go: -------------------------------------------------------------------------------- 1 | package mockd 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "io/ioutil" 7 | "net" 8 | "os" 9 | 10 | "github.com/goplus/bpl/hex" 11 | "github.com/qiniu/x/log" 12 | ) 13 | 14 | // ----------------------------------------------------------------------------- 15 | 16 | // A Server is a mockd server. 17 | // 18 | type Server struct { 19 | Addr string 20 | Listened chan bool 21 | Data io.ReaderAt 22 | Fsize int64 23 | } 24 | 25 | // ListenAndServe listens an address and serves requests. 26 | // 27 | func (p *Server) ListenAndServe() (err error) { 28 | 29 | addr := p.Addr 30 | l, err := net.Listen("tcp", addr) 31 | if err != nil { 32 | log.Fatalf("ListenAndServe(tcp proxy) %s failed: %v\n", addr, err) 33 | return 34 | } 35 | if p.Listened != nil { 36 | p.Listened <- true 37 | } 38 | err = p.Serve(l) 39 | if err != nil { 40 | log.Fatalf("ListenAndServe(mockd) %s failed: %v\n", addr, err) 41 | } 42 | return 43 | } 44 | 45 | // Serve serves requests. 46 | // 47 | func (p *Server) Serve(l net.Listener) (err error) { 48 | 49 | defer l.Close() 50 | 51 | for { 52 | c1, err1 := l.Accept() 53 | if err1 != nil { 54 | return err1 55 | } 56 | c := c1.(*net.TCPConn) 57 | go p.handle(c) 58 | } 59 | } 60 | 61 | func (p *Server) handle(c *net.TCPConn) { 62 | 63 | go func() { 64 | io.Copy(ioutil.Discard, c) 65 | c.CloseRead() 66 | }() 67 | 68 | r := io.NewSectionReader(p.Data, 0, p.Fsize) 69 | in := bufio.NewReader(r) 70 | hex.Undump(c, in, "[RESP]") 71 | c.CloseWrite() 72 | } 73 | 74 | // ----------------------------------------------------------------------------- 75 | 76 | // ListenAndServe listens an address and serves requests. 77 | // 78 | func ListenAndServe(addr string, file string) (err error) { 79 | 80 | f, err := os.Open(file) 81 | if err != nil { 82 | log.Fatalln("Open failed:", err) 83 | return 84 | } 85 | defer f.Close() 86 | 87 | fi, err := f.Stat() 88 | if err != nil { 89 | log.Fatalln("Stat failed:", err) 90 | return 91 | } 92 | 93 | server := &Server{ 94 | Addr: addr, 95 | Data: f, 96 | Fsize: fi.Size(), 97 | } 98 | return server.ListenAndServe() 99 | } 100 | 101 | // ----------------------------------------------------------------------------- 102 | -------------------------------------------------------------------------------- /repeat.go: -------------------------------------------------------------------------------- 1 | package bpl 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "reflect" 7 | ) 8 | 9 | // ----------------------------------------------------------------------------- 10 | 11 | func directRepeat(R Ruler, in *bufio.Reader, ctx *Context) (v interface{}, err error) { 12 | 13 | _, err = R.Match(in, ctx) 14 | if err != nil { 15 | return 16 | } 17 | for { 18 | _, err = in.Peek(1) 19 | if err != nil { 20 | if err == io.EOF { 21 | return nil, nil 22 | } 23 | return 24 | } 25 | _, err = R.Match(in, ctx) 26 | if err != nil { 27 | return 28 | } 29 | } 30 | } 31 | 32 | func repeat(R Ruler, in *bufio.Reader, ctx *Context) (v interface{}, err error) { 33 | 34 | if _, ok := R.(*seq); ok { 35 | return directRepeat(R, in, ctx) 36 | } 37 | 38 | _, err = R.Match(in, ctx.NewSub()) 39 | if err != nil { 40 | return 41 | } 42 | for { 43 | _, err = in.Peek(1) 44 | if err != nil { 45 | if err == io.EOF { 46 | return nil, nil 47 | } 48 | return 49 | } 50 | _, err = R.Match(in, ctx.NewSub()) 51 | if err != nil { 52 | return 53 | } 54 | } 55 | } 56 | 57 | // ----------------------------------------------------------------------------- 58 | 59 | type repeat0 struct { 60 | r Ruler 61 | } 62 | 63 | func (p *repeat0) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 64 | 65 | _, err = in.Peek(1) 66 | if err != nil { 67 | if err == io.EOF { 68 | return nil, nil 69 | } 70 | return 71 | } 72 | return repeat(p.r, in, ctx) 73 | } 74 | 75 | func (p *repeat0) RetType() reflect.Type { 76 | 77 | return TyInterface 78 | } 79 | 80 | func (p *repeat0) SizeOf() int { 81 | 82 | return -1 83 | } 84 | 85 | // Repeat0 returns a matching unit that matches R* 86 | // 87 | func Repeat0(R Ruler) Ruler { 88 | 89 | return &repeat0{r: R} 90 | } 91 | 92 | // ----------------------------------------------------------------------------- 93 | 94 | type repeat1 struct { 95 | r Ruler 96 | } 97 | 98 | func (p *repeat1) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 99 | 100 | _, err = in.Peek(1) 101 | if err != nil { 102 | return 103 | } 104 | return repeat(p.r, in, ctx) 105 | } 106 | 107 | func (p *repeat1) RetType() reflect.Type { 108 | 109 | return TyInterface 110 | } 111 | 112 | func (p *repeat1) SizeOf() int { 113 | 114 | return -1 115 | } 116 | 117 | // Repeat1 returns a matching unit that matches R+ 118 | // 119 | func Repeat1(R Ruler) Ruler { 120 | 121 | return &repeat1{r: R} 122 | } 123 | 124 | // ----------------------------------------------------------------------------- 125 | 126 | type repeat01 struct { 127 | r Ruler 128 | } 129 | 130 | func (p *repeat01) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 131 | 132 | _, err = in.Peek(1) 133 | if err != nil { 134 | if err == io.EOF { 135 | return nil, nil 136 | } 137 | return 138 | } 139 | return p.r.Match(in, ctx) 140 | } 141 | 142 | func (p *repeat01) RetType() reflect.Type { 143 | 144 | return TyInterface 145 | } 146 | 147 | func (p *repeat01) SizeOf() int { 148 | 149 | return -1 150 | } 151 | 152 | // Repeat01 returns a matching unit that matches R? 153 | // 154 | func Repeat01(R Ruler) Ruler { 155 | 156 | return &repeat01{r: R} 157 | } 158 | 159 | // ----------------------------------------------------------------------------- 160 | -------------------------------------------------------------------------------- /replay/replay.go: -------------------------------------------------------------------------------- 1 | package replay 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "net" 7 | 8 | "github.com/goplus/bpl/hex" 9 | ) 10 | 11 | // ----------------------------------------------------------------------------- 12 | 13 | // Request replays sequent requests. 14 | // 15 | func Request(host string, w io.Writer, body io.Reader) (err error) { 16 | 17 | c1, err := net.Dial("tcp", host) 18 | if err != nil { 19 | return 20 | } 21 | c := c1.(*net.TCPConn) 22 | 23 | go func() { 24 | if w == nil { 25 | w = ioutil.Discard 26 | } 27 | io.Copy(w, c) 28 | c.CloseRead() 29 | }() 30 | 31 | _, err = io.Copy(c, body) 32 | c.CloseWrite() 33 | return 34 | } 35 | 36 | // HexRequest replays sequent requests from a hexdump file. 37 | // 38 | func HexRequest(host string, w io.Writer, in io.Reader, filter string) (err error) { 39 | 40 | f, err := hex.Reader(in, filter) 41 | if err != nil { 42 | return 43 | } 44 | defer f.Close() 45 | 46 | return Request(host, w, f) 47 | } 48 | 49 | // ----------------------------------------------------------------------------- 50 | -------------------------------------------------------------------------------- /struct.go: -------------------------------------------------------------------------------- 1 | package bpl 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "reflect" 7 | "strings" 8 | 9 | "github.com/qiniu/x/log" 10 | ) 11 | 12 | // ----------------------------------------------------------------------------- 13 | 14 | type ret func(ctx *Context) (v interface{}, err error) 15 | 16 | func (p ret) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 17 | 18 | v, err = p(ctx) 19 | if err != nil { 20 | return 21 | } 22 | ctx.dom = v 23 | return 24 | } 25 | 26 | func (p ret) RetType() reflect.Type { 27 | 28 | return TyInterface 29 | } 30 | 31 | func (p ret) SizeOf() int { 32 | 33 | return -1 34 | } 35 | 36 | // Return returns a matching unit that returns fnRet(ctx). 37 | // 38 | func Return(fnRet func(ctx *Context) (v interface{}, err error)) Ruler { 39 | 40 | return ret(fnRet) 41 | } 42 | 43 | // ----------------------------------------------------------------------------- 44 | 45 | // A Member is typeinfo of a `Struct` member. 46 | // 47 | type Member struct { 48 | Name string 49 | Type Ruler 50 | } 51 | 52 | // Match is required by a matching unit. see Ruler interface. 53 | // 54 | func (p *Member) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 55 | 56 | v, err = p.Type.Match(in, ctx.NewSub()) 57 | if err != nil { 58 | return 59 | } 60 | if p.Name != "_" { 61 | ctx.SetVar(p.Name, v) 62 | } 63 | return 64 | } 65 | 66 | // RetType returns matching result type. 67 | // 68 | func (p *Member) RetType() reflect.Type { 69 | 70 | return p.Type.RetType() 71 | } 72 | 73 | // SizeOf is required by a matching unit. see Ruler interface. 74 | // 75 | func (p *Member) SizeOf() int { 76 | 77 | return p.Type.SizeOf() 78 | } 79 | 80 | // ----------------------------------------------------------------------------- 81 | 82 | type structType struct { 83 | rulers []Ruler 84 | size int 85 | } 86 | 87 | func (p *structType) Match(in *bufio.Reader, ctx *Context) (v interface{}, err error) { 88 | 89 | for _, r := range p.rulers { 90 | _, err = r.Match(in, ctx) 91 | if err != nil { 92 | return 93 | } 94 | } 95 | return ctx.Dom(), nil 96 | } 97 | 98 | func (p *structType) RetType() reflect.Type { 99 | 100 | return TyInterface 101 | } 102 | 103 | func (p *structType) SizeOf() int { 104 | 105 | if p.size == -2 { 106 | p.size = p.sizeof() 107 | } 108 | return p.size 109 | } 110 | 111 | func (p *structType) sizeof() int { 112 | 113 | size := 0 114 | for _, r := range p.rulers { 115 | if n := r.SizeOf(); n < 0 { 116 | size = -1 117 | break 118 | } else { 119 | size += n 120 | } 121 | } 122 | return size 123 | } 124 | 125 | // Struct returns a compound matching unit. 126 | // 127 | func Struct(members []Ruler) Ruler { 128 | 129 | n := len(members) 130 | if n == 0 { 131 | return Nil 132 | } 133 | 134 | return &structType{rulers: members, size: -2} 135 | } 136 | 137 | // ----------------------------------------------------------------------------- 138 | 139 | func structFrom(t reflect.Type) (r Ruler, err error) { 140 | 141 | n := t.NumField() 142 | rulers := make([]Ruler, n) 143 | for i := 0; i < n; i++ { 144 | sf := t.Field(i) 145 | r, err = TypeFrom(sf.Type) 146 | if err != nil { 147 | log.Warn("bpl.TypeFrom failed:", err) 148 | return 149 | } 150 | rulers[i] = &Member{Name: strings.ToLower(sf.Name), Type: r} 151 | } 152 | return Struct(rulers), nil 153 | } 154 | 155 | // TypeFrom creates a matching unit from a Go type. 156 | // 157 | func TypeFrom(t reflect.Type) (r Ruler, err error) { 158 | 159 | retry: 160 | kind := t.Kind() 161 | switch { 162 | case kind == reflect.Struct: 163 | return structFrom(t) 164 | case kind >= reflect.Int8 && kind <= reflect.Float64: 165 | return BaseType(kind), nil 166 | case kind == reflect.String: 167 | return CString, nil 168 | case kind == reflect.Ptr: 169 | t = t.Elem() 170 | goto retry 171 | } 172 | return nil, fmt.Errorf("bpl.TypeFrom: unsupported type - %v", t) 173 | } 174 | 175 | // ----------------------------------------------------------------------------- 176 | -------------------------------------------------------------------------------- /struct_test.go: -------------------------------------------------------------------------------- 1 | package bpl_test 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "encoding/json" 7 | "reflect" 8 | "testing" 9 | 10 | "github.com/goplus/bpl" 11 | bin "github.com/goplus/bpl/binary" 12 | "github.com/qiniu/x/bufiox" 13 | ) 14 | 15 | type fixedType struct { 16 | A int8 17 | B uint16 18 | C uint32 19 | D float32 20 | } 21 | 22 | func TestFixedStruct(t *testing.T) { 23 | v := fixedType{ 24 | A: 1, 25 | B: 2, 26 | C: 3, 27 | D: 3.14, 28 | } 29 | 30 | b := new(bytes.Buffer) 31 | err := binary.Write(b, binary.LittleEndian, &v) 32 | if err != nil { 33 | t.Fatal("binary.Write failed:", err) 34 | } 35 | if b.Len() != 11 { 36 | t.Fatal("len != 11") 37 | } 38 | 39 | members := []bpl.Ruler{ 40 | &bpl.Member{Name: "a", Type: bpl.Int8}, 41 | &bpl.Member{Name: "b", Type: bpl.Uint16}, 42 | &bpl.Member{Name: "c", Type: bpl.Uint32}, 43 | &bpl.Member{Name: "d", Type: bpl.Float32}, 44 | } 45 | struc := bpl.Struct(members) 46 | if struc.SizeOf() != 11 { 47 | t.Fatal("struct.size != 11 - ", struc.SizeOf()) 48 | } 49 | 50 | in := bufiox.NewReaderBuffer(b.Bytes()) 51 | if in.Buffered() != 11 { 52 | t.Fatal("len != 11") 53 | } 54 | 55 | ctx := bpl.NewContext() 56 | ret, err := struc.Match(in, ctx) 57 | if err != nil { 58 | t.Fatal("struc.Match failed:", err) 59 | } 60 | text, err := json.Marshal(ret) 61 | if err != nil { 62 | t.Fatal("json.Marshal failed:", err) 63 | } 64 | if string(text) != `{"a":1,"b":2,"c":3,"d":3.14}` { 65 | t.Fatal("json.Marshal result:", string(text)) 66 | } 67 | } 68 | 69 | type subType struct { 70 | Foo string 71 | } 72 | 73 | type fooType struct { 74 | A int8 75 | B uint16 76 | C uint32 77 | D float32 78 | E string 79 | F subType 80 | G float64 81 | } 82 | 83 | var ( 84 | rSubType = bpl.Seq(bpl.CString) 85 | rFooType = bpl.Seq(bpl.Int8, bpl.Uint16, bpl.Uint32, bpl.Float32, bpl.CString, rSubType, bpl.Float64) 86 | rFooType2 = bpl.And(bpl.Seq(bpl.Int8, bpl.Uint16), bpl.Uint32, bpl.Float32, bpl.CString, bpl.Seq(rSubType), bpl.Float64) 87 | ) 88 | 89 | func TestStruct(t *testing.T) { 90 | 91 | foo := &fooType{ 92 | A: 1, B: 2, C: 3, D: 3.14, E: "Hello", F: subType{Foo: "foo"}, G: 7.52, 93 | // 1 + 2 + 4 + 4 + 6 + 4 + 8 = 29 94 | } 95 | b, err := bin.Marshal(&foo) 96 | if err != nil { 97 | t.Fatal("binary.Marshal failed:", err) 98 | } 99 | if len(b) != 29 { 100 | t.Fatal("len(b) != 29, len:", len(b), "data:", string(b)) 101 | } 102 | 103 | fooTyp := reflect.TypeOf(foo) 104 | r, err := bpl.TypeFrom(fooTyp) 105 | if err != nil { 106 | t.Fatal("bpl.TypeFrom failed:", err) 107 | } 108 | in := bufiox.NewReaderBuffer(b) 109 | ctx := bpl.NewContext() 110 | v, err := r.Match(in, ctx) 111 | if err != nil { 112 | t.Fatal("Match failed:", err, "len:", len(b)) 113 | } 114 | ret, err := json.Marshal(v) 115 | if err != nil { 116 | t.Fatal("json.Marshal failed:", err) 117 | } 118 | if string(ret) != `{"a":1,"b":2,"c":3,"d":3.14,"e":"Hello","f":{"foo":"foo"},"g":7.52}` { 119 | t.Fatal("ret:", string(ret)) 120 | } 121 | } 122 | 123 | func TestSeq(t *testing.T) { 124 | 125 | foo := &fooType{ 126 | A: 1, B: 2, C: 3, D: 3.14, E: "Hello", F: subType{Foo: "foo"}, G: 7.52, 127 | // 1 + 2 + 4 + 4 + 6 + 4 + 8 = 29 128 | } 129 | b, err := bin.Marshal(&foo) 130 | if err != nil { 131 | t.Fatal("binary.Marshal failed:", err) 132 | } 133 | in := bufiox.NewReaderBuffer(b) 134 | v, err := rFooType.Match(in, bpl.NewContext()) 135 | if err != nil { 136 | t.Fatal("Match failed:", err, "len:", len(b)) 137 | } 138 | ret, err := json.Marshal(v) 139 | if err != nil { 140 | t.Fatal("json.Marshal failed:", err) 141 | } 142 | if string(ret) != `[1,2,3,3.14,"Hello",["foo"],7.52]` { 143 | t.Fatal("ret:", string(ret)) 144 | } 145 | } 146 | 147 | func TestSeq2(t *testing.T) { 148 | 149 | foo := &fooType{ 150 | A: 1, B: 2, C: 3, D: 3.14, E: "Hello", F: subType{Foo: "foo"}, G: 7.52, 151 | // 1 + 2 + 4 + 4 + 6 + 4 + 8 = 29 152 | } 153 | b, err := bin.Marshal(&foo) 154 | if err != nil { 155 | t.Fatal("binary.Marshal failed:", err) 156 | } 157 | in := bufiox.NewReaderBuffer(b) 158 | ctx := bpl.NewContext() 159 | _, err = rFooType2.Match(in, ctx) 160 | if err != nil { 161 | t.Fatal("Match failed:", err, "len:", len(b)) 162 | } 163 | v := ctx.Dom() 164 | ret, err := json.Marshal(v) 165 | if err != nil { 166 | t.Fatal("json.Marshal failed:", err) 167 | } 168 | if string(ret) != `[1,2,["foo"]]` { 169 | t.Fatal("ret:", string(ret)) 170 | } 171 | } 172 | --------------------------------------------------------------------------------