├── .flake8 ├── .github └── workflows │ └── pythonpackage.yml ├── .gitignore ├── .travis.yml ├── LICENSE ├── README-CN.md ├── README.md ├── canoser.sublime-project ├── canoser ├── __init__.py ├── array_t.py ├── base.py ├── bool_t.py ├── bytes_t.py ├── cursor.py ├── delegate_t.py ├── int_type.py ├── map_t.py ├── rust_enum.py ├── rust_optional.py ├── str_t.py ├── struct.py ├── tuple_t.py ├── types.py ├── util.py └── version.py ├── dev-requirements.in ├── dev-requirements.txt ├── docs └── design.md ├── requirements.txt ├── script ├── format_code.sh ├── gen_requirements.sh └── update_requirements.sh ├── setup.py ├── test ├── bench_int.py ├── bench_serialize.py ├── test_bytes.py ├── test_circular.py ├── test_cursor.py ├── test_delegate.py ├── test_enum.py ├── test_libra_example.py ├── test_list.py ├── test_optional.py ├── test_proptest.py ├── test_struct.py ├── test_types.py └── test_util.py └── venv.sh /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = 3 | # E501: line too long 4 | E501 5 | -------------------------------------------------------------------------------- /.github/workflows/pythonpackage.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Python package 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | python-version: [3.6, 3.7, 3.8] 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v1 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | - name: Install dependencies 27 | run: | 28 | python -m pip install --upgrade pip 29 | pip install -r requirements.txt 30 | pip install -r dev-requirements.txt 31 | - name: Lint with flake8 32 | run: | 33 | # stop the build if there are Python syntax errors or undefined names 34 | flake8 canoser --count --select=E9,F --show-source --statistics 35 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 36 | flake8 canoser --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 37 | - name: Test with pytest 38 | run: | 39 | python setup.py install 40 | pip install pytest 41 | pytest test 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.vscode/ 2 | /venv/ 3 | /build/ 4 | /dist/ 5 | /canoser.egg-info/ 6 | __pycache__/ 7 | *.pyc 8 | *.swp 9 | *.sublime-workspace 10 | pyenv -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | # Runs jobs on container based infrastructure 4 | sudo: false 5 | 6 | # Saves pip downloads/wheels between builds 7 | cache: 8 | directories: 9 | - $HOME/.cache/pip 10 | 11 | python: 12 | - "3.6" 13 | - "3.7" 14 | - "3.8" 15 | 16 | install: 17 | # Optimisation: build requirements as wheels, which get cached by Travis 18 | - pip install "pip>=7.0" wheel 19 | - pip install -r requirements.txt 20 | - pip install -r dev-requirements.txt 21 | - pip install codecov 22 | 23 | script: 24 | - python setup.py install 25 | - python -m pytest --cov=canoser 26 | 27 | after_success: 28 | - codecov -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 yuan_xin_yu@hotmail.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README-CN.md: -------------------------------------------------------------------------------- 1 | # Canoser 2 | 3 | 4 | Canoser是facebook推出的Libra网络中使用的规范序列化LCS(Libra Canonical Serialization)协议的第三方python实现框架。 5 | 6 | 规范序列化可确保内存里的数据结构在序列化的时候保证字节一致性。它适用于双方想要有效比较他们独立维护的数据结构。在共识协议中，独立验证者需要就他们独立计算的状态达成一致。共识双方比较的是序列化数据的加密散列。要实现这一点，在计算时，相同数据结构的序列化必须相同。而独立验证器可能由不同的语言编写，有不同的实现代码，但是都遵循同一个规范。 7 | 8 | 9 | ## 安装 10 | 11 | 需要系统安装了Python 3.6及以上版本。 12 | 13 | ```sh 14 | $ python3 -m pip install canoser 15 | ``` 16 | 17 | ## 使用 18 | 19 | 首先用Canoser定义一个数据结构，也就是写一个类继承自"canoser.Struct"，然后通过"_fields"来定义该结构所拥有的字段。该结构自然就拥有了序列化和反序列化的能力。例如下面的AccountResource定义了一个Libra代码中的同名数据结构： 20 | ```python 21 | #python代码，利用canoser定义数据结构 22 | from canoser import Struct, Uint8, Uint64 23 | class AccountResource(Struct): 24 | _fields = [ 25 | ('authentication_key', [Uint8]), 26 | ('balance', Uint64), 27 | ('delegated_withdrawal_capability', bool), 28 | ('received_events', EventHandle), 29 | ('sent_events', EventHandle), 30 | ('sequence_number', Uint64) 31 | ] 32 | ``` 33 | 34 | 下面是Libra中定义该数据结构以及序列化的代码： 35 | ```rust 36 | // Libra中的rust语言代码 37 | // 定义数据结构 38 | pub struct AccountResource { 39 | balance: u64, 40 | sequence_number: u64, 41 | authentication_key: ByteArray, 42 | sent_events: EventHandle, 43 | received_events: EventHandle, 44 | delegated_withdrawal_capability: bool, 45 | } 46 | // 实现序列化 47 | impl CanonicalSerialize for AccountResource { 48 | fn serialize(&self, serializer: &mut impl CanonicalSerializer) -> Result<()> { 49 | serializer 50 | .encode_struct(&self.authentication_key)? 51 | .encode_u64(self.balance)? 52 | .encode_bool(self.delegated_withdrawal_capability)? 53 | .encode_struct(self.received_events)? 54 | .encode_struct(self.sent_events)? 55 | .encode_u64(self.sequence_number)?; 56 | Ok(()) 57 | } 58 | } 59 | ``` 60 | ~~在Libra使用的rust语言中，需要手动写代码实现数据结构的序列化/反序列化，而且数据结构中的字段顺序和序列化时的顺序不一定一致。~~ 61 | 62 | 在Canoser中，定义好数据结构后，不需要写序列化和反序列化的代码。注意，Canoser中的数据结构顺序要按照Libra中序列化的顺序来定义。 63 | 64 | ### 支持的数据类型 65 | 66 | 字段支持的类型有： 67 | 68 | | 字段类型 | 可选子类型 | 说明 | 69 | | ------ | ------ | ------ | 70 | | canoser.Uint8 | | 无符号8位整数 | 71 | | canoser.Uint16 | | 无符号16位整数 | 72 | | canoser.Uint32 | | 无符号32位整数 | 73 | | canoser.Uint64 | | 无符号64位整数 | 74 | | canoser.Uint128 | | 无符号128位整数 | 75 | | canoser.Int8 | | 有符号8位整数 | 76 | | canoser.Int16 | | 有符号16位整数 | 77 | | canoser.Int32 | | 有符号32位整数 | 78 | | canoser.Int64 | | 有符号64位整数 | 79 | | canoser.Int128 | | 有符号128位整数 | 80 | | bool | | 布尔类型 | 81 | | str | | 字符串 | 82 | | bytes | | Binary String | 83 | | [] | 支持 | 数组类型 | 84 | | {} | 支持 | Map类型 | 85 | | () | 支持 | Tuple元组类型 | 86 | | A canoser.Struct | | 嵌套的另外一个结构（不能循环引用） | 87 | 88 | ### 关于数组类型 89 | 数组里的数据，如果没有定义类型，那么缺省是Uint8。下面的两个定义等价： 90 | ```python 91 | class Arr1(Struct): 92 | _fields = [(addr, [])] 93 | 94 | 95 | class Arr2(Struct): 96 | _fields = [(addr, [Uint8])] 97 | 98 | ``` 99 | 数组还可以定义长度，表示定长数据。比如Libra中的地址是256位，也就是32个字节，所以可以如下定义： 100 | ```python 101 | class Address(Struct): 102 | _fields = [(addr, [Uint8, 32])] 103 | ``` 104 | 定长数据在序列化的时候，可以添加一个标志位, 表明不写入长度信息。如下所示: 105 | 106 | ```python 107 | class Address(Struct): 108 | _fields = [(addr, [Uint8, 32, False])] 109 | 110 | 111 | ### 关于Map类型 112 | Map里的数据，如果没有定义类型，那么在libra中缺省是字节数组，也就是[Uint8]。 113 | 但是在python语言中，dict的key不支持list类型，于是在canoser中Map的key类型默认为bytes，value的类型是[Uint8]。下面的两个定义等价： 114 | ```python 115 | class Map1(Struct): 116 | _fields = [(addr, {})] 117 | 118 | 119 | class Map2(Struct): 120 | _fields = [(addr, {bytes : [Uint8] })] 121 | 122 | ``` 123 | 124 | ### 结构嵌套 125 | 下面是一个复杂的例子，包含三个数据结构： 126 | ```python 127 | class Addr(Struct): 128 | _fields = [('addr', [Uint8, 32])] 129 | 130 | 131 | class Bar(Struct): 132 | _fields = [ 133 | ('a', Uint64), 134 | ('b', [Uint8]), 135 | ('c', Addr), 136 | ('d', Uint32), 137 | ] 138 | 139 | class Foo(Struct): 140 | _fields = [ 141 | ('a', Uint64), 142 | ('b', [Uint8]), 143 | ('c', Bar), 144 | ('d', bool), 145 | ('e', {}), 146 | ] 147 | ``` 148 | 这个例子参考自libra中canonical serialization的测试代码。 149 | 150 | ### 序列化和反序列化 151 | 在定义好canoser.Struct后，不需要自己实现序列化和反序列化代码，直接调用基类的默认实现即可。以AccountResource结构为例： 152 | ```python 153 | #序列化 154 | obj = AccountResource.new(authentication_key=...,...) 155 | bbytes = obj.serialize 156 | #反序列化 157 | obj = AccountResource.deserialize(bbytes) 158 | ``` 159 | ### 从Struct对象中读取字段的值 160 | 对于所有通过_field定义的字段，可以通过field_name获取该字段的值。比如： 161 | 162 | ```python 163 | obj.authentication_key 164 | ``` 165 | 166 | 167 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Canoser [![Canoser](https://img.shields.io/pypi/v/canoser.svg)](https://pypi.org/project/canoser/) [![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) ![Python package](https://github.com/yuan-xy/canoser-python/workflows/Python%20package/badge.svg) [![Build Status](https://travis-ci.org/yuan-xy/canoser-python.svg?branch=master)](https://travis-ci.org/yuan-xy/canoser-python) [![codecov](https://codecov.io/gh/yuan-xy/canoser-python/branch/master/graph/badge.svg)](https://codecov.io/gh/yuan-xy/canoser-python) 2 | 3 | 4 | [中文文档 Chinese document](/README-CN.md) 5 | 6 | A python implementation of the canonical serialization for the Libra network. 7 | 8 | Canonical serialization guarantees byte consistency when serializing an in-memory 9 | data structure. It is useful for situations where two parties want to efficiently compare 10 | data structures they independently maintain. It happens in consensus where 11 | independent validators need to agree on the state they independently compute. A cryptographic 12 | hash of the serialized data structure is what ultimately gets compared. In order for 13 | this to work, the serialization of the same data structures must be identical when computed 14 | by independent validators potentially running different implementations 15 | of the same spec in different languages. 16 | 17 | ## Installation 18 | 19 | Require python 3.6 or above installed. 20 | 21 | ```sh 22 | $ python3 -m pip install canoser 23 | ``` 24 | 25 | 26 | ## Usage 27 | 28 | First define a data structure with Canoser, that is, write a class that inherits from "canoser.Struct", and then define the fields owned by the structure through the "\_fields" array. This structure naturally has the ability to canonical serialize and deserialize types. For example, the following AccountResource defines a data structure of the same name in the Libra code： 29 | ```python 30 | #python code，define canoser data structure 31 | from canoser import Struct, Uint8, Uint64 32 | class AccountResource(Struct): 33 | _fields = [ 34 | ('authentication_key', [Uint8]), 35 | ('balance', Uint64), 36 | ('delegated_withdrawal_capability', bool), 37 | ('received_events', EventHandle), 38 | ('sent_events', EventHandle), 39 | ('sequence_number', Uint64) 40 | ] 41 | ``` 42 | 43 | Here is the code that defines this data structure and serialization in Libra code: 44 | ```rust 45 | // rust code in Libra 46 | // define the data structure 47 | pub struct AccountResource { 48 | balance: u64, 49 | sequence_number: u64, 50 | authentication_key: ByteArray, 51 | sent_events: EventHandle, 52 | received_events: EventHandle, 53 | delegated_withdrawal_capability: bool, 54 | } 55 | // serialization 56 | impl CanonicalSerialize for AccountResource { 57 | fn serialize(&self, serializer: &mut impl CanonicalSerializer) -> Result<()> { 58 | serializer 59 | .encode_struct(&self.authentication_key)? 60 | .encode_u64(self.balance)? 61 | .encode_bool(self.delegated_withdrawal_capability)? 62 | .encode_struct(self.received_events)? 63 | .encode_struct(self.sent_events)? 64 | .encode_u64(self.sequence_number)?; 65 | Ok(()) 66 | } 67 | } 68 | ``` 69 | ~~In the rust language used by Libra, it is necessary to manually write code to serialize/deserialize the data structure, and the order of the fields in the data structure and the order of serialization are not necessarily the same.~~ 70 | 71 | In Canoser, after defining the data structure, you don't need to write code to implement serialization and deserialization. Note that the order of the data structures in Canoser is defined in the order in which they are serialized in Libra. 72 | 73 | ### Supported field types 74 | 75 | | field type | optionl sub type | description | 76 | | ------ | ------ | ------ | 77 | | canoser.Uint8 | | Unsigned 8-bit integer | 78 | | canoser.Uint16 | | Unsigned 16-bit integer| 79 | | canoser.Uint32 | | Unsigned 32-bit integer | 80 | | canoser.Uint64 | | Unsigned 64-bit integer | 81 | | canoser.Uint128 | | Unsigned 128-bit integer | 82 | | canoser.Int8 | | Signed 8-bit integer | 83 | | canoser.Int16 | | Signed 16-bit integer| 84 | | canoser.Int32 | | Signed 32-bit integer | 85 | | canoser.Int64 | | Signed 64-bit integer | 86 | | canoser.Int128 | | Signed 128-bit integer | 87 | | bool | | Boolean | 88 | | str | | String | 89 | | bytes | | Binary String | 90 | | [] | supported | Array Type | 91 | | {} | supported | Map Type | 92 | | () | supported | Tuple Type | 93 | | canoser.Struct | | Another structure nested (cannot be recycled) | 94 | | RustEnum | | Enum type | 95 | | RustOptional | | Optional type | 96 | 97 | ### About Array Type 98 | The default data type (if not defined) in the array is Uint8. The following two definitions are equivalent: 99 | ```python 100 | class Arr1(Struct): 101 | _fields = [(addr, [])] 102 | 103 | 104 | class Arr2(Struct): 105 | _fields = [(addr, [Uint8])] 106 | 107 | ``` 108 | Arrays can also define lengths to represent fixed length data. For example, the address in Libra is 256 bits, which is 32 bytes, so it can be defined as follows: 109 | ```python 110 | class Address(Struct): 111 | _fields = [(addr, [Uint8, 32])] 112 | ``` 113 | When the fixed length data is serialized, you can skip the length information written to the output. For example, following code will generate 32 bytes without writing the length of the addr during serialization. 114 | 115 | ```python 116 | class Address(Struct): 117 | _fields = [(addr, [Uint8, 32, False])] 118 | ``` 119 | 120 | 121 | ### About map type 122 | The default data type (if not defined) in the map is an array of Uint8 in Libra, both of key and value. 123 | But the python language dosn't support the array data type to be the key of a dict, so we change the key type from array of Uint8 to bytes in python, the type of value is unchanged. 124 | The following two definitions are equivalent: 125 | ```python 126 | class Map1(Struct): 127 | _fields = [(addr, {})] 128 | 129 | 130 | class Map2(Struct): 131 | _fields = [(addr, {bytes : [Uint8] })] 132 | 133 | ``` 134 | 135 | ### About enum type 136 | In python and C, enum is just enumerated constants. But in Libra(Rust), a enum has a type constant and a optional Value. A rust enumeration is typically represented as: 137 | 138 | ``` 139 | enum SomeDataType { 140 | type0(u32), 141 | type1(u64), 142 | type2 143 | } 144 | ``` 145 | 146 | To define a enum with Canoser, first write a class that inherits from "canoser.RustEnum", and then define the types owned by the enum through the "\_enums" array. 147 | For example, the following TransactionArgument defines a data structure of the same name in the Libra code. The argument of a transaction can be one of the four types: Uint64 or Address or String or IntArray.： 148 | 149 | ```python 150 | class TransactionArgument(RustEnum): 151 | _enums = [ 152 | ('U64', Uint64), 153 | ('Address', [Uint8, ADDRESS_LENGTH]), 154 | ('String', str), 155 | ('ByteArray', [Uint8]) 156 | ] 157 | ``` 158 | You can instantiate an enum obj and call its method and properties like this: 159 | 160 | ```python 161 | arg2 = TransactionArgument('String', 'abc') 162 | assert arg2.index == 2 163 | assert arg2.value == 'abc' 164 | assert arg2.String == True 165 | assert arg2.U64 == False 166 | ``` 167 | 168 | Every RustEnum object has an `index` property and a `value` property. After instantiated, the `index` can't be modified. You can only modify the `value` of an enum with correct data type. 169 | 170 | For example, the following code is valid: 171 | 172 | ```python 173 | arg2 = TransactionArgument('String', 'abc') 174 | arg2.value == 'Bcd' 175 | ``` 176 | 177 | For example, the following code is invalid: 178 | ```python 179 | arg2.index = 0 #raise an exception 180 | arg2.value = [3] #raise an exception 181 | ``` 182 | 183 | The RustEnum can have a enum without value type, which represented by `None`. 184 | 185 | ```python 186 | class Enum1(RustEnum): 187 | _enums = [('opt1', [Uint8]), ('opt2', None)] 188 | 189 | e2 = Enum1('opt2', None) 190 | #or 191 | e2 = Enum1('opt2') 192 | ``` 193 | 194 | You can also instantiate a RustEnum object by index and value. 195 | 196 | ```python 197 | e1 = Enum1.new(0, [5]) 198 | # which is equivalent to 199 | e1 = Enum1('opt1', [5]) 200 | ``` 201 | 202 | ### About optional type 203 | An optional type in libra is a nullable data either exists in its full representation or does not. For example, 204 | 205 | ``` 206 | optional_data: Option(uint8); // Rust/Libra 207 | uint8 *optional_data; // C 208 | ``` 209 | It has similar semantics meaning with the following enum type: 210 | ``` 211 | enum Option { 212 | Some(uint8), 213 | None, 214 | } 215 | ``` 216 | 217 | To define a optional with Canoser, first write a class that inherits from "canoser.RustOptional", and then define the types owned by RustOptional through the "\_type" field. For example, 218 | 219 | ```python 220 | class OptionUInt(RustOptional): 221 | _type = Uint8 222 | 223 | null = OptionUInt(None) 224 | obj = OptionUInt(8) 225 | assert obj.value == 8 226 | ``` 227 | 228 | Here's a complete example: 229 | 230 | ```python 231 | class OptionStr(RustOptional): 232 | _type = str 233 | 234 | class OptionTest(Struct): 235 | _fields = [ 236 | ('message', OptionStr) 237 | ] 238 | 239 | def __init__(self, msg=None): 240 | if msg is not None: 241 | self.message = OptionStr(msg) 242 | else: 243 | self.message = OptionStr(None) 244 | 245 | test = OptionTest('test_str') 246 | assert test.message.value == 'test_str' 247 | ``` 248 | 249 | 250 | The RustOptional type in canoser is similar to `typing.Optional` in python. Note that this is not the same concept as an optional argument, which is one that has a default. 251 | 252 | 253 | ### Nested data structure 254 | The following is a complex example with three data structures: 255 | ```python 256 | class Addr(Struct): 257 | _fields = [('addr', [Uint8, 32])] 258 | 259 | 260 | class Bar(Struct): 261 | _fields = [ 262 | ('a', Uint64), 263 | ('b', [Uint8]), 264 | ('c', Addr), 265 | ('d', Uint32), 266 | ] 267 | 268 | class Foo(Struct): 269 | _fields = [ 270 | ('a', Uint64), 271 | ('b', [Uint8]), 272 | ('c', Bar), 273 | ('d', bool), 274 | ('e', {}), 275 | ] 276 | ``` 277 | This example refers to the test code from canonical serialization in libra. 278 | 279 | 280 | ### Serialization and deserialization 281 | After defining canoser.Struct, you don't need to implement serialization and deserialization code yourself, you can directly call the default implementation of the base class. Take the AccountResource structure as an example: 282 | 283 | ```python 284 | # serialize an object 285 | obj = AccountResource(authentication_key=...,...) 286 | bbytes = obj.serialize() 287 | 288 | # deserialize an object from bytes 289 | obj = AccountResource.deserialize(bbytes) 290 | ``` 291 | 292 | ### Json pretty print 293 | 294 | For any canoser `Struct`, you can call the `to_json` method to get a formatted json string: 295 | 296 | ```python 297 | # serialize an object 298 | print(obj.to_json()) 299 | 300 | ``` 301 | 302 | ### Get field value from object 303 | 304 | For all fields defined by the "\_fields", the value of this field of an object can be obtained via field_name. such as: 305 | ```python 306 | obj.authentication_key 307 | ``` 308 | 309 | 310 | ## Rust Type Alias 311 | For simple type alias in rust, such as: 312 | ```rust 313 | // in rust 314 | pub type Round = u64; 315 | ``` 316 | 317 | We can define the alias like this: 318 | 319 | ```python 320 | # in python 321 | Round = Uint64 322 | ``` 323 | 324 | 325 | ## Rust Tuple Struct 326 | 327 | Struct like Address and ByteArray has no fields: 328 | 329 | ```rust 330 | pub struct Address([u8; ADDRESS_LENGTH]); 331 | pub struct ByteArray(Vec); 332 | ``` 333 | 334 | These struct called `tuple struct` in `Rust` programming language. Tuple struct is like `typedef` other than struct in `C` like programming languages. 335 | 336 | You can just define them as a direct type, no struct. Just code like this: 337 | ```python 338 | class TransactionArgument(RustEnum): 339 | _enums = [ 340 | ... 341 | ('Address', [Uint8, ADDRESS_LENGTH]), 342 | ... 343 | ] 344 | ``` 345 | 346 | Or you can define an `Address` class which inherit from `canoser.DelegateT` and has a `delegate_type` field with type `[Uint8, ADDRESS_LENGTH]`: 347 | 348 | ```python 349 | class Address(DelegateT): 350 | delegate_type = [Uint8, ADDRESS_LENGTH] 351 | 352 | 353 | class TransactionArgument(RustEnum): 354 | _enums = [ 355 | ... 356 | ('Address', Address), 357 | ... 358 | ] 359 | ``` 360 | 361 | Do not instantiate a `canoser.DelegateT` type in assaignment, for example: 362 | 363 | ```python 364 | transactionArgument.address = [...] #ok 365 | transactionArgument.address = Address([...]) #error 366 | ``` 367 | 368 | 369 | ## Notice 370 | 371 | ### Must define canoser struct by serialized fields and sequence, not the definition in the rust struct. 372 | 373 | For example, the SignedTransaction in Libra is defined as following code: 374 | 375 | ```rust 376 | pub struct SignedTransaction { 377 | raw_txn: RawTransaction, 378 | public_key: Ed25519PublicKey, 379 | signature: Ed25519Signature, 380 | transaction_length: usize, 381 | } 382 | ``` 383 | But field `transaction_length` doesn't write to the output. 384 | 385 | ```rust 386 | impl CanonicalSerialize for SignedTransaction { 387 | fn serialize(&self, serializer: &mut impl CanonicalSerializer) -> Result<()> { 388 | serializer 389 | .encode_struct(&self.raw_txn)? 390 | .encode_bytes(&self.public_key.to_bytes())? 391 | .encode_bytes(&self.signature.to_bytes())?; 392 | Ok(()) 393 | } 394 | } 395 | ``` 396 | 397 | So we define SignedTransaction in canoser as following code: 398 | ```python 399 | class SignedTransaction(canoser.Struct): 400 | _fields = [ 401 | ('raw_txn', RawTransaction), 402 | ('public_key', [Uint8, ED25519_PUBLIC_KEY_LENGTH]), 403 | ('signature', [Uint8, ED25519_SIGNATURE_LENGTH]) 404 | ] 405 | ``` 406 | 407 | Here is another example. The definition sequence and serialize sequence is opposite in `WriteOp` 408 | 409 | ```rust 410 | pub enum WriteOp { 411 | Value(Vec), 412 | Deletion, 413 | } 414 | 415 | enum WriteOpType { 416 | Deletion = 0, 417 | Value = 1, 418 | } 419 | 420 | impl CanonicalSerialize for WriteOp { 421 | fn serialize(&self, serializer: &mut impl CanonicalSerializer) -> Result<()> { 422 | match self { 423 | WriteOp::Deletion => serializer.encode_u32(WriteOpType::Deletion as u32)?, 424 | WriteOp::Value(value) => { 425 | serializer.encode_u32(WriteOpType::Value as u32)?; 426 | serializer.encode_vec(value)? 427 | } 428 | }; 429 | Ok(()) 430 | } 431 | } 432 | ``` 433 | 434 | So, we define `WriteOp` as follow: 435 | ```python 436 | class WriteOp(RustEnum): 437 | _enums = [ 438 | ('Deletion', None), 439 | ('Value', [Uint8]) 440 | ] 441 | 442 | ``` 443 | 444 | ## Related Projects 445 | 446 | [MoveOnLibra OpenAPI: make writing libra wallet & move program easier](https://www.MoveOnLibra.com) 447 | 448 | [A Ruby implementation of the LCS(Libra Canonical Serialization)](https://github.com/yuan-xy/canoser-ruby) 449 | 450 | [A Python implementation of client APIs and command-line tools for the Libra network](https://github.com/yuan-xy/libra-client) 451 | 452 | 453 | 454 | ## License 455 | 456 | The package is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). 457 | 458 | -------------------------------------------------------------------------------- /canoser.sublime-project: -------------------------------------------------------------------------------- 1 | { 2 | "folders": 3 | [ 4 | { 5 | "file_exclude_patterns": 6 | [ 7 | "*.sublime-workspace", 8 | "*.egg-info", 9 | ".DS_Store" 10 | ], 11 | "folder_exclude_patterns": 12 | [ 13 | "build", 14 | "tmp", 15 | "dist", 16 | "pkg" 17 | ], 18 | "path": "." 19 | } 20 | ], 21 | "settings": 22 | { 23 | "tab_size": 4, 24 | "translate_tabs_to_spaces": true, 25 | "trim_trailing_white_space_on_save": true, 26 | "draw_white_space": "all" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /canoser/__init__.py: -------------------------------------------------------------------------------- 1 | from canoser.cursor import Cursor # noqa: F401 2 | from canoser.struct import Struct # noqa: F401 3 | from canoser.rust_enum import RustEnum # noqa: F401 4 | from canoser.rust_optional import RustOptional # noqa: F401 5 | from canoser.delegate_t import DelegateT # noqa: F401 6 | from canoser.tuple_t import TupleT # noqa: F401 7 | from canoser.map_t import MapT # noqa: F401 8 | from canoser.str_t import StrT # noqa: F401 9 | from canoser.bytes_t import BytesT # noqa: F401 10 | from canoser.bool_t import BoolT # noqa: F401 11 | from canoser.array_t import ArrayT # noqa: F401 12 | from canoser.int_type import Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64 # noqa: F401 13 | from canoser.int_type import Uint128, Int128 # noqa: F401 14 | from canoser.util import bytes_to_int_list, hex_to_int_list # noqa: F401 15 | -------------------------------------------------------------------------------- /canoser/array_t.py: -------------------------------------------------------------------------------- 1 | from canoser.base import Base 2 | from canoser.int_type import Uint32, Uint8 3 | import struct 4 | 5 | 6 | class ArrayT(Base): 7 | 8 | def __init__(self, atype, fixed_len=None, encode_len=True): 9 | self.atype = atype 10 | if fixed_len is not None and fixed_len <= 0: 11 | raise TypeError("arr len must > 0".format(fixed_len)) 12 | if fixed_len is None and not encode_len: 13 | raise TypeError("variable length sequences must encode len.") 14 | self.fixed_len = fixed_len 15 | self.encode_len = encode_len 16 | 17 | def encode(self, arr): 18 | if self.fixed_len is not None and len(arr) != self.fixed_len: 19 | raise TypeError(f"{len(arr)} is not equal to predefined value: {self.fixed_len}") 20 | output = b"" 21 | if self.encode_len: 22 | output += Uint32.serialize_uint32_as_uleb128(len(arr)) 23 | for item in arr: 24 | output += self.atype.encode(item) 25 | return output 26 | 27 | def decode(self, cursor): 28 | arr = [] 29 | if self.encode_len: 30 | size = Uint32.parse_uint32_from_uleb128(cursor) 31 | if self.fixed_len is not None and size != self.fixed_len: 32 | raise TypeError(f"{size} is not equal to predefined value: {self.fixed_len}") 33 | else: 34 | size = self.fixed_len 35 | for _ in range(size): 36 | arr.append(self.atype.decode(cursor)) 37 | return arr 38 | 39 | def check_value(self, arr): 40 | if self.fixed_len is not None and len(arr) != self.fixed_len: 41 | raise TypeError("arr len not match: {}-{}".format(len(arr), self.fixed_len)) 42 | if not isinstance(arr, list): 43 | raise TypeError(f"{arr} is not a list.") 44 | for item in arr: 45 | self.atype.check_value(item) 46 | 47 | def __eq__(self, other): 48 | if not isinstance(other, ArrayT): 49 | return False 50 | return self.atype == other.atype and self.fixed_len == other.fixed_len 51 | 52 | def to_json_serializable(cls, obj): 53 | if cls.atype == Uint8: 54 | return struct.pack("<{}B".format(len(obj)), *obj).hex() 55 | ret = [] 56 | for _, item in enumerate(obj): 57 | data = cls.atype.to_json_serializable(item) 58 | ret.append(data) 59 | return ret 60 | -------------------------------------------------------------------------------- /canoser/base.py: -------------------------------------------------------------------------------- 1 | from canoser.cursor import Cursor 2 | 3 | 4 | class Base: 5 | """ 6 | All types should implment following four methods: 7 | 8 | def encode(cls_or_obj, value) 9 | 10 | def decode(cls_or_obj, cursor) 11 | 12 | def check_value(cls_or_obj, value) 13 | 14 | def to_json_serializable(obj) 15 | def to_json_serializable(acls, value) 16 | """ 17 | 18 | def serialize(self): 19 | return self.__class__.encode(self) 20 | 21 | @classmethod 22 | def deserialize(cls, buffer, check=True): 23 | cursor = Cursor(buffer) 24 | ret = cls.decode(cursor) 25 | if not cursor.is_finished() and check: 26 | raise IOError("bytes not all consumed:{}, {}".format( 27 | len(buffer), cursor.offset)) 28 | return ret 29 | -------------------------------------------------------------------------------- /canoser/bool_t.py: -------------------------------------------------------------------------------- 1 | from canoser.base import Base 2 | 3 | 4 | class BoolT(Base): 5 | @classmethod 6 | def encode(self, value): 7 | if value: 8 | return b'\1' 9 | else: 10 | return b'\0' 11 | 12 | @classmethod 13 | def decode_bytes(self, value): 14 | if value == b'\0': 15 | return False 16 | elif value == b'\1': 17 | return True 18 | else: 19 | raise TypeError("bool should be 0 or 1.") 20 | 21 | @classmethod 22 | def decode(self, cursor): 23 | value = cursor.read_bytes(1) 24 | return self.decode_bytes(value) 25 | 26 | @classmethod 27 | def check_value(self, value): 28 | if not isinstance(value, bool): 29 | raise TypeError('value {} is not bool'.format(value)) 30 | 31 | @classmethod 32 | def to_json_serializable(cls, value): 33 | return value 34 | -------------------------------------------------------------------------------- /canoser/bytes_t.py: -------------------------------------------------------------------------------- 1 | from canoser.int_type import Uint32 2 | from canoser.base import Base 3 | 4 | 5 | class BytesT(Base): 6 | 7 | def __init__(self, fixed_len=None, encode_len=True): 8 | if fixed_len is not None and fixed_len <= 0: 9 | raise TypeError("byte len must > 0".format(fixed_len)) 10 | if fixed_len is None and not encode_len: 11 | raise TypeError("variable length sequences must encode len.") 12 | self.fixed_len = fixed_len 13 | self.encode_len = encode_len 14 | 15 | def encode(self, value): 16 | output = b"" 17 | if self.encode_len: 18 | output += Uint32.serialize_uint32_as_uleb128(len(value)) 19 | output += value 20 | return output 21 | 22 | def decode(self, cursor): 23 | if self.encode_len: 24 | size = Uint32.parse_uint32_from_uleb128(cursor) 25 | if self.fixed_len is not None and size != self.fixed_len: 26 | raise TypeError(f"{size} is not equal to predefined value: {self.fixed_len}") 27 | else: 28 | size = self.fixed_len 29 | return cursor.read_bytes(size) 30 | 31 | def check_value(self, value): 32 | if not isinstance(value, bytes): 33 | raise TypeError('value {} is not bytes'.format(value)) 34 | if self.fixed_len is not None and len(value) != self.fixed_len: 35 | raise TypeError("len not match: {}-{}".format(len(value), self.fixed_len)) 36 | 37 | def __eq__(self, other): 38 | if not isinstance(other, BytesT): 39 | return False 40 | return self.fixed_len == other.fixed_len and self.encode_len == other.encode_len 41 | 42 | def to_json_serializable(cls, obj): 43 | return obj.hex() 44 | 45 | 46 | class ByteArrayT(Base): 47 | 48 | def encode(self, value): 49 | output = b"" 50 | output += Uint32.serialize_uint32_as_uleb128(len(value)) 51 | output += bytes(value) 52 | return output 53 | 54 | def decode(self, cursor): 55 | size = Uint32.parse_uint32_from_uleb128(cursor) 56 | return bytearray(cursor.read_bytes(size)) 57 | 58 | def check_value(self, value): 59 | if not isinstance(value, bytearray): 60 | raise TypeError('value {} is not bytearray'.format(value)) 61 | 62 | def __eq__(self, other): 63 | if not isinstance(other, ByteArrayT): 64 | return False 65 | return True 66 | 67 | def to_json_serializable(cls, obj): 68 | return obj.hex() 69 | -------------------------------------------------------------------------------- /canoser/cursor.py: -------------------------------------------------------------------------------- 1 | 2 | class Cursor: 3 | def __init__(self, buffer, offset=0): 4 | self.buffer = buffer 5 | if isinstance(buffer, list): 6 | self.buffer = bytes(buffer) 7 | if isinstance(buffer, bytearray): 8 | self.buffer = bytes(buffer) 9 | self.offset = offset 10 | self.buffer_len = len(self.buffer) 11 | 12 | def read_bytes(self, size): 13 | end = self.offset + size 14 | if end > self.buffer_len: 15 | raise IOError("{} exceed buffer size: {}".format(end, self.buffer_len)) 16 | ret = self.buffer[self.offset:end] 17 | self.offset = end 18 | return ret 19 | 20 | def read_to_end(self): 21 | ret = self.buffer[self.offset:] 22 | self.offset = self.buffer_len 23 | return ret 24 | 25 | def peek_bytes(self, size): 26 | end = self.offset + size 27 | if end > self.buffer_len: 28 | raise IOError("{} exceed buffer size: {}".format(end, self.buffer_len)) 29 | return self.buffer[self.offset:end] 30 | 31 | def is_finished(self): 32 | return self.offset == self.buffer_len 33 | 34 | def position(self): 35 | return self.offset 36 | 37 | def read_u8(self): 38 | arr = self.read_bytes(1) 39 | return int(arr[0]) 40 | -------------------------------------------------------------------------------- /canoser/delegate_t.py: -------------------------------------------------------------------------------- 1 | from canoser.types import type_mapping 2 | from canoser.base import Base 3 | 4 | 5 | class DelegateT(Base): 6 | delegate_type = 'delegate' 7 | 8 | @classmethod 9 | def dtype(cls): 10 | return type_mapping(cls.delegate_type) 11 | 12 | @classmethod 13 | def encode(cls, value): 14 | return cls.dtype().encode(value) 15 | 16 | @classmethod 17 | def decode(cls, cursor): 18 | return cls.dtype().decode(cursor) 19 | 20 | @classmethod 21 | def check_value(cls, value): 22 | cls.dtype().check_value(value) 23 | 24 | @classmethod 25 | def to_json_serializable(cls, value): 26 | # TODO: bad smell 27 | # if hasattr(cls, "to_json_serializable"): 28 | if 'to_json_serializable' in cls.__dict__.keys(): 29 | return cls.to_json_serializable(value) 30 | return cls.dtype().to_json_serializable(value) 31 | -------------------------------------------------------------------------------- /canoser/int_type.py: -------------------------------------------------------------------------------- 1 | from random import randint 2 | from canoser.base import Base 3 | from struct import pack, unpack 4 | from_bytes = int.from_bytes 5 | 6 | 7 | class IntType(Base): 8 | 9 | @classmethod 10 | def to_json_serializable(cls, value): 11 | return value 12 | 13 | @classmethod 14 | def encode(cls, value): 15 | return pack(cls.pack_str, value) 16 | 17 | @classmethod 18 | def encode_slow(cls, value): 19 | return value.to_bytes(cls.byte_lens, byteorder="little", signed=cls.signed) 20 | 21 | @classmethod 22 | def decode_bytes_slow(cls, bytes): 23 | return unpack(cls.pack_str, bytes)[0] 24 | 25 | @classmethod 26 | def decode_bytes(cls, bytes): 27 | return from_bytes(bytes, byteorder='little', signed=cls.signed) 28 | 29 | @classmethod 30 | def decode(cls, cursor): 31 | bytes = cursor.read_bytes(cls.byte_lens) 32 | return cls.decode_bytes(bytes) 33 | 34 | @classmethod 35 | def int_unsafe(cls, s): 36 | ret = int(s) 37 | cls.check_value(ret) 38 | return ret 39 | 40 | @classmethod 41 | def int_safe(cls, s): 42 | """ 43 | Only allow safe str and valid int to be coerced to destination IntType 44 | """ 45 | if isinstance(s, bool): 46 | raise TypeError(f"{s} is not a integer") 47 | if isinstance(s, int): 48 | cls.check_value(s) 49 | return s 50 | if not isinstance(s, str): 51 | raise TypeError(f"{s} is not instance of .") 52 | if len(s) < 1: 53 | raise TypeError(f"'{s}' is empty.") 54 | len_min = len(str(cls.min_value)) 55 | len_max = len(str(cls.max_value)) 56 | if len(s) > max(len_min, len_max): 57 | raise TypeError(f"Length of {s} is larger than max:{max(len_min, len_max)}.") 58 | ret = int(s) 59 | cls.check_value(ret) 60 | return ret 61 | 62 | @classmethod 63 | def check_value(cls, value): 64 | if isinstance(value, bool): 65 | raise TypeError(f"{value} is not a integer") 66 | if not isinstance(value, int): 67 | raise TypeError(f"{value} is not instance of .") 68 | min, max = cls.min_value, cls.max_value 69 | if value < min or value > max: 70 | raise TypeError('value {} not in range {}-{}'.format(value, min, max)) 71 | 72 | @classmethod 73 | def checked_add(cls, v1, v2): 74 | # rust style api 75 | cls.check_value(v1) 76 | cls.check_value(v2) 77 | try: 78 | ret = v1 + v2 79 | cls.check_value(ret) 80 | return ret 81 | except TypeError: 82 | return None 83 | 84 | @classmethod 85 | def random(cls): 86 | return randint(cls.min_value, cls.max_value) 87 | 88 | 89 | class Int8(IntType): 90 | pack_str = "= 0x80: 148 | # Write 7 (lowest) bits of data and set the 8th bit to 1. 149 | byte = (value & 0x7f) 150 | ret.append(byte | 0x80) 151 | value >>= 7 152 | 153 | # Write the remaining bits of data and set the highest bit to 0. 154 | ret.append(value) 155 | return bytes(ret) 156 | 157 | @classmethod 158 | def parse_uint32_from_uleb128(cls, cursor): 159 | max_shift = 28 160 | value = 0 161 | shift = 0 162 | while not cursor.is_finished(): 163 | byte = cursor.read_u8() 164 | val = byte & 0x7f 165 | value |= (val << shift) 166 | if val == byte: 167 | return value 168 | shift += 7 169 | if shift > max_shift: 170 | break 171 | raise ValueError("invalid ULEB128 representation for Uint32") 172 | 173 | 174 | class Uint64(IntType): 175 | pack_str = "= len(cls._enums): 25 | raise TypeError(f"index{index} out of bound:0-{len(cls._enums)-1}") 26 | _name, datatype = cls._enums[index] 27 | ret = cls.__new__(cls) 28 | ret._init_with_index_value(index, value, datatype) 29 | return ret 30 | 31 | def _init_with_index_value(self, index, value, datatype): 32 | self._index = index 33 | self.value_type = type_mapping(datatype) 34 | self.value = value 35 | 36 | def __init__(self, name, value=None): 37 | if not self.__class__._enums: 38 | raise TypeError(f'{self.__class__} has no _enums defined.') 39 | index = self.__class__.get_index(name) 40 | _name, datatype = self._enums[index] 41 | if name != _name: 42 | raise AssertionError(f"{name} != {_name}") 43 | self._init_with_index_value(index, value, datatype) 44 | 45 | # __getattr__ only gets called for attributes that don't actually exist. 46 | # If you set an attribute directly, referencing that attribute will retrieve it without calling __getattr__. 47 | # If you need to catch every attribute regardless whether it exists or not, use __getattribute__ instead. 48 | def __getattr__(self, name): 49 | if name == '_index': 50 | return None 51 | try: 52 | return self._index == self.__class__.get_index(name) 53 | except TypeError: 54 | return super().__getattr__(self, name) 55 | 56 | def __setattr__(self, name, value): 57 | if name == "value": 58 | TypedProperty.check_type(self.value_type, value) 59 | self.__dict__[name] = value 60 | elif name == "_index" or name == "value_type": 61 | self.__dict__[name] = value 62 | else: 63 | raise TypeError(f"{name} not allowed to modify in {self}.") 64 | 65 | @property 66 | def index(self): 67 | return self._index 68 | 69 | @property 70 | def enum_name(self): 71 | name, _ = self.__class__._enums[self._index] 72 | return name 73 | 74 | @classmethod 75 | def encode(cls, enum): 76 | ret = Uint32.serialize_uint32_as_uleb128(enum.index) 77 | if enum.value_type is not None: 78 | ret += enum.value_type.encode(enum.value) 79 | return ret 80 | 81 | @classmethod 82 | def decode(cls, cursor): 83 | index = Uint32.parse_uint32_from_uleb128(cursor) 84 | _name, datatype = cls._enums[index] 85 | if datatype is not None: 86 | value = type_mapping(datatype).decode(cursor) 87 | return cls.new_with_index_value(index, value) 88 | else: 89 | return cls.new_with_index_value(index, None) 90 | 91 | @classmethod 92 | def check_value(cls, value): 93 | if not isinstance(value, cls): 94 | raise TypeError('value {} is not {} type'.format(value, cls)) 95 | 96 | def __eq__(self, other): 97 | if not isinstance(other, self.__class__): 98 | return False 99 | return self.index == other.index and self.value == other.value 100 | 101 | def to_json_serializable(self): 102 | if self.value_type is None: 103 | return self.enum_name 104 | jj = self.value_type.to_json_serializable(self.value) 105 | return {self.enum_name: jj} 106 | 107 | def __str__(self): 108 | return self.to_json(indent=2) 109 | 110 | def __repr__(self): 111 | return self.__class__.__qualname__ + self.to_json(indent=2) 112 | 113 | def to_json(self, sort_keys=False, indent=4): 114 | amap = self.to_json_serializable() 115 | return json.dumps(amap, sort_keys=sort_keys, indent=indent) 116 | -------------------------------------------------------------------------------- /canoser/rust_optional.py: -------------------------------------------------------------------------------- 1 | from canoser.base import Base 2 | from canoser.bool_t import BoolT 3 | from canoser.struct import TypedProperty 4 | from canoser.types import type_mapping 5 | 6 | 7 | class RustOptional(Base): 8 | _type = None 9 | 10 | def __init__(self, value=None): 11 | if not self.__class__._type: 12 | raise TypeError(f'{self.__class__} has no _type defined.') 13 | self.__dict__["value_type"] = type_mapping(self.__class__._type) 14 | self.value = value 15 | 16 | def __setattr__(self, name, value): 17 | if name == "value": 18 | if value is not None: 19 | TypedProperty.check_type(self.value_type, value) 20 | self.__dict__[name] = value 21 | else: 22 | raise TypeError(f"{name} not allowed to modify in {self}.") 23 | 24 | @classmethod 25 | def encode(cls, optional): 26 | if optional.value is not None: 27 | ret = BoolT.encode(True) 28 | ret += optional.value_type.encode(optional.value) 29 | return ret 30 | else: 31 | return BoolT.encode(False) 32 | 33 | @classmethod 34 | def decode(cls, cursor): 35 | exist = BoolT.decode(cursor) 36 | if exist: 37 | value = cls._type.decode(cursor) 38 | return cls(value) 39 | else: 40 | return cls() 41 | 42 | @classmethod 43 | def check_value(cls, value): 44 | if not isinstance(value, cls): 45 | raise TypeError('value {} is not {} type'.format(value, cls)) 46 | 47 | def __eq__(self, other): 48 | if not isinstance(other, self.__class__): 49 | return False 50 | return self.value == other.value 51 | 52 | def to_json_serializable(self): 53 | if self.value is None: 54 | return None 55 | return self.value_type.to_json_serializable(self.value) 56 | -------------------------------------------------------------------------------- /canoser/str_t.py: -------------------------------------------------------------------------------- 1 | from canoser.int_type import Uint32 2 | from canoser.base import Base 3 | 4 | 5 | class StrT(Base): 6 | @classmethod 7 | def encode(self, value): 8 | output = b'' 9 | utf8 = value.encode('utf-8') 10 | output += Uint32.serialize_uint32_as_uleb128(len(utf8)) 11 | output += utf8 12 | return output 13 | 14 | @classmethod 15 | def decode(self, cursor): 16 | strlen = Uint32.parse_uint32_from_uleb128(cursor) 17 | return str(cursor.read_bytes(strlen), encoding='utf-8') 18 | 19 | @classmethod 20 | def check_value(cls, value): 21 | if not isinstance(value, str): 22 | raise TypeError('value {} is not string'.format(value)) 23 | 24 | @classmethod 25 | def to_json_serializable(cls, obj): 26 | return obj 27 | -------------------------------------------------------------------------------- /canoser/struct.py: -------------------------------------------------------------------------------- 1 | from canoser.base import Base 2 | from canoser.types import type_mapping 3 | import json 4 | 5 | 6 | class TypedProperty: 7 | def __init__(self, name, expected_type): 8 | self.name = name 9 | self.expected_type = expected_type 10 | 11 | def __set__(self, instance, value): 12 | TypedProperty.check_type(self.expected_type, value) 13 | instance.__dict__[self.name] = value 14 | 15 | @staticmethod 16 | def check_type(datatype, value): 17 | if datatype is None: 18 | if value is not None: 19 | raise TypeError(f'{datatype} mismatch {value}') 20 | else: 21 | return 22 | check = getattr(datatype, "check_value", None) 23 | if callable(check): 24 | check(value) 25 | else: 26 | raise TypeError('{} has no check_value method'.format(datatype)) 27 | 28 | 29 | class Struct(Base): 30 | _fields = [] 31 | _initialized = False 32 | 33 | @classmethod 34 | def initailize_fields_type(cls): 35 | if not cls._initialized: 36 | cls._initialized = True 37 | for name, atype in cls._fields: 38 | setattr(cls, name, TypedProperty(name, type_mapping(atype))) 39 | 40 | def __init__(self, *args, **kwargs): 41 | self.__class__.initailize_fields_type() 42 | 43 | if len(args) > len(self._fields): 44 | raise TypeError('Expected {} arguments'.format(len(self._fields))) 45 | 46 | # Set all of the positional arguments 47 | for (name, _type), value in zip(self._fields, args): 48 | typed = getattr(self, name) 49 | typed.__set__(self, value) 50 | 51 | # Set the remaining keyword arguments 52 | for name, _type in self._fields[len(args):]: 53 | if name in kwargs: 54 | typed = getattr(self, name) 55 | typed.__set__(self, kwargs.pop(name)) 56 | 57 | # Check for any remaining unknown arguments 58 | if kwargs: 59 | raise TypeError('Invalid argument(s): {}'.format(','.join(kwargs))) 60 | 61 | @classmethod 62 | def encode(cls, obj): 63 | output = b'' 64 | for name, atype in obj._fields: 65 | value = getattr(obj, name) 66 | output += type_mapping(atype).encode(value) 67 | return output 68 | 69 | @classmethod 70 | def decode(cls, cursor): 71 | ret = cls.__new__(cls) 72 | ret.__init__() 73 | for name, atype in ret._fields: 74 | prop = getattr(ret, name) 75 | mtype = type_mapping(atype) 76 | assert mtype == prop.expected_type 77 | value = mtype.decode(cursor) 78 | prop.__set__(ret, value) 79 | return ret 80 | 81 | @classmethod 82 | def check_value(cls, value): 83 | if not isinstance(value, cls): 84 | raise TypeError('value {} is not {} type'.format(value, cls)) 85 | 86 | def __eq__(self, other): 87 | if type(self) != type(other): 88 | return False 89 | for name, atype in self._fields: 90 | v1 = getattr(self, name) 91 | v2 = getattr(other, name) 92 | if v1 != v2: 93 | return False 94 | return True 95 | 96 | def to_json_serializable(self): 97 | amap = {} 98 | for name, atype in self._fields: 99 | value = getattr(self, name) 100 | if isinstance(value, TypedProperty): 101 | amap[name] = None 102 | else: 103 | atype = type_mapping(atype) 104 | amap[name] = atype.to_json_serializable(value) 105 | return amap 106 | 107 | def __str__(self): 108 | return self.to_json(indent=2) 109 | 110 | def __repr__(self): 111 | return self.__class__.__qualname__ + self.to_json(indent=2) 112 | 113 | def to_json(self, sort_keys=False, indent=4): 114 | amap = self.to_json_serializable() 115 | return json.dumps(amap, sort_keys=sort_keys, indent=indent) 116 | -------------------------------------------------------------------------------- /canoser/tuple_t.py: -------------------------------------------------------------------------------- 1 | from canoser.base import Base 2 | 3 | 4 | class TupleT(Base): 5 | 6 | def __init__(self, *ttypes): 7 | self.ttypes = ttypes 8 | 9 | def encode(self, value): 10 | output = b"" 11 | zipped = zip(self.ttypes, value) 12 | for k, v in zipped: 13 | output += k.encode(v) 14 | return output 15 | 16 | def decode(self, cursor): 17 | arr = [] 18 | for k in self.ttypes: 19 | arr.append(k.decode(cursor)) 20 | return tuple(arr) 21 | 22 | def check_value(self, value): 23 | if len(value) != len(self.ttypes): 24 | raise TypeError(f"{len(value)} is not equal to {len(self.ttypes)}") 25 | if not isinstance(value, tuple): 26 | raise TypeError(f"{value} is not a tuple.") 27 | zipped = zip(self.ttypes, value) 28 | for k, v in zipped: 29 | k.check_value(v) 30 | 31 | def __eq__(self, other): 32 | if not isinstance(other, TupleT): 33 | return False 34 | zipped = zip(self.ttypes, other.ttypes) 35 | for t1, t2 in zipped: 36 | if t1 != t2: 37 | return False 38 | return True 39 | 40 | def to_json_serializable(cls, obj): 41 | ret = [] 42 | # https://stackoverflow.com/questions/15721363/preserve-python-tuples-with-json 43 | # If need to deserialize tuple back later, above link will help. 44 | zipped = zip(cls.ttypes, obj) 45 | for k, v in zipped: 46 | data = k.to_json_serializable(v) 47 | ret.append(data) 48 | return ret 49 | -------------------------------------------------------------------------------- /canoser/types.py: -------------------------------------------------------------------------------- 1 | from canoser.int_type import Uint8 2 | from canoser.tuple_t import TupleT 3 | from canoser.map_t import MapT 4 | from canoser.str_t import StrT 5 | from canoser.bytes_t import BytesT, ByteArrayT 6 | from canoser.bool_t import BoolT 7 | from canoser.array_t import ArrayT 8 | 9 | 10 | def my_import(name): 11 | components = name.split('.') 12 | mod = __import__(components[0]) 13 | for comp in components[1:]: 14 | mod = getattr(mod, comp) 15 | return mod 16 | 17 | 18 | def type_mapping(field_type): # noqa: C901 19 | """ 20 | Mapping python types to canoser types 21 | """ 22 | if field_type == str: 23 | return StrT 24 | elif field_type == bytes: 25 | return BytesT() 26 | elif field_type == bytearray: 27 | return ByteArrayT() 28 | elif field_type == bool: 29 | return BoolT 30 | elif type(field_type) == list: 31 | if len(field_type) == 0: 32 | return ArrayT(Uint8) 33 | elif len(field_type) == 1: 34 | item = field_type[0] 35 | return ArrayT(type_mapping(item)) 36 | elif len(field_type) == 2: 37 | item = field_type[0] 38 | size = field_type[1] 39 | return ArrayT(type_mapping(item), size) 40 | elif len(field_type) == 3: 41 | item = field_type[0] 42 | size = field_type[1] 43 | encode_len = field_type[2] 44 | return ArrayT(type_mapping(item), size, encode_len) 45 | else: 46 | raise TypeError("Array has one item type, no more.") 47 | raise AssertionError("unreacheable") 48 | elif type(field_type) == dict: 49 | if len(field_type) == 0: 50 | ktype = BytesT() 51 | vtype = [Uint8] 52 | elif len(field_type) == 1: 53 | ktype = next(iter(field_type.keys())) 54 | vtype = next(iter(field_type.values())) 55 | else: 56 | raise TypeError("Map type has one item mapping key type to value type.") 57 | return MapT(type_mapping(ktype), type_mapping(vtype)) 58 | elif type(field_type) == tuple: 59 | arr = [] 60 | for item in field_type: 61 | arr.append(type_mapping(item)) 62 | return TupleT(*arr) 63 | elif type(field_type) == str: 64 | return my_import(field_type) 65 | else: 66 | return field_type 67 | -------------------------------------------------------------------------------- /canoser/util.py: -------------------------------------------------------------------------------- 1 | import struct 2 | 3 | 4 | def int_list_to_hex(ints): 5 | return bytes(ints).hex() 6 | 7 | 8 | def int_list_to_bytes(ints): 9 | return bytes(ints) 10 | # return struct.pack("<{}B".format(len(ints)), *ints).hex() 11 | 12 | 13 | def bytes_to_int_list(bytes_str): 14 | tp = struct.unpack("<{}B".format(len(bytes_str)), bytes_str) 15 | return list(tp) 16 | 17 | 18 | def bytes_to_hex(bytes_str): 19 | return bytes_str.hex() 20 | 21 | 22 | def hex_to_bytes(hex_str): 23 | return bytes.fromhex(hex_str) 24 | 25 | 26 | def hex_to_int_list(hex_str): 27 | return bytes_to_int_list(bytes.fromhex(hex_str)) 28 | -------------------------------------------------------------------------------- /canoser/version.py: -------------------------------------------------------------------------------- 1 | version = "0.8.2" 2 | -------------------------------------------------------------------------------- /dev-requirements.in: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-cov 3 | hypothesis 4 | twine 5 | setuptools 6 | wheel 7 | flake8 -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile dev-requirements.in 6 | # 7 | --index-url https://mirrors.aliyun.com/pypi/simple 8 | --trusted-host mirrors.aliyun.com 9 | 10 | attrs==19.3.0 # via hypothesis, pytest 11 | bleach==3.1.4 # via readme-renderer 12 | certifi==2019.11.28 # via requests 13 | cffi==1.14.0 # via cryptography 14 | chardet==3.0.4 # via requests 15 | coverage==5.1 # via pytest-cov 16 | cryptography==2.8 # via secretstorage 17 | docutils==0.16 # via readme-renderer 18 | entrypoints==0.3 # via flake8 19 | flake8==3.7.9 # via -r dev-requirements.in 20 | hypothesis==5.8.0 # via -r dev-requirements.in 21 | idna==2.9 # via requests 22 | importlib-metadata==1.6.0 # via keyring, pluggy, pytest, twine 23 | jeepney==0.4.3 # via keyring, secretstorage 24 | keyring==21.2.0 # via twine 25 | mccabe==0.6.1 # via flake8 26 | more-itertools==8.2.0 # via pytest 27 | packaging==20.3 # via pytest 28 | pkginfo==1.5.0.1 # via twine 29 | pluggy==0.13.1 # via pytest 30 | py==1.8.1 # via pytest 31 | pycodestyle==2.5.0 # via flake8 32 | pycparser==2.20 # via cffi 33 | pyflakes==2.1.1 # via flake8 34 | pygments==2.6.1 # via readme-renderer 35 | pyparsing==2.4.6 # via packaging 36 | pytest-cov==2.8.1 # via -r dev-requirements.in 37 | pytest==5.4.1 # via -r dev-requirements.in, pytest-cov 38 | readme-renderer==25.0 # via twine 39 | requests-toolbelt==0.9.1 # via twine 40 | requests==2.23.0 # via requests-toolbelt, twine 41 | secretstorage==3.1.2 # via keyring 42 | six==1.14.0 # via bleach, cryptography, packaging, readme-renderer 43 | sortedcontainers==2.1.0 # via hypothesis 44 | tqdm==4.44.1 # via twine 45 | twine==3.1.1 # via -r dev-requirements.in 46 | urllib3==1.25.8 # via requests 47 | wcwidth==0.1.9 # via pytest 48 | webencodings==0.5.1 # via bleach 49 | wheel==0.34.2 # via -r dev-requirements.in 50 | zipp==3.1.0 # via importlib-metadata 51 | 52 | # The following packages are considered to be unsafe in a requirements file: 53 | # setuptools 54 | -------------------------------------------------------------------------------- /docs/design.md: -------------------------------------------------------------------------------- 1 | # Design of canoser 2 | 3 | ## Syntax 4 | The basic principle of the syntax design is to use the python native syntax as many as possible. So, instead of define a type using canoser inner type class 5 | 6 | ```python 7 | _fields = [('an_array_of_boolean', canoser.ArrayT(canoser.BoolT))] 8 | ``` 9 | we support plain python array object: 10 | ```python 11 | _fields = [('an_array_of_boolean', [bool])] 12 | ``` 13 | `type_mapping` function is responsible to convert python object to canoser inner type. 14 | 15 | we support plain python object in assignment statements as well: 16 | ```python 17 | class Bar(Struct): 18 | _fields = [ 19 | ('a', Uint64), 20 | ('b', [Uint8]), 21 | ('c', Addr), 22 | ('d', Uint32), 23 | ] 24 | bar = Bar( 25 | a = 100, 26 | b = [0, 1, 2, 3, 4, 5, 6, 7, 8], 27 | c = addr, 28 | d = 99 29 | ) 30 | ``` 31 | 32 | ## Canoser supported types 33 | 34 | | field type syntax | canoser inner type | python object type | 35 | | ------ | ------ | ------ | 36 | | Uint8 | canoser.Uint8 | int | 37 | | Uint16 | canoser.Uint16 | int | 38 | | Uint32 | canoser.Uint32 | int | 39 | | Uint64 | canoser.Uint64 | int | 40 | | Uint128 | canoser.Uint128 | int | 41 | | Int8 | canoser.Int8 | int | 42 | | Int16 | canoser.Int16 | int | 43 | | Int32 | canoser.Int32 | int | 44 | | Int64 | canoser.Int64 | int | 45 | | Int128 | canoser.Int128 | int | 46 | | bool | BoolT | bool | 47 | | str | StrT | str | 48 | | bytes | BytesT | bytes | 49 | | [] | ArrayT | list | 50 | | {} | HashT | dict | 51 | | () | supported | tuple | 52 | | Struct | canoser.Struct | canoser.Struct | 53 | | RustEnum | RustEnum | RustEnum | 54 | | RustOptional | RustOptional | RustOptional | 55 | | DelegateT | DelegateT | the object type of underline delegated type | 56 | 57 | ### Type and Object Separation 58 | Because we choose to simplify the syntax of library users, so the implementation is a bitter complicated for canoser. For those types that python has support, such as int/bool/str/bytes/list/dict/tuple, the canoser type and the python type is diffrent of a canoser object. 59 | 60 | ### Canoser only Types 61 | For those type that python language didn't has equivlent types, such as struct/enum/optional, we defined new class for them. 62 | 63 | 64 | ## Type interface 65 | 66 | All types should implment following four methods: 67 | ``` 68 | def encode(cls_or_obj, value) 69 | 70 | def decode(cls_or_obj, cursor) 71 | 72 | def check_value(cls_or_obj, value) 73 | 74 | def to_json_serializable(cls_or_obj, value) 75 | 76 | ``` 77 | 78 | `cls_or_obj` is either a canoser type class or a canoser type object. So those four methods can either be classmethods or object methods. 79 | 80 | For example, `ArrayT(BoolT)` is type object, `BoolT` and `RustEnum` is type class. 81 | 82 | 83 | ## Type check 84 | `check_value` is called when struct initailization or field assignment. 85 | 86 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile 6 | # 7 | -------------------------------------------------------------------------------- /script/format_code.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #sed -i $'s/\t/ /g' `find canoser -name "*.py"` 3 | find canoser -name "*.py" | xargs rpl -e "\t" " " 4 | find canoser -name "*.py" | xargs rpl "pdb.set_trace()" "#" 5 | find canoser -name "*.py" | xargs rpl "import pdb" "" 6 | -------------------------------------------------------------------------------- /script/gen_requirements.sh: -------------------------------------------------------------------------------- 1 | pip freeze | grep -v "canoser" | grep -v "pkg-resources" > requirements.txt 2 | -------------------------------------------------------------------------------- /script/update_requirements.sh: -------------------------------------------------------------------------------- 1 | #pip3 install --upgrade pip 2 | pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1 | xargs -n1 pip install -U 3 | 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | import re 3 | 4 | with open("canoser/version.py", "r") as fp: 5 | try: 6 | version = re.findall( 7 | r"^version = \"([0-9\.]+)\"", fp.read(), re.M 8 | )[0] 9 | except IndexError: 10 | raise RuntimeError("Unable to determine version.") 11 | 12 | 13 | 14 | with open("README.md", "r") as fh: 15 | content = fh.read() 16 | arr = content.split("\n") 17 | long_description = "\n".join(arr[4:]) 18 | 19 | 20 | tests_require = [ 21 | 'pytest', 22 | 'hypothesis', 23 | ] 24 | 25 | 26 | install_requires = [ 27 | ] 28 | 29 | 30 | setuptools.setup( 31 | name="canoser", 32 | version=version, 33 | author="yuan xinyu", 34 | author_email="yuan_xin_yu@hotmail.com", 35 | description="A python implementation of the LCS(Libra Canonical Serialization) for the Libra network.", 36 | long_description=long_description, 37 | long_description_content_type="text/markdown", 38 | url="https://github.com/yuan-xy/canoser-python.git", 39 | packages=setuptools.find_packages(), 40 | install_requires=install_requires, 41 | tests_require=tests_require, 42 | classifiers=[ 43 | "Programming Language :: Python :: 3", 44 | "License :: OSI Approved :: MIT License", 45 | "Operating System :: OS Independent", 46 | ], 47 | python_requires='>=3.6', 48 | ) 49 | -------------------------------------------------------------------------------- /test/bench_int.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))) 3 | 4 | import cProfile 5 | from canoser import * 6 | import pdb 7 | 8 | def bench_int_decode(): 9 | for _i in range(553000): 10 | x1 = Uint32.decode_bytes(b"\x12\x34\x56\x78") 11 | x2 = Uint32.decode_bytes_slow(b"\x12\x34\x56\x78") 12 | assert x1 == x2 13 | ex1 = Uint32.encode(x1) 14 | ex2 = Uint32.encode_slow(x2) 15 | assert ex1 == ex2 16 | 17 | cProfile.run('bench_int_decode()', sort='cumtime') -------------------------------------------------------------------------------- /test/bench_serialize.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))) 3 | 4 | import cProfile 5 | from canoser import * 6 | import pdb 7 | 8 | 9 | class Addr(Struct): 10 | _fields = [('addr', [Uint8, 32])] 11 | 12 | 13 | class Bar(Struct): 14 | _fields = [ 15 | ('a', Uint64), 16 | ('b', [Uint8]), 17 | ('c', Addr), 18 | ('d', Uint32), 19 | ] 20 | 21 | class Foo(Struct): 22 | _fields = [ 23 | ('a', Uint64), 24 | ('b', [Uint8]), 25 | ('c', Bar), 26 | ('d', bool), 27 | ('e', {}), 28 | ] 29 | 30 | 31 | arr = [] 32 | for _ in range(32): 33 | arr.append(5) 34 | addr = Addr(addr=arr) 35 | bar = Bar( 36 | a = 100, 37 | b = [0, 1, 2, 3, 4, 5, 6, 7, 8], 38 | c = addr, 39 | d = 99 40 | ) 41 | kvs = {} 42 | kvs[bytes([0, 56, 21])] = [22, 10, 5] 43 | kvs[bytes([1])] = [22, 21, 67] 44 | kvs[bytes([20, 21, 89, 105])] = [201, 23, 90] 45 | foo = Foo( 46 | a = Uint64.max_value, 47 | b = [100, 99, 88, 77, 66, 55], 48 | c = bar, 49 | d = True, 50 | e = kvs 51 | ) 52 | 53 | def test_with_libra_case(): 54 | for _i in range(10000): 55 | str1 = foo.serialize() 56 | foo2 = Foo.deserialize(str1) 57 | assert foo == foo2 58 | 59 | cProfile.run('test_with_libra_case()', sort='cumtime') -------------------------------------------------------------------------------- /test/test_bytes.py: -------------------------------------------------------------------------------- 1 | from canoser import * 2 | import pytest 3 | 4 | 5 | ADDRESS_LENGTH = 32 6 | 7 | input = b'\xca\x82\x0b\xf90^\xb9}\rxOq\xb3\x95TW\xfb\xf6\x91\x1fS\x00\xce\xaa]~\x86!R\x9e\xae\x19' 8 | 9 | class Address1(DelegateT): 10 | delegate_type = BytesT(ADDRESS_LENGTH) 11 | 12 | 13 | class Address2(DelegateT): 14 | delegate_type = BytesT(ADDRESS_LENGTH, encode_len=False) 15 | 16 | 17 | def test_enocde_len(): 18 | expected_output = bytes([32]) + input 19 | actual_output = Address1.encode(input) 20 | assert expected_output == actual_output 21 | 22 | def test_not_enocde_len(): 23 | actual_output = Address2.encode(input) 24 | assert input == actual_output 25 | 26 | class AddrStruct(Struct): 27 | _fields = [('map', {Address2: [str]})] 28 | 29 | def test_address_as_dict_key(): 30 | amap = {input: ['test']} 31 | addrs = AddrStruct(amap) 32 | ser = addrs.serialize() 33 | addr2 = AddrStruct.deserialize(ser) 34 | assert addrs.map == addr2.map 35 | 36 | class BArrayStruct(Struct): 37 | _fields = [('map', {Address2: bytearray})] 38 | 39 | def test_bytearray(): 40 | amap = {input: bytearray(b'ba')} 41 | addrs = BArrayStruct(amap) 42 | ser = addrs.serialize() 43 | addr2 = BArrayStruct.deserialize(ser) 44 | assert addrs.map == addr2.map 45 | -------------------------------------------------------------------------------- /test/test_circular.py: -------------------------------------------------------------------------------- 1 | from canoser import * 2 | import pytest 3 | import pdb 4 | 5 | class Circular(Struct): 6 | _fields = [ 7 | ('intv', Int8), 8 | ('next', ['test_circular.Circular']) 9 | ] 10 | 11 | def test_circular(): 12 | t1 = Circular(1,[]) 13 | t2 = Circular(2,[]) 14 | t12 = Circular(12,[t1, t2]) 15 | bstr = t12.serialize() 16 | assert bstr == bytes([12]) + Uint32.serialize_uint32_as_uleb128(2) + t1.serialize() + t2.serialize() 17 | tt = Circular.deserialize(bstr) 18 | assert tt == t12 19 | assert Circular.deserialize(t1.serialize()) == t1 20 | -------------------------------------------------------------------------------- /test/test_cursor.py: -------------------------------------------------------------------------------- 1 | from canoser import * 2 | import pytest 3 | import pdb 4 | 5 | def test_read(): 6 | data = [6,2,3,4,5] 7 | cursor = Cursor(data) 8 | assert cursor.read_u8() == 6 9 | assert cursor.offset == 1 10 | assert cursor.position() == cursor.offset 11 | assert cursor.peek_bytes(3) == b'\x02\x03\x04' 12 | assert cursor.offset == 1 13 | assert cursor.read_bytes(2) == b'\x02\x03' 14 | assert cursor.offset == 3 15 | assert cursor.is_finished() == False 16 | assert cursor.read_to_end() == b'\x04\x05' 17 | assert cursor.is_finished() == True 18 | 19 | def test_bytearray(): 20 | array = bytearray() 21 | array.append(5) 22 | array.append(2) 23 | array.extend([3, 4]) 24 | cursor = Cursor(array) 25 | assert cursor.read_u8() == 5 26 | assert cursor.offset == 1 27 | assert cursor.position() == cursor.offset 28 | assert cursor.peek_bytes(3) == b'\x02\x03\x04' 29 | assert cursor.offset == 1 30 | assert cursor.read_bytes(2) == b'\x02\x03' 31 | assert cursor.offset == 3 32 | assert cursor.is_finished() == False 33 | assert cursor.read_to_end() == b'\x04' 34 | assert cursor.is_finished() == True -------------------------------------------------------------------------------- /test/test_delegate.py: -------------------------------------------------------------------------------- 1 | from canoser import * 2 | import struct 3 | import pdb 4 | import pytest 5 | 6 | ADDRESS_LENGTH = 32 7 | 8 | class Address(DelegateT): 9 | delegate_type = [Uint8, ADDRESS_LENGTH] 10 | 11 | 12 | class AddrStruct(Struct): 13 | _fields = [('addr', Address)] 14 | 15 | def test_array(): 16 | with pytest.raises(TypeError): 17 | x = AddrStruct([1,2,3]) 18 | arr = [] 19 | for _ in range(ADDRESS_LENGTH): 20 | arr.append(5) 21 | x = AddrStruct(arr) 22 | sx = x.serialize() 23 | x2 = AddrStruct.deserialize(sx) 24 | assert x.addr == x2.addr 25 | print(x2) 26 | assert x2.__str__() == """{ 27 | "addr": "0505050505050505050505050505050505050505050505050505050505050505" 28 | }""" 29 | assert AddrStruct.addr.expected_type.dtype().atype == Uint8 30 | assert AddrStruct.addr.expected_type.dtype().fixed_len == ADDRESS_LENGTH 31 | 32 | 33 | class Bools(DelegateT): 34 | delegate_type = [bool] 35 | 36 | def test_delegate(): 37 | x = [True, False, True] 38 | bs = Bools.encode(x) 39 | assert bs == b'\x03\x01\x00\x01' 40 | x2 = Bools.deserialize(bs) 41 | assert x == x2 42 | -------------------------------------------------------------------------------- /test/test_enum.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pdb 3 | from copy import copy, deepcopy 4 | from canoser import * 5 | 6 | ADDRESS_LENGTH = 32 7 | 8 | 9 | class TransactionArgument(RustEnum): 10 | _enums = [ 11 | ('U64', Uint64), 12 | ('Address', [Uint8, ADDRESS_LENGTH]), 13 | ('String', str), 14 | ('ByteArray', [Uint8]) 15 | ] 16 | 17 | 18 | 19 | def test_invalid(): 20 | with pytest.raises(TypeError): 21 | t_arg = TransactionArgument() 22 | with pytest.raises(TypeError): 23 | x = TransactionArgument('UU64', 0) 24 | with pytest.raises(TypeError): 25 | class NotDefineEnum(RustEnum): 26 | pass 27 | x = NotDefineEnum('X') 28 | 29 | 30 | def test_enum(): 31 | t_arg = TransactionArgument('U64', 2) 32 | assert hasattr(t_arg, 'value') 33 | assert hasattr(t_arg, 'U64') 34 | assert hasattr(t_arg, 'Address') 35 | assert hasattr(t_arg, '__str__') 36 | assert hasattr(t_arg, 'to_proto') == False 37 | assert t_arg.index == 0 38 | assert t_arg.value == 2 39 | assert t_arg.U64 == True 40 | assert t_arg.enum_name == 'U64' 41 | assert t_arg.value_type == Uint64 42 | assert t_arg.serialize() == b"\x00\x02\x00\x00\x00\x00\x00\x00\x00" 43 | assert t_arg == TransactionArgument.deserialize(t_arg.serialize()) 44 | t_arg.value = 3 45 | assert t_arg.value == 3 46 | with pytest.raises(TypeError): 47 | t_arg.value = 'abc' 48 | with pytest.raises(TypeError): 49 | t_arg.index = 2 50 | arg2 = TransactionArgument('String', 'abc') 51 | assert arg2.index == 2 52 | assert arg2.value == 'abc' 53 | assert arg2.String == True 54 | assert arg2.U64 == False 55 | assert arg2.value_type == StrT 56 | assert arg2.__class__ == t_arg.__class__ 57 | arg2.value = 'Bcd' 58 | assert arg2.value == 'Bcd' 59 | with pytest.raises(TypeError): 60 | t_arg.value = 'abc' 61 | with pytest.raises(TypeError): 62 | arg2.value = 0 63 | with pytest.raises(TypeError): 64 | arg2.String = 0 65 | with pytest.raises(TypeError): 66 | arg2.String = 'abc' 67 | 68 | class Enum1(RustEnum): 69 | _enums = [('opt1', [Uint8]), ('opt2', None)] 70 | 71 | 72 | def test_enum2(): 73 | e1 = Enum1.new_with_index_value(0, [5]) 74 | e2 = Enum1('opt2', None) 75 | assert Enum1.new_with_index_value(1, None) == Enum1('opt2') 76 | assert Enum1.encode(e1) == b'\x00\x01\x05' 77 | assert Enum1.encode(e2) == b'\x01' 78 | obj = Enum1.decode(Cursor(b'\x00\x01\x05')) 79 | assert obj.index == 0 80 | assert obj.value == [5] 81 | obj = Enum1.decode(Cursor(b'\x01')) 82 | assert obj.index == 1 83 | assert obj.value == None 84 | 85 | class Enum2(RustEnum): 86 | _enums = [('opt1', None), ('opt2', str)] 87 | 88 | 89 | class EStruct(Struct): 90 | _fields = [('enum', Enum2)] 91 | 92 | def test_enum_struct(): 93 | EStruct.initailize_fields_type() 94 | assert EStruct.enum.expected_type == Enum2 95 | x = EStruct(Enum2('opt1')) 96 | sx = x.serialize() 97 | assert sx == b'\x00' 98 | x2 = EStruct.deserialize(sx) 99 | assert x.enum.index == x2.enum.index 100 | assert x.enum.value == x2.enum.value 101 | 102 | 103 | class MyEnum(RustEnum): 104 | _enums = [('opt1', None), ('opt3', [[str]])] 105 | 106 | class EStruct2(Struct): 107 | _fields = [('enum', MyEnum)] 108 | 109 | def test_enum_struct2(): 110 | EStruct2.initailize_fields_type() 111 | assert EStruct2.enum.expected_type == MyEnum 112 | x = EStruct2(MyEnum('opt3', [['ab', 'c'], ['d'], []])) 113 | deepcopy(x) 114 | sx = x.serialize() 115 | assert sx == b'\x01\x03' +\ 116 | b'\x02' + b'\x02ab' + b'\x01c' +\ 117 | b'\x01' + b'\x01d' + b'\x00' 118 | x2 = EStruct2.deserialize(sx) 119 | assert x.enum.index == x2.enum.index 120 | assert x.enum.value == x2.enum.value 121 | 122 | class MyEnumWrap(MyEnum): 123 | pass 124 | 125 | def test_enum_wrap(): 126 | x1 = MyEnumWrap('opt1') 127 | x2 = MyEnumWrap('opt3', [['ab', 'c'], ['d'], []]) 128 | assert x2.value == [['ab', 'c'], ['d'], []] 129 | assert x2.opt3 == True 130 | assert x2.opt1 == False 131 | x3 = copy(x2) 132 | x2.value = [['abc']] 133 | assert x2.opt3 == True 134 | assert x2.value == [['abc']] 135 | assert x3.value == [['ab', 'c'], ['d'], []] 136 | jstr = x3.to_json() 137 | print(jstr) 138 | assert x2.__repr__() == """MyEnumWrap{ 139 | "opt3": [ 140 | [ 141 | "abc" 142 | ] 143 | ] 144 | }""" 145 | -------------------------------------------------------------------------------- /test/test_libra_example.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pdb 3 | from canoser import * 4 | 5 | 6 | #copy form libra source code 7 | TEST_VECTOR_1 = [ 8 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x06, 0x64, 0x63, 0x58, 0x4d, 0x42, 0x37, 9 | 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 10 | 0x06, 0x07, 0x08, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 11 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 12 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x63, 0x00, 0x00, 0x00, 0x01, 0x03, 0x01, 0x01, 0x03, 0x16, 13 | 0x15, 0x43, 0x03, 0x00, 0x38, 0x15, 0x03, 0x16, 0x0a, 0x05, 0x04, 0x14, 0x15, 0x59, 0x69, 14 | 0x03, 0xc9, 0x17, 0x5a, 15 | ] 16 | 17 | class Addr(Struct): 18 | _fields = [('addr', [Uint8, 32, False])] 19 | 20 | 21 | class Bar(Struct): 22 | _fields = [ 23 | ('a', Uint64), 24 | ('b', [Uint8]), 25 | ('c', Addr), 26 | ('d', Uint32), 27 | ] 28 | 29 | class Foo(Struct): 30 | _fields = [ 31 | ('a', Uint64), 32 | ('b', [Uint8]), 33 | ('c', Bar), 34 | ('d', bool), 35 | ('e', {}), 36 | ] 37 | 38 | 39 | def test_with_libra_case(): 40 | arr = [] 41 | for _ in range(32): 42 | arr.append(5) 43 | addr = Addr(addr=arr) 44 | bar = Bar( 45 | a = 100, 46 | b = [0, 1, 2, 3, 4, 5, 6, 7, 8], 47 | c = addr, 48 | d = 99 49 | ) 50 | assert Bar.a.expected_type == Uint64 51 | assert Bar.b.expected_type == ArrayT(Uint8) 52 | assert Bar.c.expected_type == Addr 53 | assert Bar.d.expected_type == Uint32 54 | kvs = {} 55 | kvs[bytes([0, 56, 21])] = [22, 10, 5] 56 | kvs[bytes([1])] = [22, 21, 67] 57 | kvs[bytes([20, 21, 89, 105])] = [201, 23, 90] 58 | foo = Foo( 59 | a = Uint64.max_value, 60 | b = [100, 99, 88, 77, 66, 55], 61 | c = bar, 62 | d = True, 63 | e = kvs 64 | ) 65 | str1 = foo.serialize() 66 | str2 = bytes(TEST_VECTOR_1) 67 | assert str1 == str2 68 | foo2 = Foo.deserialize(str1) 69 | assert foo == foo2 70 | -------------------------------------------------------------------------------- /test/test_list.py: -------------------------------------------------------------------------------- 1 | from canoser import * 2 | import pytest 3 | import pdb 4 | 5 | ADDRESS_LENGTH = 32 6 | 7 | input = [ 8 | 0xca, 0x82, 0x0b, 0xf9, 0x30, 0x5e, 0xb9, 0x7d, 0x0d, 0x78, 0x4f, 0x71, 0xb3, 0x95, 0x54, 9 | 0x57, 0xfb, 0xf6, 0x91, 0x1f, 0x53, 0x00, 0xce, 0xaa, 0x5d, 0x7e, 0x86, 0x21, 0x52, 0x9e, 10 | 0xae, 0x19, 11 | ] 12 | 13 | expected_output = [ 14 | 0xCA, 0x82, 0x0B, 0xF9, 0x30, 0x5E, 0xB9, 0x7D, 0x0D, 0x78, 0x4F, 15 | 0x71, 0xB3, 0x95, 0x54, 0x57, 0xFB, 0xF6, 0x91, 0x1F, 0x53, 0x00, 0xCE, 0xAA, 0x5D, 0x7E, 16 | 0x86, 0x21, 0x52, 0x9E, 0xAE, 0x19, 17 | ] 18 | 19 | 20 | class Address1(DelegateT): 21 | delegate_type = [Uint8, ADDRESS_LENGTH] 22 | 23 | 24 | class Address2(DelegateT): 25 | delegate_type = [Uint8, ADDRESS_LENGTH, False] 26 | 27 | 28 | def test_enocde_len(): 29 | expected_output0 = [32] + expected_output 30 | actual_output = Address1.encode(input) 31 | assert bytes(expected_output0) == actual_output 32 | 33 | def test_not_enocde_len(): 34 | actual_output = Address2.encode(input) 35 | assert bytes(input) == actual_output 36 | assert bytes(expected_output) == actual_output 37 | 38 | class AddrStruct(Struct): 39 | _fields = [('map', {Address2: [str]})] 40 | 41 | def test_int_list_as_dict_key(): 42 | amap = {bytes(input): ['test']} 43 | addrs = AddrStruct(amap) 44 | ser = addrs.serialize() 45 | addr2 = AddrStruct.deserialize(ser) 46 | assert addrs.map == addr2.map -------------------------------------------------------------------------------- /test/test_optional.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pdb 3 | from canoser import * 4 | 5 | class OptionUInt(RustOptional): 6 | _type = Uint8 7 | 8 | def test_optional(): 9 | null = OptionUInt(None) 10 | obj = OptionUInt(8) 11 | assert OptionUInt.encode(null) == b'\x00' 12 | assert OptionUInt.encode(obj) == b'\x01\x08' 13 | assert OptionUInt.decode(Cursor(b'\x01\x08')).value == 8 14 | assert OptionUInt.decode(Cursor(b'\x00')).value == None 15 | assert obj.serialize() == b'\x01\x08' 16 | assert obj == OptionUInt.deserialize(obj.serialize()) 17 | with pytest.raises(TypeError): 18 | obj.value = -1 19 | with pytest.raises(TypeError): 20 | obj.value = "abc" 21 | obj.value = None 22 | assert obj.value == None 23 | obj.value = 123 24 | assert obj.value == 123 25 | 26 | 27 | class OptionInt(RustOptional): 28 | _type = Int8 29 | 30 | class OStruct(Struct): 31 | _fields = [('opt', OptionInt)] 32 | 33 | def test_optional_struct(): 34 | x = OStruct(opt = OptionInt(-1)) 35 | assert x.to_json_serializable() == {'opt': -1} 36 | assert OStruct.opt.expected_type == OptionInt 37 | sx = x.serialize() 38 | assert sx == b'\x01\xff' 39 | x2 = OStruct.deserialize(sx) 40 | assert x.opt.value == x2.opt.value 41 | with pytest.raises(TypeError): 42 | x.opt = -1 43 | 44 | def test_optional_struct_null(): 45 | x = OStruct(opt = OptionInt()) 46 | assert x.to_json_serializable() == {'opt': None} 47 | assert x.opt.value is None 48 | sx = x.serialize() 49 | assert sx == b'\x00' 50 | x2 = OStruct.deserialize(sx) 51 | assert x2.opt.value is None 52 | -------------------------------------------------------------------------------- /test/test_proptest.py: -------------------------------------------------------------------------------- 1 | from hypothesis import given 2 | import hypothesis.strategies as st 3 | from canoser import Struct, Uint8, Uint64, Uint128, Int8, Int64, Int128 4 | 5 | 6 | def encode_decode_is_same(vtype, value): 7 | byts = vtype.encode(value) 8 | value2 = vtype.decode_bytes(byts) 9 | assert value == value2 10 | 11 | 12 | @given(st.integers()) 13 | def test_prop(x): 14 | if x >= Uint8.min_value and x <= Uint8.max_value: 15 | encode_decode_is_same(Uint8, x) 16 | if x >= Uint64.min_value and x <= Uint64.max_value: 17 | encode_decode_is_same(Uint64, x) 18 | if x >= Uint128.min_value and x <= Uint128.max_value: 19 | encode_decode_is_same(Uint128, x) 20 | if x >= Int8.min_value and x <= Int8.max_value: 21 | encode_decode_is_same(Int8, x) 22 | if x >= Int64.min_value and x <= Int64.max_value: 23 | encode_decode_is_same(Int64, x) 24 | if x >= Int128.min_value and x <= Int128.max_value: 25 | encode_decode_is_same(Int128, x) 26 | 27 | 28 | class PropStruct(Struct): 29 | _fields = [('name', str), ('value', Int128), ('flag', bool)] 30 | 31 | @given(st.text(), st.integers(), st.booleans()) 32 | def test_ser_deser(name, value, flag): 33 | t = PropStruct(name, value, flag) 34 | ser = t.serialize() 35 | t2 = PropStruct.deserialize(ser) 36 | assert t == t2 37 | -------------------------------------------------------------------------------- /test/test_struct.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pdb 3 | from canoser import * 4 | 5 | 6 | class Stock(Struct): 7 | _fields = [('name', str), ('shares', Uint8)] 8 | 9 | 10 | def test_struct_init(): 11 | s1 = Stock('ACME', 50) 12 | assert s1.name == "ACME" 13 | assert s1.shares == 50 14 | s2 = Stock('ACME', shares=50) 15 | s3 = Stock(name='ACME', shares=50) 16 | s33 = Stock(shares=50, name='ACME') 17 | assert s3 == s33 18 | s6 = Stock(shares=50) 19 | with pytest.raises(TypeError): 20 | s4 = Stock('ACME', 500) 21 | with pytest.raises(TypeError): 22 | s5 = Stock('ACME', shares=50, aa=1) 23 | with pytest.raises(TypeError): 24 | s6 = Stock(123) 25 | 26 | 27 | def test_struct_serialize(): 28 | s1 = Stock('ACME', 50) 29 | bstr = s1.serialize() 30 | s2 = Stock.deserialize(bstr) 31 | assert s2.name == "ACME" 32 | assert s2.shares == 50 33 | 34 | class BoolS(Struct): 35 | _fields = [('boolean', bool)] 36 | 37 | def test_bool(): 38 | x = BoolS(True) 39 | sx = x.serialize() 40 | x2 = BoolS.deserialize(sx) 41 | assert x.boolean == x2.boolean 42 | assert x.to_json() == """{ 43 | "boolean": true 44 | }""" 45 | 46 | class ArrayS(Struct): 47 | _fields = [('array', [bool])] 48 | 49 | def test_array(): 50 | x = ArrayS([True, False, True]) 51 | sx = x.serialize() 52 | assert b"\3\1\0\1" == sx 53 | x2 = ArrayS.deserialize(sx) 54 | assert x.array == x2.array 55 | 56 | def test_array_error(): 57 | with pytest.raises(TypeError): 58 | ArrayS.deserialize(b"\3\1\0\2") 59 | x = ArrayS([]) 60 | with pytest.raises(TypeError): 61 | x.array = ["abc"] 62 | # with pytest.raises(TypeError): 63 | # x.array.append("abc") 64 | 65 | 66 | class ArrayUint8(Struct): 67 | _fields = [('array', [Uint8])] 68 | 69 | def test_array(): 70 | with pytest.raises(Exception): 71 | x = ArrayUint8([0.1]) 72 | with pytest.raises(TypeError): 73 | x = ArrayUint8([-1]) 74 | with pytest.raises(TypeError): 75 | x = ArrayUint8(b'') 76 | x = ArrayUint8([]) 77 | with pytest.raises(TypeError): 78 | x.array = b'' 79 | 80 | 81 | class MapS(Struct): 82 | _fields = [('kvs', {str : Uint64})] 83 | 84 | def test_map(): 85 | x = MapS(kvs = {"count1":123456789, "count2":987654321}) 86 | sx = x.serialize() 87 | x2 = MapS.deserialize(sx) 88 | assert x.kvs == x2.kvs 89 | 90 | class MapS2(Struct): 91 | _fields = [('counters', {Uint16 : Uint64})] 92 | 93 | 94 | def test_map_empty(): 95 | x = MapS2({}) 96 | sx = x.serialize() 97 | x2 = MapS2.deserialize(sx) 98 | assert x.counters == x2.counters 99 | 100 | class ChineseMap(Struct): 101 | _fields = [('kvs', {str : str})] 102 | 103 | def test_map2(): 104 | x = ChineseMap(kvs = {"中文":"测试"}) 105 | sx = x.serialize() 106 | x2 = ChineseMap.deserialize(sx) 107 | assert x.kvs == x2.kvs 108 | assert x2.kvs["中文"] == "测试" 109 | assert x2.to_json() == """{ 110 | "kvs": { 111 | "\\u4e2d\\u6587": "\\u6d4b\\u8bd5" 112 | } 113 | }""" 114 | 115 | 116 | 117 | class ByteS(Struct): 118 | _fields = [('kvs', {bytes : Uint64})] 119 | 120 | def test_bytes(): 121 | x = ByteS(kvs = {b"count1":123456789, b"count2":987654321}) 122 | sx = x.serialize() 123 | x2 = ByteS.deserialize(sx) 124 | assert x.kvs == x2.kvs 125 | with pytest.raises(TypeError): 126 | x2.kvs = {'a' : 'b'} 127 | print(x2.to_json()) 128 | assert x2.to_json() == """{ 129 | "kvs": { 130 | "636f756e7431": 123456789, 131 | "636f756e7432": 987654321 132 | } 133 | }""" 134 | 135 | 136 | class TupleS(Struct): 137 | _fields = [('tp', (str, Uint8, bool, Int16))] 138 | 139 | def test_tuple_struct(): 140 | x = TupleS(tp = ("abc", 1, False, 2)) 141 | assert TupleS.tp.expected_type == TupleT(StrT, Uint8, BoolT, Int16) 142 | sx = x.serialize() 143 | assert sx == b'\x03\x61\x62\x63\x01\x00\x02\x00' 144 | x2 = TupleS.deserialize(sx) 145 | assert x.tp == x2.tp 146 | with pytest.raises(TypeError): 147 | x.tp = () 148 | 149 | 150 | 151 | class Uint128S(Struct): 152 | _fields = [('u128', Uint128)] 153 | 154 | def test_uint128_struct(): 155 | x = Uint128S(u128 = Uint128.max_value) 156 | assert Uint128S.u128.expected_type == Uint128 157 | sx = x.serialize() 158 | assert sx == b'\xff' * 16 159 | x2 = Uint128S.deserialize(sx) 160 | assert x.u128 == x2.u128 161 | 162 | def test_print_null_field(): 163 | x = Uint128S() 164 | print(x) 165 | print(x.__repr__()) -------------------------------------------------------------------------------- /test/test_types.py: -------------------------------------------------------------------------------- 1 | from canoser import * 2 | from canoser.types import type_mapping 3 | import pdb 4 | import pytest 5 | 6 | def test_random(): 7 | for _x in range(10000): 8 | rand = Uint8.random() 9 | assert rand >=0 and rand <256 10 | 11 | def test_checked_add(): 12 | assert 255 == Uint8.checked_add(254, 1) 13 | assert None == Uint8.checked_add(254, 2) 14 | 15 | def test_str_to_int(): 16 | with pytest.raises(ValueError): 17 | Uint8.int_unsafe("") 18 | assert 0 == Uint8.int_unsafe("0") 19 | assert 0 == Uint8.int_unsafe(b"0") 20 | assert 1 == Uint8.int_unsafe(b"01") 21 | assert 0 == Uint8.int_unsafe("0"*100) 22 | assert 255 == Uint8.int_unsafe("255") 23 | assert 255 == Uint8.int_unsafe("0255") 24 | with pytest.raises(TypeError): 25 | Uint8.int_unsafe("-1") 26 | with pytest.raises(TypeError): 27 | Uint8.int_unsafe("256") 28 | with pytest.raises(TypeError): 29 | Uint8.check_value(True) 30 | 31 | 32 | def test_str_to_int_strict(): 33 | assert 0 == Uint8.int_safe(0) 34 | assert 255 == Uint8.int_safe(255) 35 | assert -128 == Int8.int_safe(-128) 36 | assert 127 == Int8.int_safe(127) 37 | assert 0 == Uint8.int_safe("0") 38 | assert 255 == Uint8.int_safe("255") 39 | assert -128 == Int8.int_safe("-128") 40 | assert 127 == Int8.int_safe("127") 41 | assert 65535 == Uint16.int_safe("65535") 42 | with pytest.raises(Exception): 43 | Uint8.int_safe("") 44 | with pytest.raises(Exception): 45 | Uint8.int_safe(b"0") 46 | with pytest.raises(Exception): 47 | Uint8.int_safe(b"01") 48 | with pytest.raises(Exception): 49 | Uint8.int_safe("0"*100) 50 | with pytest.raises(Exception): 51 | Uint8.int_safe("0255") 52 | with pytest.raises(Exception): 53 | Uint8.int_safe("-1") 54 | with pytest.raises(Exception): 55 | Uint8.int_safe("256") 56 | with pytest.raises(Exception): 57 | Int8.int_safe("-129") 58 | with pytest.raises(Exception): 59 | Int8.int_safe("128") 60 | with pytest.raises(Exception): 61 | Uint8.int_safe(-1) 62 | with pytest.raises(Exception): 63 | Uint8.int_safe(256) 64 | with pytest.raises(Exception): 65 | Int8.int_safe(-129) 66 | with pytest.raises(Exception): 67 | Int8.int_safe(128) 68 | 69 | def test_bool_cast_int(): 70 | assert isinstance(True, bool) == True 71 | assert isinstance(True, int) == True 72 | assert True == 1 73 | assert False == (True < 1) 74 | with pytest.raises(Exception): 75 | Int8.int_safe(True) 76 | with pytest.raises(Exception): 77 | Int8.int_safe(False) 78 | with pytest.raises(Exception): 79 | Int8.int_safe(None) 80 | 81 | def test_int(): 82 | assert Int8.encode(16) == Uint8.encode(16) 83 | 84 | 85 | def test_uint8(): 86 | assert Uint8.encode(16) == b"\x10" 87 | assert Uint8.decode_bytes(b"\x10") == 16 88 | assert Uint8.max_value == 255 89 | assert Uint8.min_value == 0 90 | 91 | def test_uint8_illegal(): 92 | with pytest.raises(Exception): 93 | Uint8.encode(-1) 94 | with pytest.raises(Exception): 95 | Uint8.encode(0.1) 96 | with pytest.raises(Exception): 97 | Uint8.encode([0]) 98 | with pytest.raises(Exception): 99 | Uint8.encode(b'0') 100 | 101 | def test_int8(): 102 | assert Int8.encode(16) == b"\x10" 103 | assert Int8.decode_bytes(b"\x10") == 16 104 | assert Int8.max_value == 127 105 | assert Int8.min_value == -128 106 | assert Int8.encode(-1) == b"\xFF" 107 | assert Int8.decode_bytes(b"\xFF") == -1 108 | assert Int8.decode_bytes(b"\x80") == -128 109 | 110 | 111 | def test_uint16(): 112 | assert Uint16.encode(16) == b"\x10\x00" 113 | assert Uint16.encode(257) == b"\x01\x01" 114 | assert Uint16.decode_bytes(b"\x01\x01") == 257 115 | assert Uint16.max_value == 65535 116 | assert Uint16.min_value == 0 117 | 118 | def test_int16(): 119 | assert Int16.encode(16) == b"\x10\x00" 120 | assert Int16.decode_bytes(b"\x10\x00") == 16 121 | assert Int16.max_value == 32767 122 | assert Int16.min_value == -32768 123 | assert Int16.encode(-1) == b"\xFF\xFF" 124 | assert Int16.decode_bytes(b"\xFF\xFF") == -1 125 | assert Int16.decode_bytes(b"\x00\x80") == -32768 126 | 127 | def test_uint32(): 128 | assert Uint32.encode(16) == b"\x10\x00\x00\x00" 129 | assert Uint32.encode(0x12345678) == b"\x78\x56\x34\x12" 130 | assert Uint32.decode_bytes(b"\x78\x56\x34\x12") == 0x12345678 131 | 132 | 133 | def test_uint64(): 134 | assert Uint64.encode(16) == b"\x10\x00\x00\x00\x00\x00\x00\x00" 135 | assert Uint64.encode(0x1234567811223344) == b"\x44\x33\x22\x11\x78\x56\x34\x12" 136 | assert Uint64.decode_bytes(b"\x44\x33\x22\x11\x78\x56\x34\x12" ) == 0x1234567811223344 137 | 138 | def test_uint128(): 139 | assert Uint128.encode(Uint128.max_value) == b"\xff" * 16 140 | assert Uint128.decode_bytes(b"\xff" * 16) == Uint128.max_value 141 | v1 = b"\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00" 142 | assert Uint128.encode(Uint64.max_value+1) == v1 143 | assert Uint128.decode_bytes(v1) == Uint64.max_value+1 144 | v2 = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80' 145 | assert Uint128.encode(Int128.max_value+1) == v2 146 | assert Uint128.decode_bytes(v2) == Int128.max_value+1 147 | 148 | def test_int128(): 149 | v0 = b"\xff" * 15 + b"\x7f" 150 | assert Int128.encode(Int128.max_value) == v0 151 | assert Int128.decode_bytes(v0) == Int128.max_value 152 | v1 = b"\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00" 153 | assert Int128.encode(Uint64.max_value+1) == v1 154 | assert Int128.decode_bytes(v1) == Uint64.max_value+1 155 | v2 = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80' 156 | assert Int128.encode(Int128.min_value) == v2 157 | assert Int128.decode_bytes(v2) == Int128.min_value 158 | assert Int128.encode(-1) == b"\xff" * 16 159 | assert Int128.decode_bytes(b"\xff" * 16) == -1 160 | 161 | 162 | 163 | def test_bool(): 164 | assert BoolT.encode(True) == b"\1" 165 | assert BoolT.encode(False) == b"\0" 166 | assert BoolT.decode_bytes(b"\1") == True 167 | assert BoolT.decode_bytes(b"\0") == False 168 | with pytest.raises(TypeError): 169 | BoolT.decode_bytes("\x02") 170 | 171 | 172 | def test_array(): 173 | arrt = ArrayT(Uint8, 2) 174 | assert arrt.encode([1, 2]) == b'\x02\x01\x02' 175 | arr = arrt.decode(Cursor(b'\x02\x01\x02')) 176 | assert arr == [1, 2] 177 | with pytest.raises(TypeError): 178 | arrt.decode(Cursor(b'\x01\x01\x02')) 179 | with pytest.raises(TypeError): 180 | arrt.decode(Cursor(b'\x03\x01\x02')) 181 | 182 | def test_deserialize_int_array(): 183 | arrt = ArrayT(BoolT, 2) 184 | bools = arrt.decode(Cursor([2,1,0])) 185 | assert bools == [True, False] 186 | 187 | def test_tuple(): 188 | tuplet = TupleT(StrT, Uint8, BoolT) 189 | assert tuplet.encode(("abc", 1, False)) == b'\x03\x61\x62\x63\x01\x00' 190 | ret = tuplet.decode(Cursor(b'\x03\x61\x62\x63\x01\x00')) 191 | assert ret == ("abc", 1, False) 192 | 193 | def test_type_mapping(): 194 | atype = {Uint16: Uint64} 195 | mtype = type_mapping(atype) 196 | print(mtype) 197 | -------------------------------------------------------------------------------- /test/test_util.py: -------------------------------------------------------------------------------- 1 | from canoser.util import * 2 | import pdb 3 | 4 | def int_list_to_bytes_v2(ints): 5 | return struct.pack("<{}B".format(len(ints)), *ints) 6 | 7 | def test_address(): 8 | hex_a = "000000000000000000000000000000000000000000000000000000000a550c18" 9 | int_a = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 85, 12, 24] 10 | bytes_a = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\nU\x0c\x18' 11 | assert hex_a.encode() == b"000000000000000000000000000000000000000000000000000000000a550c18" 12 | assert bytes.fromhex(hex_a) == bytes_a 13 | assert bytes(int_a) == bytes_a 14 | assert int_list_to_hex(int_a) == hex_a 15 | assert bytes_a.hex() == hex_a 16 | assert bytes_to_int_list(bytes_a) == int_a 17 | assert hex_to_int_list(hex_a) == int_a 18 | assert int_list_to_bytes_v2(int_a) == int_list_to_bytes(int_a) -------------------------------------------------------------------------------- /venv.sh: -------------------------------------------------------------------------------- 1 | #python3 -m venv /mnt/d/PYENV/canoser 2 | source "/mnt/d/PYENV/canoser/bin/activate" 3 | # run with source ./venv.sh 4 | --------------------------------------------------------------------------------