├── .gitignore ├── LICENSE ├── README.md ├── bench ├── benchmark.nim └── generator.nim ├── packedjson.nim ├── packedjson.nimble └── packedjson └── deserialiser.nim /.gitignore: -------------------------------------------------------------------------------- 1 | nimcache/ 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Andreas Rumpf 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # packedjson 2 | packedjson is an alternative Nim implementation for JSON. 3 | The JSON is essentially kept as a single string in order to 4 | save memory over a more traditional tree representation. 5 | 6 | The API is mostly compatible with the stdlib's ``json.nim`` module, 7 | some features have been cut though. 8 | 9 | # To compile the benchmark, run these commands: 10 | 11 | ``` 12 | nim c -r bench\generator 13 | 14 | nim c -r -d:release bench\benchmark.nim 15 | nim c -r -d:release -d:useStdlib benchmark.nim 16 | ``` 17 | 18 | On my machine, I got these results: 19 | 20 | ``` 21 | packed json: used Mem: 94.06MiB time: 2.622s 22 | stdlib json: used Mem: 1.277GiB time: 3.759s 23 | ``` 24 | 25 | packedjson is now being used in production and seems to be reasonably stable. 26 | -------------------------------------------------------------------------------- /bench/benchmark.nim: -------------------------------------------------------------------------------- 1 | ## Benchmark packedjson again stdlib's json: 2 | 3 | when defined(useStdlib): 4 | import json 5 | else: 6 | import ".." / packedjson 7 | import strutils, times 8 | 9 | proc main = 10 | let jobj = parseFile("1.json") 11 | 12 | let coordinates = jobj["coordinates"] 13 | let len = float(coordinates.len) 14 | doAssert coordinates.len == 1000000 15 | var x = 0.0 16 | var y = 0.0 17 | var z = 0.0 18 | 19 | for coord in coordinates: 20 | x += coord["x"].getFloat 21 | y += coord["y"].getFloat 22 | z += coord["z"].getFloat 23 | 24 | echo x / len 25 | echo y / len 26 | echo z / len 27 | 28 | let start = cpuTime() 29 | main() 30 | echo "used Mem: ", formatSize getOccupiedMem(), " time: ", cpuTime() - start, "s" 31 | 32 | #[ 33 | Results on my machine: 34 | 35 | packed json: used Mem: 94.06MiB time: 2.622s 36 | old version: used Mem: 140.063MiB time: 3.127s 37 | stdlib json: used Mem: 1.277GiB time: 3.759s 38 | 39 | ]# 40 | -------------------------------------------------------------------------------- /bench/generator.nim: -------------------------------------------------------------------------------- 1 | # Ported the script from Ruby to Nim. 2 | # Original here: https://github.com/kostya/benchmarks/blob/master/json/generate_json.rb 3 | 4 | import json, random, strutils, sequtils 5 | 6 | var x = %[] 7 | 8 | for _ in 0 ..< 1_000_000: 9 | var alpha = toSeq('a'..'z') 10 | shuffle(alpha) 11 | let h = %*{ 12 | "x": rand(1.0), 13 | "y": rand(1.0), 14 | "z": rand(1.0), 15 | "name": alpha[0..4].join & ' ' & $rand(10000), 16 | "opts": {"1": [1, true]}, 17 | } 18 | x.add h 19 | 20 | writeFile("1.json", pretty(%*{"coordinates": x, "info": "some info"})) 21 | -------------------------------------------------------------------------------- /packedjson.nim: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # Nim's Runtime Library 4 | # (c) Copyright 2018 Nim contributors 5 | # 6 | # See the file "copying.txt", included in this 7 | # distribution, for details about the copyright. 8 | # 9 | 10 | ## Packedjson is an alternative JSON tree implementation that takes up far 11 | ## less space than Nim's stdlib JSON implementation. The space savings can be as much 12 | ## as 80%. It can be faster or much slower than the stdlib's JSON, depending on the 13 | ## workload. 14 | 15 | ## **Note**: This library distinguishes between `JsonTree` and `JsonNode` 16 | ## types. Only `JsonTree` can be mutated and accessors like `[]` return a 17 | ## `JsonNode` which is merely an immutable view into a `JsonTree`. This 18 | ## prevents most forms of unsupported aliasing operations like: 19 | ## 20 | ## .. code-block:: nim 21 | ## var arr = newJArray() 22 | ## arr.add newJObject() 23 | ## var x = arr[0] 24 | ## # Error: 'x' is of type JsonNode and cannot be mutated: 25 | ## x["field"] = %"value" 26 | ## 27 | ## (You can use the `copy` operation to create an explicit copy that then 28 | ## can be mutated.) 29 | ## 30 | ## A `JsonTree` that is added to another `JsonTree` gets copied: 31 | ## 32 | ## .. code-block:: nim 33 | ## var x = newJObject() 34 | ## var arr = newJArray() 35 | ## arr.add x 36 | ## x["field"] = %"value" 37 | ## assert $arr == "[{}]" 38 | ## 39 | ## These semantics also imply that code like `myobj["field"]["nested"] = %4` 40 | ## needs instead be written as `myobj["field", "nested"] = %4` so that the 41 | ## changes end up in the original tree. 42 | ## 43 | ## Some simple rules for efficiently using these types are: 44 | ## * Variables that can be mutated are of type `JsonTree`. Disallow mutations 45 | ## by using `JsonNode`. 46 | ## * Have every return/output parameter type be `JsonTree`. Make sure that you 47 | ## return a copy or a fresh `JsonTree`. 48 | ## * Input parameters should be of type `JsonNode`. Use `copy` to get a value 49 | ## that can be mutated, if needed. 50 | ## * Remove unnecessary copies by casting to `JsonTree` when you are sure about 51 | ## it, i.e. a `JsonNode` obtained from a `JsonTree` another function returned. 52 | 53 | import parsejson, parseutils, streams, strutils, macros, hashes 54 | 55 | import std / varints 56 | export JsonParsingError, JsonKindError 57 | 58 | type 59 | JsonNodeKind* = enum ## possible JSON node types 60 | JNull, 61 | JBool, 62 | JInt, 63 | JFloat, 64 | JString, 65 | JObject, 66 | JArray 67 | 68 | # An atom is always prefixed with its kind and for JString, JFloat, JInt also 69 | # with a length information. The length information is either part of the first byte 70 | # or a variable length integer. An atom is everything 71 | # that is not an object or an array. Since we don't know the elements in an array 72 | # or object upfront during parsing, these are not stored with any length 73 | # information. Instead the 'opcodeEnd' marker is produced. This means that during 74 | # traversal we have to be careful of the nesting and always jump over the payload 75 | # of an atom. However, this means we can save key names unescaped which speeds up 76 | # most operations. 77 | 78 | const 79 | opcodeBits = 3 80 | 81 | opcodeNull = ord JNull 82 | opcodeBool = ord JBool 83 | opcodeFalse = opcodeBool 84 | opcodeTrue = opcodeBool or 0b0000_1000 85 | opcodeInt = ord JInt 86 | opcodeFloat = ord JFloat 87 | opcodeString = ord JString 88 | opcodeObject = ord JObject 89 | opcodeArray = ord JArray 90 | opcodeEnd = 7 91 | 92 | opcodeMask = 0b111 93 | 94 | proc storeAtom(buf: var seq[byte]; k: JsonNodeKind; data: string) = 95 | if data.len < 0b1_1111: 96 | # we can store the length in the upper bits of the kind field. 97 | buf.add byte(data.len shl 3) or byte(k) 98 | else: 99 | # we need to store the kind field and the length separately: 100 | buf.add 0b1111_1000u8 or byte(k) 101 | # ensure we have the space: 102 | let start = buf.len 103 | buf.setLen start + maxVarIntLen 104 | let realVlen = writeVu64(toOpenArray(buf, start, start + maxVarIntLen - 1), uint64 data.len) 105 | buf.setLen start + realVlen 106 | for i in 0..high(data): 107 | buf.add byte(data[i]) 108 | 109 | proc beginContainer(buf: var seq[byte]; k: JsonNodeKind) = 110 | buf.add byte(k) 111 | 112 | proc endContainer(buf: var seq[byte]) = buf.add byte(opcodeEnd) 113 | 114 | type 115 | JsonNode* = object 116 | k: JsonNodeKind 117 | a, b: int 118 | t: ref seq[byte] 119 | 120 | JsonTree* = distinct JsonNode ## a JsonTree is a JsonNode that can be mutated. 121 | 122 | converter toJsonNode*(x: JsonTree): JsonNode {.inline.} = JsonNode(x) 123 | 124 | proc newJNull*(): JsonNode = 125 | ## Creates a new `JNull JsonNode`. 126 | result.k = JNull 127 | 128 | template newBody(kind, x) = 129 | new(result.t) 130 | result.t[] = @[] 131 | storeAtom(result.t[], kind, x) 132 | result.a = 0 133 | result.b = high(result.t[]) 134 | result.k = kind 135 | 136 | proc newJString*(s: string): JsonNode = 137 | ## Creates a new `JString JsonNode`. 138 | newBody JString, s 139 | 140 | proc newJInt*(n: BiggestInt): JsonNode = 141 | ## Creates a new `JInt JsonNode`. 142 | newBody JInt, $n 143 | 144 | proc newJFloat*(n: float): JsonNode = 145 | ## Creates a new `JFloat JsonNode`. 146 | newBody JFloat, formatFloat(n) 147 | 148 | proc newJBool*(b: bool): JsonNode = 149 | ## Creates a new `JBool JsonNode`. 150 | result.k = JBool 151 | new(result.t) 152 | result.t[] = @[if b: byte(opcodeTrue) else: byte(opcodeFalse)] 153 | result.a = 0 154 | result.b = high(result.t[]) 155 | 156 | proc newJObject*(): JsonTree = 157 | ## Creates a new `JObject JsonNode` 158 | JsonNode(result).k = JObject 159 | new(JsonNode(result).t) 160 | JsonNode(result).t[] = @[byte opcodeObject, byte opcodeEnd] 161 | JsonNode(result).a = 0 162 | JsonNode(result).b = high(JsonNode(result).t[]) 163 | 164 | proc newJArray*(): JsonTree = 165 | ## Creates a new `JArray JsonNode` 166 | JsonNode(result).k = JArray 167 | new(JsonNode(result).t) 168 | JsonNode(result).t[] = @[byte opcodeArray, byte opcodeEnd] 169 | JsonNode(result).a = 0 170 | JsonNode(result).b = high(JsonNode(result).t[]) 171 | 172 | proc kind*(x: JsonNode): JsonNodeKind = x.k 173 | 174 | proc extractLen(x: seq[byte]; pos: int): int = 175 | if (x[pos] and 0b1111_1000u8) != 0b1111_1000u8: 176 | result = int(x[pos]) shr 3 177 | else: 178 | # we had an overflow for inline length information, 179 | # so extract the variable sized integer: 180 | var varint: uint64 181 | let varintLen = readVu64(toOpenArray(x, pos+1, min(pos + 1 + maxVarIntLen, x.high)), varint) 182 | result = int(varint) + varintLen 183 | 184 | proc extractSlice(x: seq[byte]; pos: int): (int, int) = 185 | if (x[pos] and 0b1111_1000u8) != 0b1111_1000u8: 186 | result = (pos + 1, int(x[pos]) shr 3) 187 | else: 188 | # we had an overflow for inline length information, 189 | # so extract the variable sized integer: 190 | var varint: uint64 191 | let varintLen = readVu64(toOpenArray(x, pos+1, min(pos + 1 + maxVarIntLen, x.high)), varint) 192 | result = (pos + 1 + varintLen, int(varint)) 193 | 194 | proc skip(x: seq[byte]; start: int; elements: var int): int = 195 | var nested = 0 196 | var pos = start 197 | while true: 198 | let k = x[pos] and opcodeMask 199 | var nextPos = pos + 1 200 | case k 201 | of opcodeNull, opcodeBool: 202 | if nested == 0: inc elements 203 | of opcodeInt, opcodeFloat, opcodeString: 204 | let L = extractLen(x, pos) 205 | nextPos = pos + 1 + L 206 | if nested == 0: inc elements 207 | of opcodeObject, opcodeArray: 208 | if nested == 0: inc elements 209 | inc nested 210 | of opcodeEnd: 211 | if nested == 0: return nextPos 212 | dec nested 213 | else: discard 214 | pos = nextPos 215 | 216 | iterator items*(x: JsonNode): JsonNode = 217 | ## Iterator for the items of `x`. `x` has to be a JArray. 218 | assert x.kind == JArray 219 | var pos = x.a+1 220 | var dummy: int 221 | while pos <= x.b: 222 | let k = x.t[pos] and opcodeMask 223 | var nextPos = pos + 1 224 | case k 225 | of opcodeNull, opcodeBool: discard 226 | of opcodeInt, opcodeFloat, opcodeString: 227 | let L = extractLen(x.t[], pos) 228 | nextPos = pos + 1 + L 229 | of opcodeObject, opcodeArray: 230 | nextPos = skip(x.t[], pos+1, dummy) 231 | of opcodeEnd: break 232 | else: discard 233 | yield JsonNode(k: JsonNodeKind(k), a: pos, b: nextPos-1, t: x.t) 234 | pos = nextPos 235 | 236 | iterator pairs*(x: JsonNode): (string, JsonNode) = 237 | ## Iterator for the pairs of `x`. `x` has to be a JObject. 238 | assert x.kind == JObject 239 | var pos = x.a+1 240 | var dummy: int 241 | var key = newStringOfCap(60) 242 | while pos <= x.b: 243 | let k2 = x.t[pos] and opcodeMask 244 | if k2 == opcodeEnd: break 245 | 246 | assert k2 == opcodeString, $k2 247 | let (start, L) = extractSlice(x.t[], pos) 248 | key.setLen L 249 | for i in 0 ..< L: key[i] = char(x.t[start+i]) 250 | pos = start + L 251 | 252 | let k = x.t[pos] and opcodeMask 253 | var nextPos = pos + 1 254 | case k 255 | of opcodeNull, opcodeBool: discard 256 | of opcodeInt, opcodeFloat, opcodeString: 257 | let L = extractLen(x.t[], pos) 258 | nextPos = pos + 1 + L 259 | of opcodeObject, opcodeArray: 260 | nextPos = skip(x.t[], pos+1, dummy) 261 | of opcodeEnd: doAssert false, "unexpected end of object" 262 | else: discard 263 | yield (key, JsonNode(k: JsonNodeKind(k), a: pos, b: nextPos-1, t: x.t)) 264 | pos = nextPos 265 | 266 | proc rawGet(x: JsonNode; name: string): JsonNode = 267 | assert x.kind == JObject 268 | var pos = x.a+1 269 | var dummy: int 270 | while pos <= x.b: 271 | let k2 = x.t[pos] and opcodeMask 272 | if k2 == opcodeEnd: break 273 | 274 | assert k2 == opcodeString, $k2 275 | let (start, L) = extractSlice(x.t[], pos) 276 | # compare for the key without creating the temp string: 277 | var isMatch = name.len == L 278 | if isMatch: 279 | for i in 0 ..< L: 280 | if name[i] != char(x.t[start+i]): 281 | isMatch = false 282 | break 283 | pos = start + L 284 | 285 | let k = x.t[pos] and opcodeMask 286 | var nextPos = pos + 1 287 | case k 288 | of opcodeNull, opcodeBool: discard 289 | of opcodeInt, opcodeFloat, opcodeString: 290 | let L = extractLen(x.t[], pos) 291 | nextPos = pos + 1 + L 292 | of opcodeObject, opcodeArray: 293 | nextPos = skip(x.t[], pos+1, dummy) 294 | of opcodeEnd: doAssert false, "unexpected end of object" 295 | else: discard 296 | if isMatch: 297 | return JsonNode(k: JsonNodeKind(k), a: pos, b: nextPos-1, t: x.t) 298 | pos = nextPos 299 | result.a = -1 300 | 301 | proc `[]`*(x: JsonNode; name: string): JsonNode = 302 | ## Gets a field from a `JObject`. 303 | ## If the value at `name` does not exist, raises KeyError. 304 | result = rawGet(x, name) 305 | if result.a < 0: 306 | raise newException(KeyError, "key not found in object: " & name) 307 | 308 | proc len*(n: JsonNode): int = 309 | ## If `n` is a `JArray`, it returns the number of elements. 310 | ## If `n` is a `JObject`, it returns the number of pairs. 311 | ## Else it returns 0. 312 | if n.k notin {JArray, JObject}: return 0 313 | discard skip(n.t[], n.a+1, result) 314 | # divide by two because we counted the pairs wrongly: 315 | if n.k == JObject: result = result shr 1 316 | 317 | proc rawAdd(obj: var JsonNode; child: seq[byte]; a, b: int) = 318 | let pa = obj.b 319 | let L = b - a + 1 320 | let oldfull = obj.t[].len 321 | setLen(obj.t[], oldfull+L) 322 | # now move the tail to the new end so that we can insert effectively 323 | # into the middle: 324 | for i in countdown(oldfull+L-1, pa+L): 325 | when defined(gcArc) or defined(gcOrc): 326 | obj.t[][i] = move obj.t[][i-L] 327 | else: 328 | shallowCopy(obj.t[][i], obj.t[][i-L]) 329 | # insert into the middle: 330 | for i in 0 ..< L: 331 | obj.t[][pa + i] = child[a + i] 332 | inc obj.b, L 333 | 334 | proc rawAddWithNull(parent: var JsonNode; child: JsonNode) = 335 | if child.k == JNull: 336 | let pa = parent.b 337 | let oldLen = parent.t[].len 338 | setLen(parent.t[], oldLen + 1) 339 | for i in pa .. oldLen-1: 340 | parent.t[][1 + i] = parent.t[][i] 341 | parent.t[][pa] = byte opcodeNull 342 | inc parent.b, 1 343 | else: 344 | rawAdd(parent, child.t[], child.a, child.b) 345 | 346 | proc add*(parent: var JsonTree; child: JsonNode) = 347 | doAssert parent.kind == JArray, "parent is not a JArray" 348 | rawAddWithNull(JsonNode(parent), child) 349 | 350 | proc add*(obj: var JsonTree, key: string, val: JsonNode) = 351 | ## Sets a field from a `JObject`. **Warning**: It is currently not checked 352 | ## but assumed that the object does not yet have a field named `key`. 353 | assert obj.kind == JObject 354 | let k = newJString(key) 355 | # XXX optimize this further! 356 | rawAdd(JsonNode obj, k.t[], 0, high(k.t[])) 357 | rawAddWithNull(JsonNode obj, val) 358 | when false: 359 | discard "XXX assert that the key does not exist yet" 360 | 361 | proc rawPut(obj: var JsonNode, oldval: JsonNode, key: string, val: JsonNode): int = 362 | let oldlen = oldval.b - oldval.a + 1 363 | let newlen = val.b - val.a + 1 364 | result = newlen - oldlen 365 | if result == 0: 366 | for i in 0 ..< newlen: 367 | obj.t[][oldval.a + i] = (if val.k == JNull: byte opcodeNull else: val.t[][i]) 368 | else: 369 | let oldfull = obj.t[].len 370 | if newlen > oldlen: 371 | setLen(obj.t[], oldfull+result) 372 | # now move the tail to the new end so that we can insert effectively 373 | # into the middle: 374 | for i in countdown(oldfull+result-1, oldval.a+newlen): 375 | when defined(gcArc) or defined(gcOrc): 376 | obj.t[][i] = move obj.t[][i-result] 377 | else: 378 | shallowCopy(obj.t[][i], obj.t[][i-result]) 379 | else: 380 | for i in countup(oldval.a+newlen, oldfull+result-1): 381 | when defined(gcArc) or defined(gcOrc): 382 | obj.t[][i] = move obj.t[][i-result] 383 | else: 384 | shallowCopy(obj.t[][i], obj.t[][i-result]) 385 | # cut down: 386 | setLen(obj.t[], oldfull+result) 387 | # overwrite old value: 388 | for i in 0 ..< newlen: 389 | obj.t[][oldval.a + i] = (if val.k == JNull: byte opcodeNull else: val.t[][i]) 390 | 391 | proc `[]=`*(obj: var JsonTree, key: string, val: JsonNode) = 392 | let oldval = rawGet(obj, key) 393 | if oldval.a < 0: 394 | add(obj, key, val) 395 | else: 396 | let diff = rawPut(JsonNode obj, oldval, key, val) 397 | inc JsonNode(obj).b, diff 398 | 399 | macro `[]=`*(obj: var JsonTree, keys: varargs[typed], val: JsonNode): untyped = 400 | ## keys can be strings or integers for the navigation. 401 | result = newStmtList() 402 | template t0(obj, key) {.dirty.} = 403 | var oldval = obj[key] 404 | 405 | template ti(key) {.dirty.} = 406 | oldval = oldval[key] 407 | 408 | template tput(obj, finalkey, val) = 409 | let diff = rawPut(JsonNode obj, oldval, finalkey, val) 410 | inc JsonNode(obj).b, diff 411 | 412 | result.add getAst(t0(obj, keys[0])) 413 | for i in 1..= uint64(high(BiggestInt))+1: 600 | result = low(BiggestInt) 601 | else: 602 | result = -BiggestInt(res) 603 | else: 604 | if res >= uint64(high(BiggestInt)): 605 | result = high(BiggestInt) 606 | else: 607 | result = BiggestInt(res) 608 | 609 | proc getInt*(n: JsonNode, default: int = 0): int = 610 | ## Retrieves the int value of a `JInt JsonNode`. 611 | ## 612 | ## Returns ``default`` if ``n`` is not a ``JInt``, or if ``n`` is nil. 613 | if n.kind != JInt: return default 614 | let (start, L) = extractSlice(n.t[], n.a) 615 | result = int(myParseInt(n.t[], start, start + L - 1)) 616 | 617 | proc getBiggestInt*(n: JsonNode, default: BiggestInt = 0): BiggestInt = 618 | ## Retrieves the BiggestInt value of a `JInt JsonNode`. 619 | ## 620 | ## Returns ``default`` if ``n`` is not a ``JInt``, or if ``n`` is nil. 621 | if n.kind != JInt: return default 622 | let (start, L) = extractSlice(n.t[], n.a) 623 | result = myParseInt(n.t[], start, start + L - 1) 624 | 625 | proc getFloat*(n: JsonNode, default: float = 0.0): float = 626 | ## Retrieves the float value of a `JFloat JsonNode`. 627 | ## 628 | ## Returns ``default`` if ``n`` is not a ``JFloat`` or ``JInt``, or if ``n`` is nil. 629 | case n.kind 630 | of JFloat, JInt: 631 | let (start, L) = extractSlice(n.t[], n.a) 632 | if parseFloat(cast[string](n.t[]), result, start) != L: 633 | # little hack ahead: If parsing failed because of following control bytes, 634 | # patch the control byte, do the parsing and patch the control byte back: 635 | let old = n.t[][start+L] 636 | n.t[][start+L] = 0 637 | doAssert parseFloat(cast[string](n.t[]), result, start) == L 638 | n.t[][start+L] = old 639 | else: 640 | result = default 641 | 642 | proc getBool*(n: JsonNode, default: bool = false): bool = 643 | ## Retrieves the bool value of a `JBool JsonNode`. 644 | ## 645 | ## Returns ``default`` if ``n`` is not a ``JBool``, or if ``n`` is nil. 646 | if n.kind == JBool: result = (n.t[n.a] shr opcodeBits) == 1 647 | else: result = default 648 | 649 | proc isEmpty(n: JsonNode): bool = 650 | assert n.kind in {JArray, JObject} 651 | result = n.t[n.a+1] == opcodeEnd 652 | 653 | template escape(result, c) = 654 | case c 655 | of '\L': result.add("\\n") 656 | of '\b': result.add("\\b") 657 | of '\f': result.add("\\f") 658 | of '\t': result.add("\\t") 659 | of '\r': result.add("\\r") 660 | of '"': result.add("\\\"") 661 | of '\\': result.add("\\\\") 662 | else: result.add(c) 663 | 664 | proc escapeJson*(s: string; result: var string) = 665 | ## Converts a string `s` to its JSON representation. 666 | ## Appends to ``result``. 667 | result.add("\"") 668 | for c in s: escape(result, c) 669 | result.add("\"") 670 | 671 | proc escapeJson*(s: string): string = 672 | ## Converts a string `s` to its JSON representation. 673 | result = newStringOfCap(s.len + s.len shr 3) 674 | escapeJson(s, result) 675 | 676 | proc indent(s: var string, i: int) = 677 | for _ in 1..i: s.add ' ' 678 | 679 | proc newIndent(curr, indent: int, ml: bool): int = 680 | if ml: return curr + indent 681 | else: return indent 682 | 683 | proc nl(s: var string, ml: bool) = 684 | s.add(if ml: '\L' else: ' ') 685 | 686 | proc emitAtom(result: var string, n: JsonNode) = 687 | let (start, L) = extractSlice(n.t[], n.a) 688 | if n.k == JString: result.add("\"") 689 | for i in 0 ..< L: 690 | let c = char(n.t[start+i]) 691 | escape(result, c) 692 | if n.k == JString: result.add("\"") 693 | 694 | proc toPretty(result: var string, n: JsonNode, indent = 2, ml = true, 695 | lstArr = false, currIndent = 0) = 696 | case n.kind 697 | of JObject: 698 | if lstArr: result.indent(currIndent) # Indentation 699 | if not n.isEmpty: 700 | result.add("{") 701 | result.nl(ml) # New line 702 | var i = 0 703 | for key, val in pairs(n): 704 | if i > 0: 705 | result.add(",") 706 | result.nl(ml) # New Line 707 | inc i 708 | # Need to indent more than { 709 | result.indent(newIndent(currIndent, indent, ml)) 710 | escapeJson(key, result) 711 | result.add(": ") 712 | toPretty(result, val, indent, ml, false, 713 | newIndent(currIndent, indent, ml)) 714 | result.nl(ml) 715 | result.indent(currIndent) # indent the same as { 716 | result.add("}") 717 | else: 718 | result.add("{}") 719 | of JString, JInt, JFloat: 720 | if lstArr: result.indent(currIndent) 721 | emitAtom(result, n) 722 | of JBool: 723 | if lstArr: result.indent(currIndent) 724 | result.add(if n.getBool: "true" else: "false") 725 | of JArray: 726 | if lstArr: result.indent(currIndent) 727 | if not n.isEmpty: 728 | result.add("[") 729 | result.nl(ml) 730 | var i = 0 731 | for x in items(n): 732 | if i > 0: 733 | result.add(",") 734 | result.nl(ml) # New Line 735 | toPretty(result, x, indent, ml, 736 | true, newIndent(currIndent, indent, ml)) 737 | inc i 738 | result.nl(ml) 739 | result.indent(currIndent) 740 | result.add("]") 741 | else: result.add("[]") 742 | of JNull: 743 | if lstArr: result.indent(currIndent) 744 | result.add("null") 745 | 746 | proc pretty*(node: JsonNode, indent = 2): string = 747 | ## Returns a JSON Representation of `node`, with indentation and 748 | ## on multiple lines. 749 | result = "" 750 | toPretty(result, node, indent) 751 | 752 | proc toUgly*(result: var string, node: JsonNode) = 753 | ## Converts `node` to its JSON Representation, without 754 | ## regard for human readability. Meant to improve ``$`` string 755 | ## conversion performance. 756 | ## 757 | ## JSON representation is stored in the passed `result` 758 | ## 759 | ## This provides higher efficiency than the ``pretty`` procedure as it 760 | ## does **not** attempt to format the resulting JSON to make it human readable. 761 | var comma = false 762 | case node.kind: 763 | of JArray: 764 | result.add "[" 765 | for child in node: 766 | if comma: result.add "," 767 | else: comma = true 768 | result.toUgly child 769 | result.add "]" 770 | of JObject: 771 | result.add "{" 772 | for key, value in pairs(node): 773 | if comma: result.add "," 774 | else: comma = true 775 | key.escapeJson(result) 776 | result.add ":" 777 | result.toUgly value 778 | result.add "}" 779 | of JString, JInt, JFloat: 780 | emitAtom(result, node) 781 | of JBool: 782 | result.add(if node.getBool: "true" else: "false") 783 | of JNull: 784 | result.add "null" 785 | 786 | proc `$`*(node: JsonNode): string = 787 | ## Converts `node` to its JSON Representation on one line. 788 | result = newStringOfCap(node.len shl 1) 789 | toUgly(result, node) 790 | 791 | proc `[]`*(node: JsonNode, index: int): JsonNode = 792 | ## Gets the node at `index` in an Array. Result is undefined if `index` 793 | ## is out of bounds, but as long as array bound checks are enabled it will 794 | ## result in an exception. 795 | assert(node.kind == JArray) 796 | var i = index 797 | for x in items(node): 798 | if i == 0: return x 799 | dec i 800 | raise newException(IndexDefect, "index out of bounds") 801 | 802 | proc contains*(node: JsonNode, key: string): bool = 803 | ## Checks if `key` exists in `node`. 804 | assert(node.kind == JObject) 805 | let x = rawGet(node, key) 806 | result = x.a >= 0 807 | 808 | proc hasKey*(node: JsonNode, key: string): bool = 809 | ## Checks if `key` exists in `node`. 810 | assert(node.kind == JObject) 811 | result = node.contains(key) 812 | 813 | proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode = 814 | ## Traverses the node and gets the given value. If any of the 815 | ## keys do not exist, returns ``JNull``. Also returns ``JNull`` if one of the 816 | ## intermediate data structures is not an object. 817 | result = node 818 | for kk in keys: 819 | if result.kind != JObject: return newJNull() 820 | block searchLoop: 821 | for k, v in pairs(result): 822 | if k == kk: 823 | result = v 824 | break searchLoop 825 | return newJNull() 826 | 827 | proc `{}`*(node: JsonNode, indexes: varargs[int]): JsonNode = 828 | ## Traverses the node and gets the given value. If any of the 829 | ## indexes do not exist, returns ``JNull``. Also returns ``JNull`` if one of the 830 | ## intermediate data structures is not an array. 831 | result = node 832 | for j in indexes: 833 | if result.kind != JArray: return newJNull() 834 | block searchLoop: 835 | var i = j 836 | for x in items(result): 837 | if i == 0: 838 | result = x 839 | break searchLoop 840 | dec i 841 | return newJNull() 842 | 843 | proc `{}=`*(node: var JsonTree, keys: varargs[string], value: JsonNode) = 844 | ## Traverses the node and tries to set the value at the given location 845 | ## to ``value``. If any of the keys are missing, they are added. 846 | if keys.len == 1: 847 | node[keys[0]] = value 848 | elif keys.len != 0: 849 | var v = value 850 | for i in countdown(keys.len-1, 1): 851 | var x = newJObject() 852 | x[keys[i]] = v 853 | v = x 854 | node[keys[0]] = v 855 | 856 | proc getOrDefault*(node: JsonNode, key: string): JsonNode = 857 | ## Gets a field from a `node`. If `node` is nil or not an object or 858 | ## value at `key` does not exist, returns JNull 859 | for k, v in pairs(node): 860 | if k == key: return v 861 | result = newJNull() 862 | 863 | proc `==`*(x, y: JsonNode): bool = 864 | # Equality for two JsonNodes. Note that the order in field 865 | # declarations is also part of the equality requirement as 866 | # everything else would be too costly to implement. 867 | if x.k != y.k: return false 868 | if x.k == JNull: return true 869 | if x.b - x.a != y.b - y.a: return false 870 | for i in 0 ..< x.b - x.a + 1: 871 | if x.t[][x.a + i] != y.t[][i + y.a]: return false 872 | return true 873 | 874 | proc parseJson(p: var JsonParser; buf: var seq[byte]) = 875 | ## Parses JSON from a JSON Parser `p`. We break the abstraction here 876 | ## and construct the low level representation directly for speed. 877 | case p.tok 878 | of tkString: 879 | storeAtom(buf, JString, p.a) 880 | discard getTok(p) 881 | of tkInt: 882 | storeAtom(buf, JInt, p.a) 883 | discard getTok(p) 884 | of tkFloat: 885 | storeAtom(buf, JFloat, p.a) 886 | discard getTok(p) 887 | of tkTrue: 888 | buf.add opcodeTrue 889 | discard getTok(p) 890 | of tkFalse: 891 | buf.add opcodeFalse 892 | discard getTok(p) 893 | of tkNull: 894 | buf.add opcodeNull 895 | discard getTok(p) 896 | of tkCurlyLe: 897 | beginContainer(buf, JObject) 898 | discard getTok(p) 899 | while p.tok != tkCurlyRi: 900 | if p.tok != tkString: 901 | raiseParseErr(p, "string literal as key") 902 | storeAtom(buf, JString, p.a) 903 | discard getTok(p) 904 | eat(p, tkColon) 905 | parseJson(p, buf) 906 | if p.tok != tkComma: break 907 | discard getTok(p) 908 | eat(p, tkCurlyRi) 909 | endContainer(buf) 910 | of tkBracketLe: 911 | beginContainer(buf, JArray) 912 | discard getTok(p) 913 | while p.tok != tkBracketRi: 914 | parseJson(p, buf) 915 | if p.tok != tkComma: break 916 | discard getTok(p) 917 | eat(p, tkBracketRi) 918 | endContainer(buf) 919 | of tkError, tkCurlyRi, tkBracketRi, tkColon, tkComma, tkEof: 920 | raiseParseErr(p, "{") 921 | 922 | proc parseJson*(s: Stream, filename: string = ""): JsonTree = 923 | ## Parses from a stream `s` into a `JsonNode`. `filename` is only needed 924 | ## for nice error messages. 925 | ## If `s` contains extra data, it will raise `JsonParsingError`. 926 | var p: JsonParser 927 | p.open(s, filename) 928 | new JsonNode(result).t 929 | JsonNode(result).t[] = newSeqOfCap[byte](64) 930 | try: 931 | discard getTok(p) # read first token 932 | p.parseJson(JsonNode(result).t[]) 933 | JsonNode(result).a = 0 934 | JsonNode(result).b = high(JsonNode(result).t[]) 935 | JsonNode(result).k = JsonNodeKind(JsonNode(result).t[][0] and opcodeMask) 936 | eat(p, tkEof) # check if there is no extra data 937 | finally: 938 | p.close() 939 | 940 | proc parseJson*(buffer: string): JsonTree = 941 | ## Parses JSON from `buffer`. 942 | ## If `buffer` contains extra data, it will raise `JsonParsingError`. 943 | result = parseJson(newStringStream(buffer), "input") 944 | 945 | proc parseFile*(filename: string): JsonTree = 946 | ## Parses `file` into a `JsonNode`. 947 | ## If `file` contains extra data, it will raise `JsonParsingError`. 948 | var stream = newFileStream(filename, fmRead) 949 | if stream == nil: 950 | raise newException(IOError, "cannot read from file: " & filename) 951 | result = parseJson(stream, filename) 952 | 953 | proc hash*(x: JsonNode): Hash {.noSideEffect.} = 954 | result = hash(x.t[], x.a, x.b) 955 | 956 | when isMainModule: 957 | when false: 958 | var b: seq[byte] = @[] 959 | storeAtom(b, JString, readFile("packedjson.nim")) 960 | let (start, L) = extractSlice(b, 0) 961 | var result = newString(L) 962 | for i in 0 ..< L: 963 | result[i] = char(b[start+i]) 964 | echo result 965 | 966 | template test(a, b) = 967 | let x = a 968 | if x != b: 969 | echo "test failed ", astToStr(a), ":" 970 | echo x 971 | echo b 972 | 973 | let testJson = parseJson"""{ "a": [1, 2, 3, 4], "b": "asd", "c": "\ud83c\udf83", "d": "\u00E6"}""" 974 | test $testJson{"a"}[3], "4" 975 | 976 | var moreStuff = %*{"abc": 3, "more": 6.6, "null": nil} 977 | test $moreStuff, """{"abc":3,"more":6.600000000000000,"null":null}""" 978 | 979 | moreStuff["more"] = %"foo bar" 980 | test $moreStuff, """{"abc":3,"more":"foo bar","null":null}""" 981 | 982 | moreStuff["more"] = %"a" 983 | moreStuff["null"] = %678 984 | 985 | test $moreStuff, """{"abc":3,"more":"a","null":678}""" 986 | 987 | moreStuff.delete "more" 988 | test $moreStuff, """{"abc":3,"null":678}""" 989 | 990 | moreStuff{"keyOne", "keyTwo", "keyThree"} = %*{"abc": 3, "more": 6.6, "null": nil} 991 | 992 | test $moreStuff, """{"abc":3,"null":678,"keyOne":{"keyTwo":{"keyThree":{"abc":3,"more":6.600000000000000,"null":null}}}}""" 993 | 994 | moreStuff["alias"] = newJObject() 995 | when false: 996 | # now let's test aliasing works: 997 | var aa = moreStuff["alias"] 998 | 999 | aa["a"] = %1 1000 | aa["b"] = %3 1001 | 1002 | when true: 1003 | delete moreStuff, "keyOne" 1004 | test $moreStuff, """{"abc":3,"null":678,"alias":{}}""" 1005 | moreStuff["keyOne"] = %*{"keyTwo": 3} #, "more": 6.6, "null": nil} 1006 | #echo moreStuff 1007 | 1008 | moreStuff{"keyOne", "keyTwo", "keyThree"} = %*{"abc": 3, "more": 6.6, "null": nil} 1009 | moreStuff["keyOne", "keyTwo"] = %"ZZZZZ" 1010 | #echo moreStuff 1011 | 1012 | block: 1013 | var x = newJObject() 1014 | var arr = newJArray() 1015 | arr.add x 1016 | x["field"] = %"value" 1017 | assert $arr == "[{}]" 1018 | 1019 | block: 1020 | var x = newJObject() 1021 | x["field"] = %"value" 1022 | var arr = newJArray() 1023 | arr.add x 1024 | assert arr == %*[{"field":"value"}] 1025 | 1026 | when false: 1027 | var arr = newJArray() 1028 | arr.add newJObject() 1029 | var x = arr[0] 1030 | x["field"] = %"value" 1031 | assert $arr == """[{"field":"value"}]""" 1032 | 1033 | block: 1034 | var testJson = parseJson"""{ "a": [1, 2, {"key": [4, 5]}, 4]}""" 1035 | testJson["a", 2, "key"] = %10 1036 | test $testJson, """{"a":[1,2,{"key":10},4]}""" 1037 | 1038 | block: 1039 | var mjson = %*{"properties":{"subnet":"a","securitygroup":"b"}} 1040 | mjson["properties","subnet"] = %"" 1041 | mjson["properties","securitygroup"] = %"" 1042 | test $mjson, """{"properties":{"subnet":"","securitygroup":""}}""" 1043 | 1044 | block: 1045 | # bug #1 1046 | var msg = %*{ 1047 | "itemId":25, 1048 | "cityId":15, 1049 | "less": low(BiggestInt), 1050 | "more": high(BiggestInt), 1051 | "million": 1_000_000 1052 | } 1053 | var itemId = msg["itemId"].getInt 1054 | var cityId = msg["cityId"].getInt 1055 | assert itemId == 25 1056 | assert cityId == 15 1057 | doAssert msg["less"].getBiggestInt == low(BiggestInt) 1058 | doAssert msg["more"].getBiggestInt == high(BiggestInt) 1059 | doAssert msg["million"].getBiggestInt == 1_000_000 1060 | -------------------------------------------------------------------------------- /packedjson.nimble: -------------------------------------------------------------------------------- 1 | # Package 2 | 3 | version = "0.2.2" 4 | author = "Araq" 5 | description = "packedjson is an alternative Nim implementation for JSON. The JSON is essentially kept as a single string in order to save memory over a more traditional tree representation." 6 | license = "MIT" 7 | 8 | skipDirs = @["bench"] 9 | 10 | # Dependencies 11 | 12 | requires "nim >= 0.18.1" 13 | -------------------------------------------------------------------------------- /packedjson/deserialiser.nim: -------------------------------------------------------------------------------- 1 | import ".." / packedjson, macros, strutils, options, tables 2 | # -- Json deserialiser. -- 3 | 4 | template verifyJsonKind(node: JsonNode, kinds: set[JsonNodeKind], ast: string) = 5 | if node.kind == JNull: 6 | raise newException(KeyError, "key not found: " & ast) 7 | elif node.kind notin kinds: 8 | let msg = "Incorrect JSON kind. Wanted '$1' in '$2' but got '$3'." % [ 9 | $kinds, 10 | ast, 11 | $node.kind 12 | ] 13 | raise newException(JsonKindError, msg) 14 | 15 | macro isRefSkipDistinct*(arg: typed): untyped = 16 | ## internal only, do not use 17 | var impl = getTypeImpl(arg) 18 | if impl.kind == nnkBracketExpr and impl[0].eqIdent("typeDesc"): 19 | impl = getTypeImpl(impl[1]) 20 | while impl.kind == nnkDistinctTy: 21 | impl = getTypeImpl(impl[0]) 22 | result = newLit(impl.kind == nnkRefTy) 23 | 24 | # The following forward declarations don't work in older versions of Nim 25 | 26 | # forward declare all initFromJson 27 | 28 | proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) 29 | proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) 30 | proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) 31 | proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) 32 | proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) 33 | proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) 34 | proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) 35 | proc initFromJson[S, T](dst: var array[S, T]; jsonNode: JsonNode; jsonPath: var string) 36 | proc initFromJson[T](dst: var Table[string, T]; jsonNode: JsonNode; jsonPath: var string) 37 | proc initFromJson[T](dst: var OrderedTable[string, T]; jsonNode: JsonNode; jsonPath: var string) 38 | proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) 39 | proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) 40 | proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) 41 | proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) 42 | 43 | # initFromJson definitions 44 | 45 | proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) = 46 | verifyJsonKind(jsonNode, {JString, JNull}, jsonPath) 47 | # since strings don't have a nil state anymore, this mapping of 48 | # JNull to the default string is questionable. `none(string)` and 49 | # `some("")` have the same potentional json value `JNull`. 50 | dst = jsonNode.getStr 51 | 52 | proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) = 53 | verifyJsonKind(jsonNode, {JBool}, jsonPath) 54 | dst = jsonNode.getBool 55 | 56 | proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) = 57 | if jsonNode.kind == JNull: 58 | raise newException(KeyError, "key not found: " & jsonPath) 59 | dst = jsonNode.copy 60 | 61 | proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = 62 | verifyJsonKind(jsonNode, {JInt}, jsonPath) 63 | dst = T(jsonNode.getInt) 64 | 65 | proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) = 66 | verifyJsonKind(jsonNode, {JInt, JFloat}, jsonPath) 67 | dst = T(jsonNode.getFloat) 68 | 69 | proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) = 70 | verifyJsonKind(jsonNode, {JString}, jsonPath) 71 | dst = parseEnum[T](jsonNode.getStr) 72 | 73 | proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) = 74 | verifyJsonKind(jsonNode, {JArray}, jsonPath) 75 | dst.setLen jsonNode.len 76 | let orignalJsonPathLen = jsonPath.len 77 | for i in 0 ..< jsonNode.len: 78 | jsonPath.add '[' 79 | jsonPath.addInt i 80 | jsonPath.add ']' 81 | initFromJson(dst[i], jsonNode[i], jsonPath) 82 | jsonPath.setLen orignalJsonPathLen 83 | 84 | proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) = 85 | verifyJsonKind(jsonNode, {JArray}, jsonPath) 86 | let originalJsonPathLen = jsonPath.len 87 | for i in 0 ..< jsonNode.len: 88 | jsonPath.add '[' 89 | jsonPath.addInt i 90 | jsonPath.add ']' 91 | initFromJson(dst[i], jsonNode[i], jsonPath) 92 | jsonPath.setLen originalJsonPathLen 93 | 94 | proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = 95 | dst = initTable[string, T]() 96 | verifyJsonKind(jsonNode, {JObject}, jsonPath) 97 | let originalJsonPathLen = jsonPath.len 98 | for key, value in jsonNode.pairs: 99 | jsonPath.add '.' 100 | jsonPath.add key 101 | initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) 102 | jsonPath.setLen originalJsonPathLen 103 | 104 | proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = 105 | dst = initOrderedTable[string,T]() 106 | verifyJsonKind(jsonNode, {JObject}, jsonPath) 107 | let originalJsonPathLen = jsonPath.len 108 | for key, value in jsonNode.pairs: 109 | jsonPath.add '.' 110 | jsonPath.add key 111 | initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) 112 | jsonPath.setLen originalJsonPathLen 113 | 114 | proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = 115 | verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) 116 | if jsonNode.kind == JNull: 117 | dst = nil 118 | else: 119 | dst = new(T) 120 | initFromJson(dst[], jsonNode, jsonPath) 121 | 122 | proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) = 123 | if jsonNode.kind != JNull: 124 | when T is ref: 125 | dst = some(new(T)) 126 | else: 127 | dst = some(default(T)) 128 | initFromJson(dst.get, jsonNode, jsonPath) 129 | 130 | macro assignDistinctImpl[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = 131 | let typInst = getTypeInst(dst) 132 | let typImpl = getTypeImpl(dst) 133 | let baseTyp = typImpl[0] 134 | 135 | result = quote do: 136 | when nimvm: 137 | # workaround #12282 138 | var tmp: `baseTyp` 139 | initFromJson( tmp, `jsonNode`, `jsonPath`) 140 | `dst` = `typInst`(tmp) 141 | else: 142 | initFromJson( `baseTyp`(`dst`), `jsonNode`, `jsonPath`) 143 | 144 | proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = 145 | assignDistinctImpl(dst, jsonNode, jsonPath) 146 | 147 | proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode): void = 148 | if typeExpr.kind == nnkTupleConstr: 149 | error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) 150 | 151 | proc foldObjectBody(dst, typeNode, tmpSym, jsonNode, jsonPath, originalJsonPathLen: NimNode) = 152 | case typeNode.kind 153 | of nnkEmpty: 154 | discard 155 | of nnkRecList, nnkTupleTy: 156 | for it in typeNode: 157 | foldObjectBody(dst, it, tmpSym, jsonNode, jsonPath, originalJsonPathLen) 158 | 159 | of nnkIdentDefs: 160 | typeNode.expectLen 3 161 | let fieldSym = typeNode[0] 162 | let fieldNameLit = newLit(fieldSym.strVal) 163 | let fieldPathLit = newLit("." & fieldSym.strVal) 164 | let fieldType = typeNode[1] 165 | 166 | # Detecting incompatiple tuple types in `assignObjectImpl` only 167 | # would be much cleaner, but the ast for tuple types does not 168 | # contain usable type information. 169 | detectIncompatibleType(fieldType, fieldSym) 170 | 171 | dst.add quote do: 172 | jsonPath.add `fieldPathLit` 173 | when nimvm: 174 | when isRefSkipDistinct(`tmpSym`.`fieldSym`): 175 | # workaround #12489 176 | var tmp: `fieldType` 177 | initFromJson(tmp, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) 178 | `tmpSym`.`fieldSym` = tmp 179 | else: 180 | initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) 181 | else: 182 | initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) 183 | jsonPath.setLen `originalJsonPathLen` 184 | 185 | of nnkRecCase: 186 | let kindSym = typeNode[0][0] 187 | let kindNameLit = newLit(kindSym.strVal) 188 | let kindPathLit = newLit("." & kindSym.strVal) 189 | let kindType = typeNode[0][1] 190 | let kindOffsetLit = newLit(uint(getOffset(kindSym))) 191 | dst.add quote do: 192 | var kindTmp: `kindType` 193 | jsonPath.add `kindPathLit` 194 | initFromJson(kindTmp, `jsonNode`[`kindNameLit`], `jsonPath`) 195 | jsonPath.setLen `originalJsonPathLen` 196 | when defined js: 197 | `tmpSym`.`kindSym` = kindTmp 198 | else: 199 | when nimvm: 200 | `tmpSym`.`kindSym` = kindTmp 201 | else: 202 | # fuck it, assign kind field anyway 203 | ((cast[ptr `kindType`](cast[uint](`tmpSym`.addr) + `kindOffsetLit`))[]) = kindTmp 204 | dst.add nnkCaseStmt.newTree(nnkDotExpr.newTree(tmpSym, kindSym)) 205 | for i in 1 ..< typeNode.len: 206 | foldObjectBody(dst, typeNode[i], tmpSym, jsonNode, jsonPath, originalJsonPathLen) 207 | 208 | of nnkOfBranch, nnkElse: 209 | let ofBranch = newNimNode(typeNode.kind) 210 | for i in 0 ..< typeNode.len-1: 211 | ofBranch.add copyNimTree(typeNode[i]) 212 | let dstInner = newNimNode(nnkStmtListExpr) 213 | foldObjectBody(dstInner, typeNode[^1], tmpSym, jsonNode, jsonPath, originalJsonPathLen) 214 | # resOuter now contains the inner stmtList 215 | ofBranch.add dstInner 216 | dst[^1].expectKind nnkCaseStmt 217 | dst[^1].add ofBranch 218 | 219 | of nnkObjectTy: 220 | typeNode[0].expectKind nnkEmpty 221 | typeNode[1].expectKind {nnkEmpty, nnkOfInherit} 222 | if typeNode[1].kind == nnkOfInherit: 223 | let base = typeNode[1][0] 224 | var impl = getTypeImpl(base) 225 | while impl.kind in {nnkRefTy, nnkPtrTy}: 226 | impl = getTypeImpl(impl[0]) 227 | foldObjectBody(dst, impl, tmpSym, jsonNode, jsonPath, originalJsonPathLen) 228 | let body = typeNode[2] 229 | foldObjectBody(dst, body, tmpSym, jsonNode, jsonPath, originalJsonPathLen) 230 | 231 | else: 232 | error("unhandled kind: " & $typeNode.kind, typeNode) 233 | 234 | macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) = 235 | let typeSym = getTypeInst(dst) 236 | let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen") 237 | result = newStmtList() 238 | result.add quote do: 239 | let `originalJsonPathLen` = len(`jsonPath`) 240 | if typeSym.kind in {nnkTupleTy, nnkTupleConstr}: 241 | # both, `dst` and `typeSym` don't have good lineinfo. But nothing 242 | # else is available here. 243 | detectIncompatibleType(typeSym, dst) 244 | foldObjectBody(result, typeSym, dst, jsonNode, jsonPath, originalJsonPathLen) 245 | else: 246 | foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen) 247 | 248 | proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) = 249 | assignObjectImpl(dst, jsonNode, jsonPath) 250 | 251 | proc to*[T](node: JsonNode, t: typedesc[T]): T = 252 | ## `Unmarshals`:idx: the specified node into the object type specified. 253 | ## 254 | ## Known limitations: 255 | ## 256 | ## * Heterogeneous arrays are not supported. 257 | ## * Sets in object variants are not supported. 258 | ## * Not nil annotations are not supported. 259 | ## 260 | var jsonPath = "" 261 | initFromJson(result, node, jsonPath) 262 | 263 | when isMainModule: 264 | type 265 | ContentNodeKind = enum 266 | P, Br, Text 267 | ContentNode = object 268 | case kind: ContentNodeKind 269 | of P: pChildren: seq[ContentNode] 270 | of Br: nil 271 | of Text: textStr: string 272 | block: 273 | let mynode = ContentNode(kind: P, pChildren: @[ 274 | ContentNode(kind: Text, textStr: "mychild"), 275 | ContentNode(kind: Br) 276 | ]) 277 | 278 | doAssert $mynode == """(kind: P, pChildren: @[(kind: Text, textStr: "mychild"), (kind: Br)])""" 279 | let jsonNode = %*mynode 280 | doAssert $jsonNode["pChildren"].to(seq[JsonNode]) == """@[{"kind":"Text","textStr":"mychild"}, {"kind":"Br"}]""" 281 | doAssert $jsonNode == """{"kind":"P","pChildren":[{"kind":"Text","textStr":"mychild"},{"kind":"Br"}]}""" 282 | doAssert $jsonNode.to(ContentNode) == """(kind: P, pChildren: @[(kind: Text, textStr: "mychild"), (kind: Br)])""" 283 | --------------------------------------------------------------------------------