├── .codecov.yml ├── .editorconfig ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .travis.yml ├── LICENSE.txt ├── LICENSE_DE.txt ├── README.md ├── dub.sdl ├── source └── sdlite │ ├── ast.d │ ├── generator.d │ ├── internal.d │ ├── lexer.d │ ├── package.d │ └── parser.d └── travis.sh /.codecov.yml: -------------------------------------------------------------------------------- 1 | # Documentation: https://docs.codecov.io/docs/codecov-yaml 2 | 3 | coverage: 4 | precision: 3 5 | round: down 6 | range: "80...100" 7 | 8 | status: 9 | # Learn more at https://docs.codecov.io/docs/commit-status 10 | project: true 11 | patch: true 12 | changes: true 13 | 14 | comment: false 15 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*.{c,h,d,di,dd,json}] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | indent_style = tab 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | charset = utf-8 10 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Test Suite 2 | 3 | # Only triggers on pushes/PRs to master 4 | on: 5 | pull_request: 6 | branches: 7 | - master 8 | push: 9 | branches: 10 | - master 11 | 12 | jobs: 13 | test: 14 | name: CI 15 | strategy: 16 | fail-fast: false 17 | matrix: 18 | os: [ubuntu-latest] 19 | dc: [dmd-latest, ldc-latest, ldc-1.15.0] 20 | arch: [x86, x86_64] 21 | 22 | runs-on: ${{ matrix.os }} 23 | steps: 24 | - uses: actions/checkout@v2 25 | 26 | - name: Install D compiler 27 | uses: dlang-community/setup-dlang@v1 28 | with: 29 | compiler: ${{ matrix.dc }} 30 | 31 | - name: Run tests 32 | env: 33 | CONFIG: ${{matrix.config}} 34 | ARCH: ${{matrix.arch}} 35 | shell: bash 36 | run: dub test 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.dub 2 | dub.selections.json 3 | sdlang-test-* 4 | *.lib 5 | *.a 6 | *.so 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: d 2 | 3 | matrix: 4 | include: 5 | - d: dmd-beta 6 | - d: dmd-2.088.0 7 | env: [COVERAGE=true] 8 | - d: dmd-2.082.1 9 | - d: ldc-1.17.0 10 | - d: ldc-1.12.0 11 | 12 | sudo: false 13 | 14 | script: ./travis.sh 15 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Sönke Ludwig 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /LICENSE_DE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/s-ludwig/sdlite/8ddbbb474f5ddccb454b4c78540057e8b47c805e/LICENSE_DE.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | SDLite - a lightweight SDLang parser/generator 2 | ============================================== 3 | 4 | This library implements a small and efficient parser/generator for [SDLang][1] 5 | documents, providing a range based API. While the parser still uses the GC to 6 | allocate identifiers, strings etc., it uses a very efficient pool based 7 | allocation scheme that has very low computation and memory overhead. 8 | 9 | [![DUB Package](https://img.shields.io/dub/v/sdlite.svg)](https://code.dlang.org/packages/sdlite) 10 | [![Build Status](https://travis-ci.org/s-ludwig/sdlite.svg?branch=master)](https://travis-ci.org/s-ludwig/sdlite) 11 | [![codecov](https://codecov.io/gh/s-ludwig/sdlite/branch/master/graph/badge.svg)](https://codecov.io/gh/s-ludwig/sdlite) 12 | 13 | 14 | Project origins 15 | --------------- 16 | 17 | The motivation for writing another SDLang implementation for D came from the 18 | high overhead that the original [sdlang-d][2] implementation has. Parsing a 19 | particular 200 MB file took well over 30 seconds and used up almost 10GB of 20 | memory if parsed into a DOM. The following changes to the parsing approach 21 | brought the parsing time down to around 2.5 seconds: 22 | 23 | - Using a more efficient allocation scheme 24 | - Using only "native" range implementations instead of the more comfortable 25 | fiber-based approach taken by sdlang-d 26 | - Using `TaggedUnion` ([taggedalgebraic][3]) instead of `Variant` 27 | 28 | Further substantial improvements at this point are more difficult and likely 29 | require the use of bit-level tricks and SIMD for speeding up the lexer, as well 30 | as exploiting the properties of pure array inputs. 31 | 32 | [1]: https://sdlang.org/ 33 | [2]: https://github.com/Abscissa/SDLang-D 34 | [3]: https://github.com/s-ludwig/taggedalgebraic 35 | -------------------------------------------------------------------------------- /dub.sdl: -------------------------------------------------------------------------------- 1 | name "sdlite" 2 | description "Small and fast range based SDLang parser/generator" 3 | copyright "Copyright © 2019, Sönke Ludwig" 4 | license "MIT" 5 | 6 | dependency "taggedalgebraic" version="~>0.11.4" 7 | -------------------------------------------------------------------------------- /source/sdlite/ast.d: -------------------------------------------------------------------------------- 1 | /** Types for holding SDLang document data. 2 | */ 3 | module sdlite.ast; 4 | 5 | import taggedalgebraic.taggedunion; 6 | import std.datetime; 7 | import std.string : indexOf; 8 | 9 | @safe pure: 10 | 11 | void validateQualifiedIdentiifier(string qualified_ident) 12 | { 13 | auto idx = qualified_ident.indexOf(':'); 14 | if (idx >= 0) { 15 | if (qualified_ident[idx+1 .. $].indexOf(':') >= 0) 16 | throw new Exception("Multiple namespace separators in identifier: "~qualified_ident); 17 | validateIdentifier(qualified_ident[0 .. idx]); 18 | validateIdentifier(qualified_ident[idx+1 .. $]); 19 | } else validateIdentifier(qualified_ident); 20 | } 21 | 22 | void validateIdentifier(string ident) 23 | { 24 | // TODO 25 | } 26 | 27 | 28 | /** Represents a single SDL node. 29 | */ 30 | struct SDLNode { 31 | private string m_qualifiedName; 32 | SDLValue[] values; 33 | SDLAttribute[] attributes; 34 | SDLNode[] children; 35 | Location location; 36 | 37 | this(string qualified_name, SDLValue[] values = null, 38 | SDLAttribute[] attributes = null, SDLNode[] children = null) 39 | { 40 | this.qualifiedName = qualified_name; 41 | this.values = values; 42 | this.attributes = attributes; 43 | this.children = children; 44 | } 45 | 46 | @safe pure: 47 | /** Qualified name of the tag 48 | 49 | The form of this value is either "namespace:name" or just "name". 50 | */ 51 | @property string qualifiedName() const nothrow { return m_qualifiedName; } 52 | /// ditto 53 | @property void qualifiedName(string qualified_ident) 54 | { 55 | validateQualifiedIdentiifier(qualified_ident); 56 | m_qualifiedName = qualified_ident; 57 | } 58 | 59 | 60 | /// Namespace (if any) of the tag 61 | @property string namespace() 62 | const nothrow { 63 | auto idx = m_qualifiedName.indexOf(':'); 64 | if (idx >= 0) return m_qualifiedName[0 .. idx]; 65 | return null; 66 | } 67 | 68 | /// Unqualified name of the tag (use `namespace` to disambiguate) 69 | @property string name() 70 | const nothrow { 71 | auto idx = m_qualifiedName.indexOf(':'); 72 | if (idx >= 0) return m_qualifiedName[idx+1 .. $]; 73 | return m_qualifiedName; 74 | } 75 | 76 | /// Looks up an attribute by qualified name 77 | SDLValue getAttribute(string qualified_name, SDLValue default_ = SDLValue.null_) 78 | nothrow { 79 | foreach (ref a; attributes) 80 | if (a.qualifiedName == qualified_name) 81 | return a.value; 82 | return default_; 83 | } 84 | } 85 | 86 | 87 | /** Attribute of a node 88 | */ 89 | struct SDLAttribute { 90 | private string m_qualifiedName; 91 | 92 | SDLValue value; 93 | 94 | @safe pure: 95 | this(string qualified_ident, SDLValue value) 96 | { 97 | this.qualifiedName = qualified_ident; 98 | this.value = value; 99 | } 100 | 101 | /** Qualified name of the attribute 102 | 103 | The form of this value is either "namespace:name" or just "name". 104 | */ 105 | @property string qualifiedName() const nothrow { return m_qualifiedName; } 106 | /// ditto 107 | @property void qualifiedName(string qualified_ident) 108 | { 109 | validateQualifiedIdentiifier(qualified_ident); 110 | m_qualifiedName = qualified_ident; 111 | } 112 | 113 | /// Namespace (if any) of the attribute 114 | @property string namespace() 115 | const nothrow { 116 | auto idx = m_qualifiedName.indexOf(':'); 117 | if (idx >= 0) return m_qualifiedName[0 .. idx]; 118 | return null; 119 | } 120 | 121 | /// Unqualified name of the attribute (use `namespace` to disambiguate) 122 | @property string name() 123 | const nothrow { 124 | auto idx = m_qualifiedName.indexOf(':'); 125 | if (idx >= 0) return m_qualifiedName[idx+1 .. $]; 126 | return m_qualifiedName; 127 | } 128 | } 129 | 130 | 131 | /** A single SDLang value 132 | */ 133 | alias SDLValue = TaggedUnion!SDLValueFields; 134 | 135 | struct SDLValueFields { 136 | Void null_; 137 | string text; 138 | immutable(ubyte)[] binary; 139 | int int_; 140 | long long_; 141 | long[2] decimal; 142 | float float_; 143 | double double_; 144 | bool bool_; 145 | SysTime dateTime; 146 | Date date; 147 | Duration duration; 148 | } 149 | 150 | struct Location { 151 | /// Name of the source file 152 | string file; 153 | /// Line number within the file (Unix/Windows/Mac line endings are recognized, zero based) 154 | size_t line; 155 | /// Code unit offset from the start of the line 156 | size_t column; 157 | /// Code unit offset from the start of the input string 158 | size_t offset; 159 | } 160 | -------------------------------------------------------------------------------- /source/sdlite/generator.d: -------------------------------------------------------------------------------- 1 | /** Functionality for converting DOM nodes to SDLang documents. 2 | */ 3 | module sdlite.generator; 4 | 5 | import sdlite.ast; 6 | 7 | import core.time; 8 | import std.datetime; 9 | import std.range; 10 | import taggedalgebraic.taggedunion : visit; 11 | 12 | 13 | /** Writes out a range of `SDLNode`s to a `char` based output range. 14 | */ 15 | void generateSDLang(R, NR)(ref R dst, NR nodes, size_t level = 0) 16 | if (isOutputRange!(R, char) && isInputRange!NR && is(ElementType!NR : const(SDLNode))) 17 | { 18 | foreach (ref n; nodes) 19 | generateSDLang(dst, n, level); 20 | } 21 | 22 | unittest { 23 | auto app = appender!string; 24 | app.generateSDLang([ 25 | SDLNode("na"), 26 | SDLNode("nb", [SDLValue.int_(1), SDLValue.int_(2)]), 27 | SDLNode("nc", [SDLValue.int_(1)], [SDLAttribute("a", SDLValue.int_(2))]), 28 | SDLNode("nd", null, [SDLAttribute("a", SDLValue.int_(1)), SDLAttribute("b", SDLValue.int_(2))]), 29 | SDLNode("ne", null, null, [ 30 | SDLNode("foo:nf", null, null, [ 31 | SDLNode("ng") 32 | ]), 33 | ]) 34 | ]); 35 | assert(app.data == 36 | `na 37 | nb 1 2 38 | nc 1 a=2 39 | nd a=1 b=2 40 | ne { 41 | foo:nf { 42 | ng 43 | } 44 | } 45 | `, app.data); 46 | } 47 | 48 | 49 | /** Writes out single `SDLNode` to a `char` based output range. 50 | */ 51 | void generateSDLang(R)(ref R dst, auto ref const(SDLNode) node, size_t level = 0) 52 | { 53 | auto name = node.qualifiedName == "content" ? "" : node.qualifiedName; 54 | dst.putIndentation(level); 55 | dst.put(name); 56 | foreach (ref v; node.values) { 57 | dst.put(' '); 58 | dst.generateSDLang(v); 59 | } 60 | foreach (ref a; node.attributes) { 61 | dst.put(' '); 62 | dst.put(a.qualifiedName); 63 | dst.put('='); 64 | dst.generateSDLang(a.value); 65 | } 66 | if (node.children) { 67 | dst.put(" {\n"); 68 | dst.generateSDLang(node.children, level + 1); 69 | dst.putIndentation(level); 70 | dst.put("}\n"); 71 | } else dst.put('\n'); 72 | } 73 | 74 | 75 | /** Writes a single SDLang value to the given output range. 76 | */ 77 | void generateSDLang(R)(ref R dst, auto ref const(SDLValue) value) 78 | { 79 | import std.format : formattedWrite; 80 | 81 | // NOTE: using final switch instead of visit, because the latter causes 82 | // the creation of a heap delegate 83 | 84 | final switch (value.kind) { 85 | case SDLValue.Kind.null_: 86 | dst.put("null"); 87 | break; 88 | case SDLValue.Kind.text: 89 | dst.put('"'); 90 | dst.escapeSDLString(value.textValue); 91 | dst.put('"'); 92 | break; 93 | case SDLValue.Kind.binary: 94 | dst.put('['); 95 | dst.generateBase64(value.binaryValue); 96 | dst.put(']'); 97 | break; 98 | case SDLValue.Kind.int_: 99 | dst.formattedWrite("%s", value.intValue); 100 | break; 101 | case SDLValue.Kind.long_: 102 | dst.formattedWrite("%sL", value.longValue); 103 | break; 104 | case SDLValue.Kind.decimal: 105 | assert(false); 106 | case SDLValue.Kind.float_: 107 | dst.writeFloat(value.floatValue); 108 | dst.put('f'); 109 | break; 110 | case SDLValue.Kind.double_: 111 | dst.writeFloat(value.doubleValue); 112 | break; 113 | case SDLValue.Kind.bool_: 114 | dst.put(value.boolValue ? "true" : "false"); 115 | break; 116 | case SDLValue.Kind.dateTime: 117 | auto dt = cast(DateTime)value.dateTimeValue; 118 | auto fracsec = value.dateTimeValue.fracSecs; 119 | dst.formattedWrite("%d/%02d/%02d %02d:%02d:%02d", 120 | dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second); 121 | dst.writeFracSecs(fracsec.total!"hnsecs"); 122 | 123 | auto tz = value.dateTimeValue.timezone; 124 | 125 | if (tz is LocalTime()) {} 126 | else if (tz is UTC()) dst.put("-UTC"); 127 | else if (auto sz = cast(immutable(SimpleTimeZone))tz) { 128 | long hours, minutes; 129 | sz.utcOffset.split!("hours", "minutes")(hours, minutes); 130 | if (hours < 0 || minutes < 0) 131 | dst.formattedWrite("-GMT-%02d:%02d", -hours, -minutes); // NOTE: should really be UTC, but we are following the spec here 132 | else dst.formattedWrite("-GMT+%02d:%02d", hours, minutes); 133 | } else { 134 | auto offset = tz.utcOffsetAt(value.dateTimeValue.stdTime); 135 | long hours, minutes; 136 | offset.split!("hours", "minutes")(hours, minutes); 137 | if (hours < 0 || minutes < 0) 138 | dst.formattedWrite("-GMT-%02d:%02d", -hours, -minutes); // NOTE: should really be UTC, but we are following the spec here 139 | else dst.formattedWrite("-GMT+%02d:%02d", hours, minutes); 140 | //dst.formattedWrite("-%s", tz.stdName); // Q: should this be name instead (e.g. CEST vs. CET) 141 | } 142 | break; 143 | case SDLValue.Kind.date: 144 | auto d = value.dateValue; 145 | dst.formattedWrite("%d/%02d/%02d", d.year, d.month, d.day); 146 | break; 147 | case SDLValue.Kind.duration: 148 | long days, hours, minutes, seconds, hnsecs; 149 | value.durationValue.split!("days", "hours", "minutes", "seconds", "hnsecs") 150 | (days, hours, minutes, seconds, hnsecs); 151 | if (days > 0) dst.formattedWrite("%sd:", days); 152 | dst.formattedWrite("%02d:%02d", hours, minutes); 153 | if (seconds != 0 || hnsecs != 0) { 154 | dst.formattedWrite(":%02d", seconds); 155 | dst.writeFracSecs(hnsecs); 156 | } 157 | break; 158 | } 159 | } 160 | 161 | unittest { 162 | import std.array : appender; 163 | 164 | void test(SDLValue v, string exp) 165 | { 166 | auto app = appender!string; 167 | app.generateSDLang(v); 168 | assert(app.data == exp, app.data); 169 | } 170 | 171 | test(SDLValue.null_, "null"); 172 | test(SDLValue.bool_(false), "false"); 173 | test(SDLValue.bool_(true), "true"); 174 | test(SDLValue.text("foo\"bar"), `"foo\"bar"`); 175 | test(SDLValue.binary(cast(immutable(ubyte)[])"hello, world!"), "[aGVsbG8sIHdvcmxkIQ==]"); 176 | test(SDLValue.int_(int.max), "2147483647"); 177 | test(SDLValue.int_(int.min), "-2147483648"); 178 | test(SDLValue.long_(long.max), "9223372036854775807L"); 179 | test(SDLValue.long_(long.min), "-9223372036854775808L"); 180 | test(SDLValue.float_(2.2f), "2.2f"); 181 | test(SDLValue.double_(2.2), "2.2"); 182 | test(SDLValue.double_(1.0), "1.0"); // make sure there is always a fractional part 183 | test(SDLValue.date(Date(2015, 12, 6)), "2015/12/06"); 184 | test(SDLValue.duration(12.hours + 14.minutes + 34.seconds), "12:14:34"); 185 | test(SDLValue.duration(12.hours + 14.minutes + 34.seconds + 123.msecs), "12:14:34.123"); 186 | test(SDLValue.duration(2.days + 12.hours + 14.minutes + 34.seconds), "2d:12:14:34"); 187 | test(SDLValue.dateTime(SysTime(DateTime(2015, 12, 6, 12, 0, 0))), "2015/12/06 12:00:00"); 188 | test(SDLValue.dateTime(SysTime(DateTime(2015, 12, 6, 12, 0, 0), 123.msecs)), "2015/12/06 12:00:00.123"); 189 | test(SDLValue.dateTime(SysTime(DateTime(2015, 12, 6, 12, 0, 0), UTC())), "2015/12/06 12:00:00-UTC"); 190 | test(SDLValue.dateTime(SysTime(DateTime(2015, 12, 6, 12, 0, 0), new immutable SimpleTimeZone(-2.hours - 30.minutes))), "2015/12/06 12:00:00-GMT-02:30"); 191 | test(SDLValue.dateTime(SysTime(DateTime(2015, 12, 6, 12, 0, 0), new immutable SimpleTimeZone(31.minutes))), "2015/12/06 12:00:00-GMT+00:31"); 192 | test(SDLValue.dateTime(SysTime(DateTime(2017, 11, 22, 18, 0, 0), new immutable SimpleTimeZone(0.hours))), "2017/11/22 18:00:00-GMT+00:00"); 193 | } 194 | 195 | 196 | /** Escapes a given string to ensure safe usage within an SDLang quoted string. 197 | */ 198 | void escapeSDLString(R)(ref R dst, in char[] str) 199 | { 200 | // TODO: insert line breaks 201 | foreach (char ch; str) { 202 | switch (ch) { 203 | default: dst.put(ch); break; 204 | case '"': dst.put(`\"`); break; 205 | case '\\': dst.put(`\\`); break; 206 | case '\t': dst.put(`\t`); break; 207 | case '\n': dst.put(`\n`); break; 208 | case '\r': dst.put(`\r`); break; 209 | } 210 | } 211 | } 212 | 213 | unittest { 214 | import std.array : appender; 215 | 216 | auto app = appender!string; 217 | app.escapeSDLString("foo\\bar\r\n\t\tbäz\""); 218 | assert(app.data == `foo\\bar\r\n\t\tbäz\"`, app.data); 219 | } 220 | 221 | private void putIndentation(R)(ref R dst, size_t level) 222 | { 223 | foreach (i; 0 .. level) 224 | dst.put('\t'); 225 | } 226 | 227 | // output a floating point number in pure decimal format, without losing 228 | // precision (at least approximately) and without redundant zeros 229 | private void writeFloat(R, F)(ref R dst, const(F) num) 230 | { 231 | import std.format : formattedWrite; 232 | import std.math : floor, fmod, isNaN, log10; 233 | 234 | static if (is(F == float)) enum sig = 7; 235 | else enum sig = 15; 236 | 237 | if (num.isNaN || num == F.infinity || num == -F.infinity) { 238 | dst.put("0.0"); 239 | return; 240 | } 241 | 242 | if (!num) { 243 | dst.put("0.0"); 244 | return; 245 | } 246 | 247 | if (fmod(num, F(1)) == 0) dst.formattedWrite("%.1f", num); 248 | else { 249 | F unum; 250 | if (num < 0) { 251 | dst.put('-'); 252 | unum = -num; 253 | } else unum = num; 254 | 255 | auto firstdig = cast(long)floor(log10(unum)); 256 | if (firstdig >= sig) dst.formattedWrite("%.1f", unum); 257 | else { 258 | char[32] fmt; 259 | char[] fmtdst = fmt[]; 260 | fmtdst.formattedWrite("%%.%sg", sig - firstdig); 261 | dst.formattedWrite(fmt[0 .. $-fmtdst.length], unum); 262 | } 263 | } 264 | } 265 | 266 | unittest { 267 | void test(F)(F v, string txt) 268 | { 269 | auto app = appender!string; 270 | app.writeFloat(v); 271 | assert(app.data == txt, app.data); 272 | } 273 | 274 | test(float.infinity, "0.0"); 275 | test(-float.infinity, "0.0"); 276 | test(float.nan, "0.0"); 277 | 278 | test(double.infinity, "0.0"); 279 | test(-double.infinity, "0.0"); 280 | test(double.nan, "0.0"); 281 | 282 | test(0.0, "0.0"); 283 | test(1.0, "1.0"); 284 | test(-1.0, "-1.0"); 285 | test(0.0f, "0.0"); 286 | test(1.0f, "1.0"); 287 | test(-1.0f, "-1.0"); 288 | 289 | test(100.0, "100.0"); 290 | test(0.0078125, "0.0078125"); 291 | test(100.001, "100.001"); 292 | test(-100.0, "-100.0"); 293 | test(-0.0078125, "-0.0078125"); 294 | test(-100.001, "-100.001"); 295 | test(100.0f, "100.0"); 296 | test(0.0078125f, "0.0078125"); 297 | test(100.01f, "100.01"); 298 | test(-100.0f, "-100.0"); 299 | test(-0.0078125f, "-0.0078125"); 300 | test(-100.01f, "-100.01"); 301 | } 302 | 303 | private void writeFracSecs(R)(ref R dst, long hnsecs) 304 | { 305 | import std.format : formattedWrite; 306 | 307 | assert(hnsecs >= 0 && hnsecs < 10_000_000); 308 | 309 | if (hnsecs > 0) { 310 | if (hnsecs % 10_000 == 0) 311 | dst.formattedWrite(".%03d", hnsecs / 10_000); 312 | else dst.formattedWrite(".%07d", hnsecs); 313 | } 314 | } 315 | 316 | unittest { 317 | import std.array : appender; 318 | 319 | void test(Duration dur, string exp) 320 | { 321 | auto app = appender!string; 322 | app.writeFracSecs(dur.total!"hnsecs"); 323 | assert(app.data == exp, app.data); 324 | } 325 | 326 | test(0.msecs, ""); 327 | test(123.msecs, ".123"); 328 | test(123400.usecs, ".1234000"); 329 | test(1234567.hnsecs, ".1234567"); 330 | } 331 | 332 | 333 | private void generateBase64(R)(ref R dst, in ubyte[] bytes) 334 | { 335 | import std.base64 : Base64; 336 | 337 | Base64.encode(bytes, dst); 338 | } 339 | -------------------------------------------------------------------------------- /source/sdlite/internal.d: -------------------------------------------------------------------------------- 1 | module sdlite.internal; 2 | 3 | import std.traits : Unqual; 4 | 5 | 6 | package struct MultiAppender(T) 7 | { 8 | import std.algorithm.comparison : max; 9 | 10 | enum bufferMinSize = max(100, 64*1024 / T.sizeof); 11 | 12 | private { 13 | Unqual!T[] m_buffer; 14 | size_t m_base; 15 | size_t m_fill; 16 | } 17 | 18 | @disable this(this); 19 | 20 | @safe: 21 | 22 | static if (T.sizeof > 2*long.sizeof) { 23 | void put(ref Unqual!T item) 24 | { 25 | reserve(1); 26 | m_buffer[m_fill++] = cast(Unqual!T)item; 27 | } 28 | } else { 29 | void put(T item) 30 | { 31 | reserve(1); 32 | m_buffer[m_fill++] = cast(Unqual!T)item; 33 | } 34 | 35 | void put(Unqual!T[] items) 36 | { 37 | reserve(items.length); 38 | m_buffer[m_fill .. m_fill + items.length] = items; 39 | m_fill += items.length; 40 | } 41 | } 42 | 43 | T[] extractArray() 44 | @trusted { 45 | auto ret = m_buffer[m_base .. m_fill]; 46 | m_base = m_fill; 47 | // NOTE: cast to const/immutable is okay here, because this is the only 48 | // reference to the returned bytes 49 | return cast(T[])ret; 50 | } 51 | 52 | private void reserve(size_t n) 53 | { 54 | if (m_fill + n <= m_buffer.length) return; 55 | 56 | if (m_base == 0) { 57 | m_buffer.length = max(bufferMinSize, m_buffer.length * 2, m_fill + n); 58 | } else { 59 | auto newbuf = new Unqual!T[]((m_fill - m_base + n + bufferMinSize - 1) / bufferMinSize * bufferMinSize); 60 | newbuf[0 .. m_fill - m_base] = m_buffer[m_base .. m_fill]; 61 | m_buffer = newbuf; 62 | m_fill = m_fill - m_base; 63 | m_base = 0; 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /source/sdlite/lexer.d: -------------------------------------------------------------------------------- 1 | module sdlite.lexer; 2 | 3 | import sdlite.ast : Location, SDLValue; 4 | import sdlite.internal : MultiAppender; 5 | 6 | import std.algorithm.comparison : among; 7 | import std.algorithm.mutation : move, swap; 8 | import std.range; 9 | import std.datetime; 10 | import std.uni : isAlpha, isWhite; 11 | import std.utf : byCodeUnit, decodeFront; 12 | import core.time : Duration, days, hours, minutes, seconds, msecs, hnsecs; 13 | 14 | 15 | /** Represents a single token within an SDL document. 16 | */ 17 | struct Token(R) { 18 | alias SourceRange = R; 19 | 20 | /** Token type. 21 | */ 22 | TokenType type; 23 | 24 | /** Location of the token within the source document. 25 | */ 26 | Location location; 27 | 28 | /** Contains any white space precedes the token. 29 | 30 | Note that this will also include any line continuations between tokens. 31 | The characters that can occur are spaces, tabs, newlines, 32 | carriage returns, as well as the backslash character. 33 | */ 34 | Take!R whitespacePrefix; 35 | 36 | /** Raw string representation of the token. 37 | 38 | Note that certain token types, such as strings, base64 data and 39 | date/time values will have to be parsed using `parseValue` in order to 40 | be semantically usable. 41 | */ 42 | Take!R text; 43 | } 44 | 45 | enum TokenType { 46 | invalid, /// Malformed token 47 | eof, /// Denotes the end of the document 48 | eol, /// Line break 49 | assign, /// Equal sign 50 | namespace, /// Colon, used to separate namespace from name 51 | blockOpen, /// Opening brace 52 | blockClose, /// Closing brace 53 | semicolon, /// Semicolon 54 | comment, /// Any kind of comment 55 | identifier, /// A single identifier 56 | null_, /// The null keyword 57 | text, /// Any string value 58 | binary, /// Base-64 encoded binary data value 59 | number, /// Any kind of number value 60 | boolean, /// 'true' or 'false' 61 | dateTime, /// Date/time value 62 | date, /// Date value 63 | duration /// Duration value 64 | } 65 | 66 | 67 | /** Returns a range of `SDLToken`s by lexing the given SDLang input. 68 | */ 69 | auto lexSDLang(const(char)[] input, string filename = "") 70 | { 71 | return lexSDLang(input.byCodeUnit, filename); 72 | } 73 | /// ditto 74 | auto lexSDLang(R)(R input, string filename = "") 75 | if (isForwardRange!R && is(immutable(ElementType!R) == immutable(char))) 76 | { 77 | return SDLangLexer!R(input, filename); 78 | } 79 | 80 | 81 | package SDLValue parseValue(R)(ref Token!R t, 82 | ref MultiAppender!(immutable(char)) char_appender, 83 | ref MultiAppender!(immutable(ubyte)) byte_appender) 84 | { 85 | import std.algorithm.comparison : min, max; 86 | import std.algorithm.iteration : splitter; 87 | import std.algorithm.searching : endsWith, findSplit; 88 | import std.conv : parse, to; 89 | import std.exception : assumeUnique; 90 | import std.format : formattedRead; 91 | import std.typecons : Rebindable; 92 | import std.uni : icmp; 93 | 94 | final switch (t.type) { 95 | case TokenType.invalid: 96 | case TokenType.eof: 97 | case TokenType.eol: 98 | case TokenType.assign: 99 | case TokenType.namespace: 100 | case TokenType.blockOpen: 101 | case TokenType.blockClose: 102 | case TokenType.semicolon: 103 | case TokenType.comment: 104 | case TokenType.identifier: 105 | return SDLValue.null_; 106 | case TokenType.null_: 107 | return SDLValue.null_; 108 | case TokenType.text: 109 | assert(!t.text.empty); 110 | if (t.text.front == '`') { 111 | auto txt = t.text 112 | .save 113 | .dropOne 114 | .take(t.text.length - 2); 115 | foreach (ch; txt) char_appender.put(ch); 116 | return SDLValue.text(char_appender.extractArray); 117 | } else { 118 | assert(t.text.front == '"'); 119 | t.parseTextValue(char_appender); 120 | return SDLValue.text(char_appender.extractArray); 121 | } 122 | case TokenType.binary: 123 | t.parseBinaryValue(byte_appender); 124 | return SDLValue.binary(byte_appender.extractArray); 125 | case TokenType.number: 126 | auto numparts = t.text.save.findSplit("."); 127 | if (numparts[1].empty) { // integer or integer-like float 128 | auto num = parse!long(numparts[0]); 129 | if (numparts[0].empty) 130 | return SDLValue.int_(cast(int)cast(uint)cast(ulong)num); 131 | 132 | switch (numparts[0].front) { 133 | default: assert(false); 134 | case 'l', 'L': return SDLValue.long_(num); 135 | case 'd', 'D': return SDLValue.double_(num); 136 | case 'f', 'F': return SDLValue.float_(num); 137 | } 138 | } 139 | 140 | auto r = t.text.save; 141 | 142 | if (numparts[2].length >= 2) { 143 | if (numparts[2].save.tail(2).icmp("bd") == 0) 144 | return SDLValue.null_; // decimal not yet supported 145 | if (numparts[2].save.retro.front.among!('f', 'F')) 146 | return SDLValue.float_(r.parse!float); 147 | } 148 | return SDLValue.double_(r.parse!double); 149 | case TokenType.boolean: 150 | switch (t.text.front) { 151 | default: assert(false); 152 | case 't': return SDLValue.bool_(true); 153 | case 'f': return SDLValue.bool_(false); 154 | case 'o': 155 | auto txt = t.text.save.dropOne; 156 | return SDLValue.bool_(txt.front == 'n'); 157 | } 158 | case TokenType.date: 159 | int y, m, d; 160 | t.text.save.formattedRead("%d/%d/%d", y, m, d); 161 | return SDLValue.date(Date(y, m, d)); 162 | case TokenType.duration: 163 | auto parts = t.text.save.splitter(":"); 164 | int d, h, m, s; 165 | if (parts.front.save.endsWith("d")) { 166 | d = parts.front.dropBackOne.to!int(); 167 | parts.popFront(); 168 | } 169 | h = parts.front.to!int(); 170 | parts.popFront(); 171 | m = parts.front.to!int(); 172 | parts.popFront(); 173 | auto sec = parts.front.findSplit("."); 174 | s = sec[0].to!int; 175 | Duration fracsec = Duration.zero; 176 | if (!sec[1].empty) { 177 | auto l0 = sec[2].length; 178 | long fs = sec[2].parse!long(); 179 | fracsec = (fs * (10 ^^ (7 - l0))).hnsecs; 180 | } 181 | return SDLValue.duration(d.days + h.hours + m.minutes + s.seconds + fracsec); 182 | case TokenType.dateTime: 183 | int y, m, d, hh, mm, ss; 184 | auto txt = t.text.save; 185 | txt.formattedRead("%d/%d/%d %d:%d", y, m, d, hh, mm); 186 | if (!txt.empty && txt.front == ':') { 187 | txt.popFront(); 188 | ss = txt.parse!int(); 189 | } 190 | auto dt = DateTime(y, m, d, hh, mm, ss); 191 | Rebindable!(immutable(TimeZone)) tz; 192 | Duration fracsec = Duration.zero; 193 | 194 | if (!txt.empty && txt.front == '.') { 195 | txt.popFront(); 196 | auto l0 = txt.length; 197 | long fs = txt.parse!long(); 198 | fracsec = (fs * (10 ^^ (7 - (l0 - txt.length)))).hnsecs; 199 | } 200 | 201 | if (!txt.empty) { 202 | txt.popFront(); 203 | char[3] tzt; 204 | txt.formattedRead("%c%c%c", tzt[0], tzt[1], tzt[2]); 205 | if (!txt.empty) { 206 | int mul = txt.front == '-' ? -1 : 1; 207 | txt.popFront(); 208 | int dh = txt.parse!int(); 209 | int dm = 0; 210 | if (!txt.empty) { 211 | txt.formattedRead(":%d", dm); 212 | } 213 | tz = new immutable SimpleTimeZone((mul*dh).hours + (mul*dm).minutes); 214 | } else if (tzt == "UTC" || tzt == "GMT") { 215 | tz = UTC(); 216 | } else { 217 | version (Windows) tz = WindowsTimeZone.getTimeZone(tzt[].idup); 218 | else tz = PosixTimeZone.getTimeZone(tzt[].idup); 219 | } 220 | } else tz = LocalTime(); 221 | 222 | return SDLValue.dateTime(SysTime(dt, fracsec, tz)); 223 | } 224 | } 225 | 226 | package void parseTextValue(R, DR)(ref Token!R t, ref DR dst) 227 | { 228 | import std.algorithm.mutation : copy; 229 | 230 | assert(t.type == TokenType.text); 231 | assert(!t.text.empty); 232 | 233 | auto content = t.text.save.dropOne().take(t.text.length - 2); 234 | 235 | if (t.text.front == '`') { // WYSIWYG string 236 | foreach (char ch; content) 237 | dst.put(ch); 238 | return; 239 | } 240 | 241 | assert(t.text.front == '"'); 242 | 243 | static void skipWhitespace(R)(ref R r) 244 | { 245 | while (!r.empty && r.front.among!(' ', '\t')) 246 | r.popFront(); 247 | } 248 | 249 | while (content.length) { 250 | char ch = content.front; 251 | content.popFront(); 252 | 253 | if (ch != '\\') dst.put(ch); 254 | else { 255 | assert(!content.empty); 256 | ch = content.front; 257 | content.popFront(); 258 | 259 | switch (ch) { 260 | default: assert(false); 261 | case '\r': 262 | if (!content.empty && content.front == '\n') 263 | content.popFront(); 264 | skipWhitespace(content); 265 | break; 266 | case '\n': skipWhitespace(content); break; 267 | case 'r': dst.put('\r'); break; 268 | case 'n': dst.put('\n'); break; 269 | case 't': dst.put('\t'); break; 270 | case '"': dst.put('"'); break; 271 | case '\\': dst.put('\\'); break; 272 | } 273 | } 274 | } 275 | } 276 | 277 | package void parseBinaryValue(R, DR)(ref Token!R t, ref DR dst) 278 | { 279 | import std.base64 : Base64; 280 | 281 | assert(!t.text.empty); 282 | assert(t.text.front == '['); 283 | 284 | auto content = t.text.save.dropOne.take(t.text.length - 2); 285 | char[4] buf; 286 | 287 | while (!content.empty) { 288 | foreach (i; 0 .. 4) { 289 | while (content.front.among!(' ', '\t', '\r', '\n')) 290 | content.popFront(); 291 | buf[i] = content.front; 292 | content.popFront(); 293 | } 294 | 295 | ubyte[3] bytes; 296 | dst.put(Base64.decode(buf[], bytes[])); 297 | } 298 | } 299 | 300 | private struct SDLangLexer(R) 301 | if (isForwardRange!R && is(immutable(ElementType!R) == immutable(char))) 302 | { 303 | private { 304 | R m_input; 305 | Location m_location; 306 | Token!R m_token; 307 | bool m_empty; 308 | } 309 | 310 | /** Initializes a lexer for the given input SDL document. 311 | 312 | The document must be given in the form of a UTF-8 encoded text that is 313 | stored as a `ubyte` forward range. 314 | */ 315 | this(R input, string filename) 316 | { 317 | m_input = input.move; 318 | m_location.file = filename; 319 | 320 | readNextToken(); 321 | } 322 | 323 | @property bool empty() const { return m_empty; } 324 | 325 | ref inout(Token!R) front() 326 | inout { 327 | return m_token; 328 | } 329 | 330 | SDLangLexer save() 331 | { 332 | SDLangLexer ret; 333 | ret.m_input = m_input.save; 334 | ret.m_location = m_location; 335 | ret.m_token = m_token; 336 | ret.m_empty = m_empty; 337 | return ret; 338 | } 339 | 340 | void popFront() 341 | { 342 | assert(!empty); 343 | if (m_token.type == TokenType.eof) m_empty = true; 344 | else readNextToken(); 345 | } 346 | 347 | private void readNextToken() 348 | { 349 | import std.algorithm.comparison : equal; 350 | 351 | m_token.whitespacePrefix = skipWhitespace(); 352 | m_token.location = m_location; 353 | 354 | if (m_input.empty) { 355 | m_token.type = TokenType.eof; 356 | m_token.text = m_input.take(0); 357 | return; 358 | } 359 | 360 | auto tstart = m_input.save; 361 | m_token.type = skipToken(); 362 | m_token.text = tstart.take(m_location.offset - m_token.location.offset); 363 | 364 | // keywords are initially parsed as identifiers 365 | if (m_token.type == TokenType.identifier 366 | && m_token.text.front.among!('o', 't', 'f', 'n')) 367 | { 368 | if (m_token.text.equal("on") || m_token.text.equal("off") || 369 | m_token.text.equal("true") || m_token.text.equal("false")) 370 | { 371 | m_token.type = TokenType.boolean; 372 | } else if (m_token.text.equal("null")) { 373 | m_token.type = TokenType.null_; 374 | } 375 | } 376 | } 377 | 378 | private TokenType skipToken() 379 | { 380 | switch (m_input.front) { 381 | case '\r': 382 | skipChar!true(); 383 | if (!m_input.empty && m_input.front == '\n') 384 | skipChar!false(); 385 | return TokenType.eol; 386 | case '\n': 387 | skipChar!true(); 388 | return TokenType.eol; 389 | case '/': // C/C++ style comment 390 | skipChar!false(); 391 | if (m_input.empty || !m_input.front.among!('/', '*')) { 392 | return TokenType.invalid; 393 | } 394 | if (m_input.front == '/') { 395 | skipChar!false(); 396 | skipLine(false); 397 | 398 | return TokenType.comment; 399 | } 400 | 401 | skipChar!false(); 402 | 403 | while (true) { 404 | while (!m_input.empty && m_input.front != '*') 405 | skipChar!true(); 406 | 407 | if (!m_input.empty) skipChar!false(); 408 | 409 | if (m_input.empty) { 410 | return TokenType.invalid; 411 | } 412 | 413 | if (m_input.front == '/') { 414 | skipChar!false(); 415 | return TokenType.comment; 416 | } 417 | } 418 | assert(false); 419 | case '-': // LUA style comment or negative number 420 | skipChar!false(); 421 | 422 | if (m_input.empty) return TokenType.invalid; 423 | 424 | auto ch = m_input.front; 425 | if (ch >= '0' && ch <= '9') 426 | return skipNumericToken(); 427 | 428 | if (ch != '-') return TokenType.invalid; 429 | 430 | skipChar!false(); 431 | skipLine(); 432 | 433 | return TokenType.comment; 434 | case '#': // shell style comment 435 | skipChar!false(); 436 | skipLine(); 437 | return TokenType.comment; 438 | case '"': // normal string 439 | skipChar!false(); 440 | 441 | outerstr: while (!m_input.empty) { 442 | char ch = m_input.front; 443 | if (ch.among!('\r', '\n')) break; 444 | 445 | skipChar!false(); 446 | 447 | if (ch == '"') { 448 | return TokenType.text; 449 | } else if (ch == '\\') { 450 | ch = m_input.front; 451 | skipChar!false(); 452 | switch (ch) { 453 | default: break outerstr; 454 | case '"', '\\', 'n', 'r', 't': break; 455 | case '\n', '\r': 456 | skipChar!true(); 457 | skipWhitespace(); 458 | break; 459 | } 460 | } 461 | } 462 | 463 | return TokenType.invalid; 464 | case '`': // WYSIWYG string 465 | skipChar!false(); 466 | 467 | while (!m_input.empty) { 468 | if (m_input.front == '`') { 469 | skipChar!false(); 470 | return TokenType.text; 471 | } 472 | 473 | skipChar!true(); 474 | } 475 | 476 | return TokenType.invalid; 477 | case '[': // base64 data 478 | import std.array : appender; 479 | 480 | skipChar!false(); 481 | 482 | 483 | uint chunklen = 0; 484 | 485 | while (!m_input.empty) { 486 | auto ch = m_input.front; 487 | switch (ch) { 488 | case ']': 489 | skipChar!false(); 490 | if (chunklen != 0) { // content length must be a multiple of 4 491 | return TokenType.invalid; 492 | } 493 | return TokenType.binary; 494 | case '0': .. case '9': 495 | case 'A': .. case 'Z': 496 | case 'a': .. case 'z': 497 | case '+', '/', '=': 498 | if (++chunklen == 4) 499 | chunklen = 0; 500 | skipChar!false(); 501 | break; 502 | case ' ', '\t': skipChar!false(); break; 503 | case '\r', '\n': skipChar!true(); break; 504 | default: return TokenType.invalid; 505 | } 506 | 507 | } 508 | 509 | return TokenType.invalid; 510 | case '{': skipChar!false(); return TokenType.blockOpen; 511 | case '}': skipChar!false(); return TokenType.blockClose; 512 | case ';': skipChar!false(); return TokenType.semicolon; 513 | case '=': skipChar!false(); return TokenType.assign; 514 | case ':': skipChar!false(); return TokenType.namespace; 515 | case '0': .. case '9': // number or date/time 516 | return skipNumericToken(); 517 | default: // identifier 518 | const chf = m_input.front; 519 | switch (chf) { 520 | case '0': .. case '9': 521 | case 'A': .. case 'Z': 522 | case 'a': .. case 'z': 523 | case '_': 524 | skipChar!false(); 525 | break; 526 | default: 527 | size_t n; 528 | auto dch = m_input.decodeFront(n); 529 | m_location.offset += n; 530 | m_location.column += n; 531 | if (!dch.isAlpha && dch != '_') 532 | return TokenType.invalid; 533 | break; 534 | } 535 | 536 | outer: while (!m_input.empty) { 537 | const ch = m_input.front; 538 | switch (ch) { 539 | case '0': .. case '9': 540 | case 'A': .. case 'Z': 541 | case 'a': .. case 'z': 542 | case '_', '-', '.', '$': 543 | skipChar!false(); 544 | break; 545 | default: 546 | // all eglible ASCII characters are handled above 547 | if (!(ch & 0x80)) break outer; 548 | 549 | // test if this is a Unicode alphabectical character 550 | auto inp = m_input.save; 551 | size_t n; 552 | dchar dch = m_input.decodeFront(n); 553 | if (!isAlpha(dch)) { 554 | swap(inp, m_input); 555 | break outer; 556 | } 557 | m_location.offset += n; 558 | m_location.column += n; 559 | break; 560 | 561 | } 562 | } 563 | 564 | return TokenType.identifier; 565 | } 566 | } 567 | 568 | private TokenType skipNumericToken() 569 | { 570 | assert(m_input.front >= '0' && m_input.front <= '9'); 571 | skipChar!false(); 572 | 573 | while (!m_input.empty && m_input.front >= '0' && m_input.front <= '9') 574 | skipChar!false(); 575 | 576 | if (m_input.empty) // unqualified integer 577 | return TokenType.number; 578 | 579 | auto ch = m_input.front; 580 | switch (ch) { // unqualified integer 581 | default: 582 | return TokenType.number; 583 | case ':': // time span 584 | if (!skipDuration(No.includeFirstNumber)) { 585 | return TokenType.invalid; 586 | } 587 | return TokenType.duration; 588 | case 'D': // double with no fractional part 589 | skipChar!false(); 590 | return TokenType.number; 591 | case 'f', 'F': // float with no fractional part 592 | skipChar!false(); 593 | return TokenType.number; 594 | case 'd': // time span with days or double value 595 | skipChar!false(); 596 | if (m_input.empty || m_input.front != ':') { 597 | return TokenType.number; 598 | } 599 | 600 | skipChar!false(); 601 | 602 | if (!skipDuration(Yes.includeFirstNumber)) { 603 | return TokenType.invalid; 604 | } 605 | return TokenType.duration; 606 | case '/': // date 607 | if (!skipDate(No.includeFirstNumber)) { 608 | return TokenType.invalid; 609 | } 610 | if (m_input.empty || m_input.front != ' ') { 611 | return TokenType.date; 612 | } 613 | 614 | auto input_saved = m_input.save; 615 | auto loc_saved = m_location; 616 | 617 | skipChar!false(); 618 | 619 | if (!skipTimeOfDay()) { 620 | swap(m_input, input_saved); 621 | swap(m_location, loc_saved); 622 | return TokenType.date; 623 | } 624 | 625 | if (!m_input.empty && m_input.front == '-') { 626 | skipChar!false(); 627 | if (!skipTimeZone()) { 628 | return TokenType.invalid; 629 | } 630 | } 631 | 632 | return TokenType.dateTime; 633 | case '.': // floating point 634 | skipChar!false(); 635 | if (m_input.front < '0' || m_input.front > '9') { 636 | return TokenType.invalid; 637 | } 638 | 639 | while (!m_input.empty && m_input.front >= '0' && m_input.front <= '9') 640 | skipChar!false(); 641 | 642 | if (m_input.empty || m_input.front.among!('f', 'F', 'd', 'D')) { // IEEE floating-point 643 | if (!m_input.empty) skipChar!false(); 644 | return TokenType.number; 645 | } 646 | 647 | if (m_input.front.among!('b', 'B')) { // decimal 648 | skipChar!false(); 649 | if (!m_input.front.among!('d', 'D')) { // FIXME: only "bd" or "BD" should be allowed, not "bD" 650 | return TokenType.invalid; 651 | } 652 | 653 | skipChar!false(); 654 | return TokenType.number; 655 | } 656 | 657 | return TokenType.number; 658 | case 'l', 'L': // long integer 659 | skipChar!false(); 660 | return TokenType.number; 661 | } 662 | } 663 | 664 | private Take!R skipWhitespace() 665 | { 666 | import std.algorithm.searching : startsWith; 667 | 668 | size_t n = 0; 669 | auto ret = m_input.save; 670 | while (!m_input.empty) { 671 | if (m_input.front.among!(' ', '\t')) { 672 | skipChar!false(); 673 | n++; 674 | } else if (m_input.front == '\\') { 675 | if (m_input.save.startsWith("\\\n")) { 676 | skipChar!false(); 677 | skipChar!true(); 678 | n += 2; 679 | } else if (m_input.save.startsWith("\\\r\n")) { 680 | skipChar!false(); 681 | skipChar!true(); 682 | n += 3; 683 | } else break; 684 | } else break; 685 | } 686 | return ret.take(n); 687 | } 688 | 689 | private bool skipOver(string s) 690 | { 691 | while (!m_input.empty && s.length > 0) { 692 | if (m_input.front != s[0]) return false; 693 | s = s[1 .. $]; 694 | m_location.offset++; 695 | m_location.column++; 696 | m_input.popFront(); 697 | } 698 | return s.length == 0; 699 | } 700 | 701 | private void skipLine(bool skip_newline_char = true) 702 | { 703 | while (!m_input.empty && !m_input.front.among!('\r', '\n')) 704 | skipChar!false(); 705 | if (!m_input.empty && skip_newline_char) skipChar!true(); 706 | } 707 | 708 | private void skipChar(bool could_be_eol)() 709 | { 710 | static if (could_be_eol) { 711 | auto c = m_input.front; 712 | m_input.popFront(); 713 | m_location.offset++; 714 | if (c == '\r') { 715 | m_location.line++; 716 | m_location.column = 0; 717 | if (!m_input.empty && m_input.front == '\n') { 718 | m_input.popFront(); 719 | m_location.offset++; 720 | } 721 | } else if (c == '\n') { 722 | m_location.line++; 723 | m_location.column = 0; 724 | } else m_location.column++; 725 | } else { 726 | m_input.popFront(); 727 | m_location.offset++; 728 | m_location.column++; 729 | } 730 | } 731 | 732 | private bool skipDate(Flag!"includeFirstNumber" include_first_number) 733 | { 734 | if (include_first_number) 735 | if (!skipInteger()) return false; 736 | if (!skipOver("/")) return false; 737 | if (!skipInteger()) return false; 738 | if (!skipOver("/")) return false; 739 | if (!skipInteger()) return false; 740 | return true; 741 | } 742 | 743 | private bool skipDuration(Flag!"includeFirstNumber" include_first_number) 744 | { 745 | if (include_first_number) 746 | if (!skipInteger()) return false; 747 | if (!skipOver(":")) return false; 748 | if (!skipInteger()) return false; 749 | if (!skipOver(":")) return false; 750 | if (!skipInteger()) return false; 751 | if (!m_input.empty && m_input.front == '.') { 752 | skipChar!false(); 753 | if (!skipInteger()) return false; 754 | } 755 | return true; 756 | } 757 | 758 | private bool skipTimeOfDay() 759 | { 760 | if (!skipInteger()) return false; 761 | if (!skipOver(":")) return false; 762 | if (!skipInteger()) return false; 763 | if (!m_input.empty && m_input.front != ':') return true; 764 | skipChar!false(); 765 | if (!skipInteger()) return false; 766 | if (!m_input.empty && m_input.front == '.') { 767 | skipChar!false(); 768 | if (!skipInteger()) return false; 769 | } 770 | return true; 771 | } 772 | 773 | private bool skipTimeZone() 774 | { 775 | foreach (i; 0 .. 3) { 776 | auto ch = m_input.front; 777 | if (ch < 'A' || ch > 'Z') return false; 778 | skipChar!false(); 779 | } 780 | 781 | if (m_input.empty || !m_input.front.among!('-', '+')) 782 | return true; 783 | skipChar!false(); 784 | 785 | if (!skipInteger()) return false; 786 | 787 | if (m_input.empty || m_input.front != ':') 788 | return true; 789 | skipChar!false(); 790 | 791 | if (!skipInteger()) return false; 792 | 793 | return true; 794 | } 795 | 796 | private bool skipInteger() 797 | { 798 | if (m_input.empty) return false; 799 | 800 | char ch = m_input.front; 801 | if (ch < '0' || ch > '9') return false; 802 | skipChar!false(); 803 | 804 | while (!m_input.empty) { 805 | ch = m_input.front; 806 | if (ch < '0' || ch > '9') break; 807 | skipChar!false(); 808 | } 809 | 810 | return true; 811 | } 812 | } 813 | 814 | @safe unittest { // single token tests 815 | MultiAppender!(immutable(char)) chapp; 816 | MultiAppender!(immutable(ubyte)) btapp; 817 | 818 | void test(string sdl, TokenType tp, string txt, SDLValue val = SDLValue.null_, string ws = "", bool multiple = false) 819 | { 820 | auto t = SDLangLexer!(typeof(sdl.byCodeUnit))(sdl.byCodeUnit, "test"); 821 | assert(!t.empty); 822 | assert(t.front.type == tp); 823 | assert(t.front.whitespacePrefix.source == ws); 824 | assert(t.front.text.source == txt); 825 | assert(t.front.parseValue(chapp, btapp) == val); 826 | t.popFront(); 827 | assert(multiple || t.front.type == TokenType.eof); 828 | } 829 | 830 | test("\n", TokenType.eol, "\n"); 831 | test("\r", TokenType.eol, "\r"); 832 | test("\r\n", TokenType.eol, "\r\n"); 833 | test("=", TokenType.assign, "="); 834 | test(":", TokenType.namespace, ":"); 835 | test("{", TokenType.blockOpen, "{"); 836 | test("}", TokenType.blockClose, "}"); 837 | test("// foo", TokenType.comment, "// foo"); 838 | test("# foo", TokenType.comment, "# foo"); 839 | test("-- foo", TokenType.comment, "-- foo"); 840 | test("-- foo\n", TokenType.comment, "-- foo\n"); 841 | test("foo", TokenType.identifier, "foo"); 842 | test("foo ", TokenType.identifier, "foo"); 843 | test("foo$.-_ ", TokenType.identifier, "foo$.-_"); 844 | test("föö", TokenType.identifier, "föö"); 845 | test("null", TokenType.null_, "null", SDLValue.null_); 846 | test("true", TokenType.boolean, "true", SDLValue.bool_(true)); 847 | test("false", TokenType.boolean, "false", SDLValue.bool_(false)); 848 | test("on", TokenType.boolean, "on", SDLValue.bool_(true)); 849 | test("off", TokenType.boolean, "off", SDLValue.bool_(false)); 850 | test("on_", TokenType.identifier, "on_"); 851 | test("off_", TokenType.identifier, "off_"); 852 | test("true_", TokenType.identifier, "true_"); 853 | test("false_", TokenType.identifier, "false_"); 854 | test("null_", TokenType.identifier, "null_"); 855 | test("-", TokenType.invalid, "-"); 856 | test("%", TokenType.invalid, "%"); 857 | test("\\", TokenType.invalid, "\\"); 858 | //test("\\\n", TokenType.eof, "\\\n"); 859 | test("`foo`", TokenType.text, "`foo`", SDLValue.text("foo")); 860 | test("`fo\\\"o`", TokenType.text, "`fo\\\"o`", SDLValue.text("fo\\\"o")); 861 | test(`"foo"`, TokenType.text, `"foo"`, SDLValue.text("foo")); 862 | test(`"f\"oo"`, TokenType.text, `"f\"oo"`, SDLValue.text("f\"oo")); 863 | test("\"f \\\n oo\"", TokenType.text, "\"f \\\n oo\"", SDLValue.text("f oo")); 864 | test("[aGVsbG8sIHdvcmxkIQ==]", TokenType.binary, "[aGVsbG8sIHdvcmxkIQ==]", SDLValue.binary(cast(immutable(ubyte)[])"hello, world!")); 865 | test("[aGVsbG8sI \t \n \t HdvcmxkIQ==]", TokenType.binary, "[aGVsbG8sI \t \n \t HdvcmxkIQ==]", SDLValue.binary(cast(immutable(ubyte)[])"hello, world!")); 866 | test("[aGVsbG8sIHdvcmxkIQ]", TokenType.invalid, "[aGVsbG8sIHdvcmxkIQ]"); 867 | test("[aGVsbG8sIHdvcmxk$Q==]", TokenType.invalid, "[aGVsbG8sIHdvcmxk", SDLValue.null_, "", true); 868 | test("5", TokenType.number, "5", SDLValue.int_(5)); 869 | test("123", TokenType.number, "123", SDLValue.int_(123)); 870 | test("-123", TokenType.number, "-123", SDLValue.int_(-123)); 871 | test("4294967295", TokenType.number, "4294967295", SDLValue.int_(-1)); // handle integer overflow so that uint->int->uint conversion doesn't lose information 872 | test("123l", TokenType.number, "123l", SDLValue.long_(123)); 873 | test("123L", TokenType.number, "123L", SDLValue.long_(123)); 874 | test("123.123", TokenType.number, "123.123", SDLValue.double_(123.123)); 875 | test("123.123f", TokenType.number, "123.123f", SDLValue.float_(123.123)); 876 | test("123.123F", TokenType.number, "123.123F", SDLValue.float_(123.123)); 877 | test("123.123d", TokenType.number, "123.123d", SDLValue.double_(123.123)); 878 | test("123.123D", TokenType.number, "123.123D", SDLValue.double_(123.123)); 879 | test("123d", TokenType.number, "123d", SDLValue.double_(123)); 880 | test("123D", TokenType.number, "123D", SDLValue.double_(123)); 881 | test("1.0", TokenType.number, "1.0", SDLValue.double_(1.0)); 882 | test("123.123bd", TokenType.number, "123.123bd"); // TODO 883 | test("123.123BD", TokenType.number, "123.123BD"); // TODO 884 | test("2015/12/06", TokenType.date, "2015/12/06", SDLValue.date(Date(2015, 12, 6))); 885 | test("12:14:34", TokenType.duration, "12:14:34", SDLValue.duration(12.hours + 14.minutes + 34.seconds)); 886 | test("12:14:34.123", TokenType.duration, "12:14:34.123", SDLValue.duration(12.hours + 14.minutes + 34.seconds + 123.msecs)); 887 | test("2d:12:14:34", TokenType.duration, "2d:12:14:34", SDLValue.duration(2.days + 12.hours + 14.minutes + 34.seconds)); 888 | test("2015/12/06 12:00:00.000", TokenType.dateTime, "2015/12/06 12:00:00.000", SDLValue.dateTime(SysTime(DateTime(2015, 12, 6, 12, 0, 0)))); 889 | test("2015/12/06 12:00:00.000-UTC", TokenType.dateTime, "2015/12/06 12:00:00.000-UTC", SDLValue.dateTime(SysTime(DateTime(2015, 12, 6, 12, 0, 0), UTC()))); 890 | test("2015/12/06 12:00:00-GMT-2:30", TokenType.dateTime, "2015/12/06 12:00:00-GMT-2:30", SDLValue.dateTime(SysTime(DateTime(2015, 12, 6, 12, 0, 0), new immutable SimpleTimeZone(-2.hours - 30.minutes)))); 891 | test("2015/12/06 12:00:00-GMT+0:31", TokenType.dateTime, "2015/12/06 12:00:00-GMT+0:31", SDLValue.dateTime(SysTime(DateTime(2015, 12, 6, 12, 0, 0), new immutable SimpleTimeZone(31.minutes)))); 892 | test("2015/12/06 ", TokenType.date, "2015/12/06", SDLValue.date(Date(2015, 12, 6))); 893 | test("2017/11/22 18:00-GMT+00:00", TokenType.dateTime, "2017/11/22 18:00-GMT+00:00", SDLValue.dateTime(SysTime(DateTime(2017, 11, 22, 18, 0, 0), new immutable SimpleTimeZone(0.hours)))); 894 | test("2017/11/22 18:00-gmt+00:00", TokenType.invalid, "2017/11/22 18:00-", SDLValue.null_, "", true); 895 | test("2015/12/06 12:00:00.123", TokenType.dateTime, "2015/12/06 12:00:00.123", SDLValue.dateTime(SysTime(DateTime(2015, 12, 6, 12, 0, 0), 123.msecs))); 896 | test("2015/12/06 12:00:00.123456", TokenType.dateTime, "2015/12/06 12:00:00.123456", SDLValue.dateTime(SysTime(DateTime(2015, 12, 6, 12, 0, 0), 123456.usecs))); 897 | test("2015/12/06 12:00:00.9876543", TokenType.dateTime, "2015/12/06 12:00:00.9876543", SDLValue.dateTime(SysTime(DateTime(2015, 12, 6, 12, 0, 0), 9876543.hnsecs))); 898 | test("2015/12/06 12:00:00.9876543-UTC", TokenType.dateTime, "2015/12/06 12:00:00.9876543-UTC", SDLValue.dateTime(SysTime(DateTime(2015, 12, 6, 12, 0, 0), 9876543.hnsecs, UTC()))); 899 | 900 | test(" {", TokenType.blockOpen, "{", SDLValue.null_, " "); 901 | test("\t {", TokenType.blockOpen, "{", SDLValue.null_, "\t "); 902 | test("0.5\n", TokenType.number, "0.5", SDLValue(0.5), "", true); 903 | 904 | test("\\\n {", TokenType.blockOpen, "{", SDLValue.null_, "\\\n "); 905 | test(" \\\r\n {", TokenType.blockOpen, "{", SDLValue.null_, " \\\r\n "); 906 | } 907 | -------------------------------------------------------------------------------- /source/sdlite/package.d: -------------------------------------------------------------------------------- 1 | module sdlite; 2 | 3 | public import sdlite.ast; 4 | public import sdlite.generator; 5 | public import sdlite.lexer; 6 | public import sdlite.parser; 7 | -------------------------------------------------------------------------------- /source/sdlite/parser.d: -------------------------------------------------------------------------------- 1 | module sdlite.parser; 2 | 3 | import sdlite.ast; 4 | import sdlite.internal : MultiAppender; 5 | import sdlite.lexer; 6 | 7 | void parseSDLDocument(alias NodeHandler, R)(R input, string filename) 8 | { 9 | import std.algorithm.comparison : among; 10 | import std.algorithm.iteration : filter; 11 | 12 | auto tokens = lexSDLang(input, filename) 13 | .filter!(t => t.type != TokenType.comment); 14 | 15 | ParserContext ctx; 16 | 17 | parseNodes!NodeHandler(tokens, ctx, 0); 18 | 19 | while (!tokens.empty) { 20 | if (!tokens.front.type.among(TokenType.eof, TokenType.comment)) 21 | throw new SDLParserException(tokens.front, "Expected end of file"); 22 | tokens.popFront(); 23 | } 24 | } 25 | 26 | @safe unittest { 27 | void clearloc(SDLNode[] nodes) { 28 | foreach (ref n; nodes) { 29 | n.location = Location.init; 30 | clearloc(n.children); 31 | } 32 | } 33 | 34 | void test(string sdl, SDLNode[] expected) 35 | @safe { 36 | SDLNode[] result; 37 | parseSDLDocument!((n) { result ~= n; })(sdl, "test"); 38 | clearloc(result); // ignore location field for the comparison 39 | import std.conv : to; 40 | assert(result == expected, () @trusted { return result.to!string; } ()); 41 | } 42 | 43 | test("foo", [SDLNode("foo")]); 44 | test("foo:bar", [SDLNode("foo:bar")]); 45 | test("foo 123", [SDLNode("foo", 46 | [SDLValue.int_(123)])]); 47 | test("foo null\nbar", [SDLNode("foo", [SDLValue.null_]), SDLNode("bar")]); 48 | test("foo null;bar", [SDLNode("foo", [SDLValue.null_]), SDLNode("bar")]); 49 | test("foo {\n}\n\nbar", [SDLNode("foo"), SDLNode("bar")]); 50 | test("foo bar=123", [SDLNode("foo", null, 51 | [SDLAttribute("bar", SDLValue.int_(123))])]); 52 | test("foo 42 bar=123", [SDLNode("foo", 53 | [SDLValue.int_(42)], 54 | [SDLAttribute("bar", SDLValue.int_(123))])]); 55 | test("foo {\nbar\n}", [SDLNode("foo", null, null, [SDLNode("bar")])]); 56 | test("foo {\nbar\n}\nbaz", [SDLNode("foo", null, null, [SDLNode("bar")]), SDLNode("baz")]); 57 | test("\nfoo", [SDLNode("foo")]); 58 | test("foo //\nbar", [SDLNode("foo"), SDLNode("bar")]); 59 | } 60 | 61 | final class SDLParserException : Exception { 62 | private { 63 | Location m_location; 64 | string m_error; 65 | } 66 | 67 | nothrow: 68 | 69 | this(R)(ref Token!R token, string error, string file = __FILE__, int line = __LINE__, Throwable next_in_chain = null) 70 | { 71 | this(token.location, error, file, line, next_in_chain); 72 | } 73 | 74 | @safe: 75 | this(Location location, string error, string file = __FILE__, int line = __LINE__, Throwable next_in_chain = null) 76 | { 77 | import std.exception : assumeWontThrow; 78 | import std.format : format; 79 | 80 | string msg = format("%s:%s: %s", location.file, location.line+1, error).assumeWontThrow; 81 | 82 | super(msg, file, line, next_in_chain); 83 | 84 | m_location = location; 85 | m_error = error; 86 | } 87 | 88 | @property string error() const { return m_error; } 89 | @property Location location() const { return m_location; } 90 | } 91 | 92 | private void parseNodes(alias NodeHandler, R)(ref R tokens, ref ParserContext ctx, size_t depth) 93 | { 94 | import std.algorithm.comparison : among; 95 | 96 | while (!tokens.empty && tokens.front.type.among(TokenType.eol, TokenType.semicolon)) 97 | tokens.popFront(); 98 | 99 | while (!tokens.empty && !tokens.front.type.among(TokenType.eof, TokenType.blockClose)) { 100 | bool nested; 101 | // NOTE: we need to use @trusted here, because due to the recursive 102 | // call to parseNodes, the compiler fails to infer @safe 103 | auto n = () @trusted { return tokens.parseNode(ctx, depth, nested); } (); 104 | NodeHandler(n); 105 | 106 | if (!nested && !tokens.empty && !tokens.front.type.among(TokenType.eol, TokenType.semicolon, TokenType.eof)) 107 | throwUnexpectedToken(tokens.front, "end of node"); 108 | 109 | while (!tokens.empty && tokens.front.type.among(TokenType.eol, TokenType.semicolon)) 110 | tokens.popFront(); 111 | } 112 | } 113 | 114 | private SDLNode parseNode(R)(ref R tokens, ref ParserContext ctx, size_t depth, out bool is_nested) 115 | { 116 | SDLNode ret; 117 | 118 | bool require_parameters = false; 119 | 120 | ret.location = tokens.front.location; 121 | auto n = tokens.parseQualifiedName(false, ctx); 122 | if (n is null) { 123 | n = "content"; 124 | require_parameters = true; 125 | } 126 | 127 | ret.qualifiedName = n; 128 | ret.values = tokens.parseValues(ctx); 129 | import std.conv; 130 | if (require_parameters && ret.values.length == 0) 131 | throwUnexpectedToken(tokens.front, "values for anonymous node"); 132 | ret.attributes = tokens.parseAttributes(ctx); 133 | 134 | if (!tokens.empty && tokens.front.type == TokenType.blockOpen) { 135 | is_nested = true; 136 | tokens.popFront(); 137 | tokens.skipToken(TokenType.eol); 138 | 139 | if (ctx.nodeAppender.length <= depth) 140 | ctx.nodeAppender.length = depth+1; 141 | tokens.parseNodes!((ref n) @safe { ctx.nodeAppender[depth].put(n); })(ctx, depth+1); 142 | ret.children = ctx.nodeAppender[depth].extractArray; 143 | 144 | if (tokens.empty || tokens.front.type != TokenType.blockClose) 145 | throwUnexpectedToken(tokens.front, "'}'"); 146 | 147 | tokens.popFront(); 148 | if (!tokens.empty && tokens.front.type != TokenType.eof) 149 | tokens.skipToken(TokenType.eol); 150 | } 151 | 152 | return ret; 153 | } 154 | 155 | private SDLAttribute[] parseAttributes(R)(ref R tokens, ref ParserContext ctx) 156 | { 157 | while (!tokens.empty && tokens.front.type == TokenType.identifier) { 158 | SDLAttribute att; 159 | att.qualifiedName = tokens.parseQualifiedName(true, ctx); 160 | tokens.skipToken(TokenType.assign); 161 | if (!tokens.parseValue(att.value, ctx)) 162 | throwUnexpectedToken(tokens.front, "attribute value"); 163 | ctx.attributeAppender.put(att); 164 | } 165 | 166 | return ctx.attributeAppender.extractArray; 167 | } 168 | 169 | private SDLValue[] parseValues(R)(ref R tokens, ref ParserContext ctx) 170 | { 171 | while (!tokens.empty) { 172 | SDLValue v; 173 | if (!tokens.parseValue(v, ctx)) 174 | break; 175 | ctx.valueAppender.put(v); 176 | } 177 | 178 | return ctx.valueAppender.extractArray; 179 | } 180 | 181 | private bool parseValue(R)(ref R tokens, ref SDLValue dst, ref ParserContext ctx) 182 | { 183 | switch (tokens.front.type) { 184 | default: return false; 185 | case TokenType.null_: 186 | case TokenType.text: 187 | case TokenType.binary: 188 | case TokenType.number: 189 | case TokenType.boolean: 190 | case TokenType.dateTime: 191 | case TokenType.date: 192 | case TokenType.duration: 193 | dst = sdlite.lexer.parseValue!(typeof(tokens.front).SourceRange)(tokens.front, ctx.charAppender, ctx.bytesAppender); 194 | tokens.popFront(); 195 | return true; 196 | } 197 | } 198 | 199 | private string parseQualifiedName(R)(ref R tokens, bool required, ref ParserContext ctx) 200 | { 201 | import std.array : array; 202 | import std.exception : assumeUnique; 203 | import std.range : chain; 204 | import std.utf : byCodeUnit; 205 | 206 | if (tokens.front.type != TokenType.identifier) { 207 | if (required) throwUnexpectedToken(tokens.front, "identifier"); 208 | else return null; 209 | } 210 | 211 | foreach (ch; tokens.front.text) 212 | ctx.charAppender.put(ch); 213 | tokens.popFront(); 214 | 215 | if (!tokens.empty && tokens.front.type == TokenType.namespace) { 216 | tokens.popFront(); 217 | if (tokens.empty || tokens.front.type != TokenType.identifier) 218 | throwUnexpectedToken(tokens.front, "identifier"); 219 | 220 | ctx.charAppender.put(':'); 221 | foreach (ch; tokens.front.text) 222 | ctx.charAppender.put(ch); 223 | tokens.popFront(); 224 | } 225 | 226 | return ctx.charAppender.extractArray; 227 | } 228 | 229 | private void skipToken(R)(ref R tokens, scope TokenType[] allowed_types...) 230 | { 231 | import std.algorithm.iteration : map; 232 | import std.algorithm.searching : canFind; 233 | import std.format : format; 234 | 235 | if (tokens.empty) throw new SDLParserException(tokens.front, "Unexpected end of file"); 236 | if (!allowed_types.canFind(tokens.front.type)) { 237 | string msg = allowed_types.length == 1 238 | ? format("Unexpected %s, expected %s", 239 | stringRepresentation(tokens.front), 240 | stringRepresentation(allowed_types[0])) 241 | : format("Unexpected %s, expected any of %(%s/%)", 242 | stringRepresentation(tokens.front), 243 | allowed_types.map!(t => stringRepresentation(t))); 244 | throw new SDLParserException(tokens.front, msg); 245 | } 246 | 247 | tokens.popFront(); 248 | } 249 | 250 | private void throwUnexpectedToken(R)(ref Token!R t, string expected) 251 | { 252 | throw new SDLParserException(t, "Unexpected " ~ stringRepresentation(t) ~ ", expected " ~ expected); 253 | } 254 | 255 | private string stringRepresentation(R)(ref Token!R t) 256 | @safe { 257 | import std.conv : to; 258 | 259 | switch (t.type) with (TokenType) { 260 | case invalid: return "malformed token '" ~ t.text.to!string ~ "'"; 261 | case identifier: return "identifier '"~t.text.to!string~"'"; 262 | default: return stringRepresentation(t.type); 263 | } 264 | } 265 | 266 | private string stringRepresentation(TokenType tp) 267 | @safe { 268 | import std.conv : to; 269 | 270 | final switch (tp) with (TokenType) { 271 | case invalid: return "malformed token"; 272 | case eof: return "end of file"; 273 | case eol: return "end of line"; 274 | case assign: return "'='"; 275 | case namespace: return "':'"; 276 | case blockOpen: return "'{'"; 277 | case blockClose: return "'}'"; 278 | case semicolon: return "';'"; 279 | case comment: return "comment"; 280 | case identifier: return "identifier"; 281 | case null_: return "'null'"; 282 | case text: return "string"; 283 | case binary: return "binary data"; 284 | case number: return "number"; 285 | case boolean: return "Boolean value"; 286 | case dateTime: return "date/time value"; 287 | case date: return "date value"; 288 | case duration: return "duration value"; 289 | } 290 | } 291 | 292 | private struct ParserContext { 293 | MultiAppender!SDLValue valueAppender; 294 | MultiAppender!SDLAttribute attributeAppender; 295 | MultiAppender!(immutable(char)) charAppender; 296 | MultiAppender!(immutable(ubyte)) bytesAppender; 297 | MultiAppender!SDLNode[] nodeAppender; // one for each recursion depth 298 | } 299 | 300 | 301 | @safe unittest { 302 | void test(string code, string error, int line = 1) 303 | { 304 | import std.format : format; 305 | auto msg = format("foo.sdl:%s: %s", line, error); 306 | try { 307 | parseSDLDocument!((n) {})(code, "foo.sdl"); 308 | assert(false, "Expected parsing to fail"); 309 | } 310 | catch (SDLParserException ex) assert(ex.msg == msg, ">"~ex.msg~"< >"~msg~"<"); 311 | catch (Exception e) assert(false, "Unexpected exception type"); 312 | } 313 | 314 | test("foo=bar", "Unexpected '=', expected end of node"); 315 | test("foo bar=15/34/x", "Unexpected malformed token '15/34/', expected attribute value"); 316 | test("foo bar=baz", "Unexpected identifier 'baz', expected attribute value"); 317 | test("foo \"bar\" \\ \"bar\"", "Unexpected malformed token '\\', expected end of node"); 318 | test("foo:", "Unexpected end of file, expected identifier"); 319 | test("foo:\n", "Unexpected end of line, expected identifier"); 320 | test(":", "Unexpected ':', expected values for anonymous node"); 321 | test("{\n}", "Unexpected '{', expected values for anonymous node"); 322 | test(" foo {\n}:", "Unexpected ':', expected end of line", 2); 323 | test(" foo {\n}\n:", "Unexpected ':', expected values for anonymous node", 3); 324 | test("foo { bar }", "Unexpected identifier 'bar', expected end of line"); 325 | } 326 | -------------------------------------------------------------------------------- /travis.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ueo pipefail 4 | 5 | dub test 6 | 7 | if [ ! -z "${COVERAGE:-}" ]; then 8 | dub build --build=docs 9 | 10 | dub test -b unittest-cov 11 | wget https://codecov.io/bash -O codecov.sh 12 | bash codecov.sh 13 | fi 14 | --------------------------------------------------------------------------------