├── examples ├── simple.yml ├── explicit_doc.yml ├── lists.yml ├── map_of_lists.yml ├── maps.yml └── yaml.zig ├── .gitignore ├── .gitmodules ├── .envrc ├── test ├── simple.yaml ├── multi_lib.tbd ├── single_lib.tbd ├── test.zig └── spec.zig ├── shell.nix ├── src ├── lib.zig ├── stringify.zig ├── Tree.zig ├── Tokenizer.zig ├── Yaml.zig ├── Parser.zig ├── Yaml │ └── test.zig └── Parser │ └── test.zig ├── .github ├── FUNDING.yml └── workflows │ └── ci.yml ├── LICENSE ├── flake.nix ├── README.md └── flake.lock /examples/simple.yml: -------------------------------------------------------------------------------- 1 | key: value 2 | other_key: other_value 3 | -------------------------------------------------------------------------------- /examples/explicit_doc.yml: -------------------------------------------------------------------------------- 1 | --- !tapi-tbd 2 | a: b 3 | c : d 4 | ... 5 | -------------------------------------------------------------------------------- /examples/lists.yml: -------------------------------------------------------------------------------- 1 | - a 2 | - b 3 | - c 4 | - d: 5 | - 0 6 | - 1 7 | - 2 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .gyro 2 | .zigmod 3 | deps.zig 4 | zig-cache 5 | .zig-cache 6 | zig-out 7 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "test/yaml-test-suite"] 2 | path = test/yaml-test-suite 3 | url = https://github.com/yaml/yaml-test-suite.git 4 | branch = v2022-01-17 5 | -------------------------------------------------------------------------------- /examples/map_of_lists.yml: -------------------------------------------------------------------------------- 1 | map: 2 | - 0 3 | - 1 4 | - 2 5 | another: 6 | - key: value 7 | - keys: [ a, b, 8 | c, d ] 9 | final: what is that? 10 | -------------------------------------------------------------------------------- /examples/maps.yml: -------------------------------------------------------------------------------- 1 | key1: 2 | key1_1: value1_1 3 | key1_2: value1_2 4 | key2: value2 5 | key3: 6 | key3_1: value3_1 7 | key3_2: value3_2 8 | key3_3: value3_3 9 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | # If we are a computer with nix-shell available, then use that to setup 2 | # the build environment with exactly what we need. 3 | if has nix; then 4 | use nix 5 | fi 6 | -------------------------------------------------------------------------------- /test/simple.yaml: -------------------------------------------------------------------------------- 1 | names: [ John Doe, MacIntosh, Jane Austin ] 2 | numbers: 3 | - 10 4 | - -8 5 | - 6 6 | isyaml: false 7 | hasBoolean: NO 8 | nested: 9 | some: one 10 | wick: john doe 11 | ok: TRUE 12 | finally: [ 8.17, 13 | 19.78 , 17 , 14 | 21 ] 15 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | (import 2 | ( 3 | let 4 | flake-compat = (builtins.fromJSON (builtins.readFile ./flake.lock)).nodes.flake-compat; 5 | in 6 | fetchTarball { 7 | url = "https://github.com/edolstra/flake-compat/archive/${flake-compat.locked.rev}.tar.gz"; 8 | sha256 = flake-compat.locked.narHash; 9 | } 10 | ) 11 | {src = ./.;}) 12 | .shellNix 13 | -------------------------------------------------------------------------------- /src/lib.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | 3 | pub const Parser = @import("Parser.zig"); 4 | pub const Tokenizer = @import("Tokenizer.zig"); 5 | pub const Tree = @import("Tree.zig"); 6 | pub const Yaml = @import("Yaml.zig"); 7 | 8 | pub const stringify = @import("stringify.zig").stringify; 9 | 10 | test { 11 | std.testing.refAllDecls(Parser); 12 | std.testing.refAllDecls(Tokenizer); 13 | std.testing.refAllDecls(Tree); 14 | std.testing.refAllDecls(Yaml); 15 | } 16 | -------------------------------------------------------------------------------- /src/stringify.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | 3 | const Yaml = @import("Yaml.zig"); 4 | 5 | pub fn stringify(gpa: std.mem.Allocator, input: anytype, writer: *std.Io.Writer) Yaml.StringifyError!void { 6 | var arena = std.heap.ArenaAllocator.init(gpa); 7 | defer arena.deinit(); 8 | 9 | const maybe_value = try Yaml.Value.encode(arena.allocator(), input); 10 | 11 | if (maybe_value) |value| { 12 | // TODO should we output as an explicit doc? 13 | // How can allow the user to specify? 14 | try value.stringify(writer, .{}); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [kubkon] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 13 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 14 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | name: Build and test 8 | runs-on: ${{ matrix.os }}-latest 9 | strategy: 10 | fail-fast: false 11 | matrix: 12 | os: [macos, ubuntu, windows] 13 | 14 | steps: 15 | - if: matrix.os == 'windows' 16 | run: git config --global core.autocrlf false 17 | - uses: actions/checkout@v2 18 | with: 19 | submodules: true 20 | - uses: mlugg/setup-zig@v2 21 | with: 22 | version: master 23 | - run: zig fmt --check src 24 | - run: zig build test 25 | - run: zig build run -- examples/lists.yml 26 | 27 | spec-test: 28 | name: YAML Test Suite 29 | runs-on: ${{ matrix.os }}-latest 30 | strategy: 31 | fail-fast: false 32 | matrix: 33 | os: [macos, ubuntu] 34 | 35 | steps: 36 | - uses: actions/checkout@v2 37 | with: 38 | submodules: true 39 | - uses: mlugg/setup-zig@v2 40 | with: 41 | version: master 42 | - run: zig build test -Denable-spec-tests 43 | -------------------------------------------------------------------------------- /test/multi_lib.tbd: -------------------------------------------------------------------------------- 1 | --- !tapi-tbd 2 | tbd-version: 4 3 | targets: [ x86_64-macos ] 4 | uuids: 5 | - target: x86_64-macos 6 | value: F86CC732-D5E4-30B5-AA7D-167DF5EC2708 7 | install-name: '/usr/lib/libSystem.B.dylib' 8 | current-version: 1292.60.1 9 | reexported-libraries: 10 | - targets: [ x86_64-macos ] 11 | libraries: [ '/usr/lib/system/libcache.dylib' ] 12 | exports: 13 | - targets: [ x86_64-macos ] 14 | symbols: [ 'R8289209$_close', 'R8289209$_fork' ] 15 | - targets: [ x86_64-macos ] 16 | symbols: [ ___crashreporter_info__, _libSystem_atfork_child ] 17 | --- !tapi-tbd 18 | tbd-version: 4 19 | targets: [ x86_64-macos ] 20 | uuids: 21 | - target: x86_64-macos 22 | value: 2F7F7303-DB23-359E-85CD-8B2F93223E2A 23 | install-name: '/usr/lib/system/libcache.dylib' 24 | current-version: 83 25 | parent-umbrella: 26 | - targets: [ x86_64-macos ] 27 | umbrella: System 28 | exports: 29 | - targets: [ x86_64-macos ] 30 | symbols: [ _cache_create, _cache_destroy ] 31 | ... 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Jakub Konka 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "Flake for developing zig-yaml"; 3 | 4 | inputs = { 5 | nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11"; 6 | flake-utils.url = "github:numtide/flake-utils"; 7 | zig.url = "github:mitchellh/zig-overlay"; 8 | zls.url = "github:zigtools/zls"; 9 | poop.url = "github:kubkon/poop/nix"; 10 | 11 | # Used for shell.nix 12 | flake-compat = { 13 | url = "github:edolstra/flake-compat"; 14 | flake = false; 15 | }; 16 | }; 17 | 18 | outputs = 19 | { 20 | self, 21 | nixpkgs, 22 | flake-utils, 23 | ... 24 | }@inputs: 25 | let 26 | # Our supported systems are the same supported systems as the Zig binaries 27 | systems = builtins.attrNames inputs.zig.packages; 28 | in 29 | flake-utils.lib.eachSystem systems ( 30 | system: 31 | let 32 | pkgs = nixpkgs.legacyPackages.${system}; 33 | zig = inputs.zig.packages.${system}.master; 34 | zls = inputs.zls.packages.${system}.default.overrideAttrs (old: { 35 | nativeBuildInputs = [ zig ]; 36 | }); 37 | poop = inputs.poop.packages.${system}.default.overrideAttrs (old: { 38 | nativeBuildInputs = [ zig ]; 39 | }); 40 | 41 | linuxSpecific = pkgs.lib.optionals pkgs.stdenv.isLinux [ poop ]; 42 | in 43 | rec { 44 | devShells.default = pkgs.mkShell { 45 | name = "zig-yaml"; 46 | buildInputs = [ 47 | zig 48 | zls 49 | ] ++ linuxSpecific; 50 | }; 51 | 52 | # For compatibility with older versions of the `nix` binary 53 | devShell = self.devShells.${system}.default; 54 | } 55 | ); 56 | } 57 | -------------------------------------------------------------------------------- /test/single_lib.tbd: -------------------------------------------------------------------------------- 1 | --- !tapi-tbd 2 | tbd-version: 4 3 | targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, 4 | arm64e-macos, arm64e-maccatalyst ] 5 | uuids: 6 | - target: x86_64-macos 7 | value: F86CC732-D5E4-30B5-AA7D-167DF5EC2708 8 | - target: x86_64-maccatalyst 9 | value: F86CC732-D5E4-30B5-AA7D-167DF5EC2708 10 | - target: arm64-macos 11 | value: 00000000-0000-0000-0000-000000000000 12 | - target: arm64-maccatalyst 13 | value: 00000000-0000-0000-0000-000000000000 14 | - target: arm64e-macos 15 | value: A17E8744-051E-356E-8619-66F2A6E89AD4 16 | - target: arm64e-maccatalyst 17 | value: A17E8744-051E-356E-8619-66F2A6E89AD4 18 | install-name: '/usr/lib/libSystem.B.dylib' 19 | current-version: 1292.60.1 20 | reexported-libraries: 21 | - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, 22 | arm64e-macos, arm64e-maccatalyst ] 23 | libraries: [ '/usr/lib/system/libcache.dylib', '/usr/lib/system/libcommonCrypto.dylib', 24 | '/usr/lib/system/libcompiler_rt.dylib', '/usr/lib/system/libcopyfile.dylib', 25 | '/usr/lib/system/libxpc.dylib' ] 26 | exports: 27 | - targets: [ x86_64-maccatalyst, x86_64-macos ] 28 | symbols: [ 'R8289209$_close', 'R8289209$_fork', 'R8289209$_fsync', 'R8289209$_getattrlist', 29 | 'R8289209$_write' ] 30 | - targets: [ x86_64-maccatalyst, x86_64-macos, arm64e-maccatalyst, arm64e-macos, 31 | arm64-macos, arm64-maccatalyst ] 32 | symbols: [ ___crashreporter_info__, _libSystem_atfork_child, _libSystem_atfork_parent, 33 | _libSystem_atfork_prepare, _mach_init_routine ] 34 | -------------------------------------------------------------------------------- /examples/yaml.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const assert = std.debug.assert; 3 | const build_options = @import("build_options"); 4 | const Yaml = @import("yaml").Yaml; 5 | 6 | const mem = std.mem; 7 | 8 | var gpa = std.heap.GeneralPurposeAllocator(.{}){}; 9 | 10 | const usage = 11 | \\Usage: yaml 12 | \\ 13 | \\General options: 14 | \\--debug-log [scope] Turn on debugging logs for [scope] (requires program compiled with -Dlog) 15 | \\-h, --help Print this help and exit 16 | \\ 17 | ; 18 | 19 | var log_scopes: std.ArrayList([]const u8) = std.ArrayList([]const u8).init(gpa.allocator()); 20 | 21 | fn logFn( 22 | comptime level: std.log.Level, 23 | comptime scope: @TypeOf(.EnumLiteral), 24 | comptime format: []const u8, 25 | args: anytype, 26 | ) void { 27 | // Hide debug messages unless: 28 | // * logging enabled with `-Dlog`. 29 | // * the --debug-log arg for the scope has been provided 30 | if (@intFromEnum(level) > @intFromEnum(std.options.log_level) or 31 | @intFromEnum(level) > @intFromEnum(std.log.Level.info)) 32 | { 33 | if (!build_options.enable_logging) return; 34 | 35 | const scope_name = @tagName(scope); 36 | for (log_scopes.items) |log_scope| { 37 | if (mem.eql(u8, log_scope, scope_name)) break; 38 | } else return; 39 | } 40 | 41 | // We only recognize 4 log levels in this application. 42 | const level_txt = switch (level) { 43 | .err => "error", 44 | .warn => "warning", 45 | .info => "info", 46 | .debug => "debug", 47 | }; 48 | const prefix1 = level_txt; 49 | const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): "; 50 | 51 | // Print the message to stderr, silently ignoring any errors 52 | std.debug.print(prefix1 ++ prefix2 ++ format ++ "\n", args); 53 | } 54 | 55 | pub const std_options: std.Options = .{ .logFn = logFn }; 56 | 57 | pub fn main() !void { 58 | var arena = std.heap.ArenaAllocator.init(gpa.allocator()); 59 | defer arena.deinit(); 60 | const allocator = arena.allocator(); 61 | 62 | const all_args = try std.process.argsAlloc(allocator); 63 | const args = all_args[1..]; 64 | 65 | const stdout = std.fs.File.stdout(); 66 | const stderr = std.fs.File.stderr(); 67 | 68 | var file_path: ?[]const u8 = null; 69 | var arg_index: usize = 0; 70 | while (arg_index < args.len) : (arg_index += 1) { 71 | if (mem.eql(u8, "-h", args[arg_index]) or mem.eql(u8, "--help", args[arg_index])) { 72 | return stdout.writeAll(usage); 73 | } else if (mem.eql(u8, "--debug-log", args[arg_index])) { 74 | if (arg_index + 1 >= args.len) { 75 | return stderr.writeAll("fatal: expected [scope] after --debug-log\n\n"); 76 | } 77 | arg_index += 1; 78 | if (!build_options.enable_logging) { 79 | try stderr.writeAll("warn: --debug-log will have no effect as program was not built with -Dlog\n\n"); 80 | } else { 81 | try log_scopes.append(args[arg_index]); 82 | } 83 | } else { 84 | file_path = args[arg_index]; 85 | } 86 | } 87 | 88 | if (file_path == null) { 89 | return stderr.writeAll("fatal: no input path to yaml file specified\n\n"); 90 | } 91 | 92 | const file = try std.fs.cwd().openFile(file_path.?, .{}); 93 | defer file.close(); 94 | 95 | const source = try file.readToEndAlloc(allocator, std.math.maxInt(u32)); 96 | 97 | var yaml: Yaml = .{ .source = source }; 98 | defer yaml.deinit(allocator); 99 | 100 | yaml.load(allocator) catch |err| switch (err) { 101 | error.ParseFailure => { 102 | assert(yaml.parse_errors.errorMessageCount() > 0); 103 | yaml.parse_errors.renderToStdErr(.{ .ttyconf = std.io.tty.detectConfig(stderr) }); 104 | return error.ParseFailure; 105 | }, 106 | else => return err, 107 | }; 108 | 109 | var writer = stdout.writer(&[0]u8{}); 110 | try yaml.stringify(&writer.interface); 111 | } 112 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # zig-yaml 2 | 3 | YAML parser for Zig 4 | 5 | ## What is it? 6 | 7 | This lib is meant to serve as a basic (or maybe not?) YAML parser for Zig. It will strive to be YAML 1.2 compatible 8 | but one step at a time. 9 | 10 | This is very much a work-in-progress, so expect things to break on a regular basis. Oh, I'd love to get the 11 | community involved in helping out with this btw! Feel free to fork and submit patches, enhancements, and of course 12 | issues. 13 | 14 | 15 | ## Basic installation 16 | 17 | The library can be installed using the Zig tools. First, you need to fetch the required release of the library into your project. 18 | ``` 19 | zig fetch --save https://github.com/kubkon/zig-yaml/archive/refs/tags/[RELEASE_VERSION].tar.gz 20 | ``` 21 | 22 | It's more convenient to save the library with a desired name, for example, like this (assuming you are targeting latest release of Zig): 23 | ``` 24 | zig fetch --save=yaml https://github.com/kubkon/zig-yaml/archive/refs/tags/0.1.1.tar.gz 25 | ``` 26 | 27 | And then add those lines to your project's `build.zig` file: 28 | ``` 29 | // add that code after "b.installArtifact(exe)" line 30 | const yaml = b.dependency("yaml", .{ 31 | .target = target, 32 | .optimize = optimize, 33 | }); 34 | exe.root_module.addImport("yaml", yaml.module("yaml")); 35 | ``` 36 | 37 | After that, you can simply import the zig-yaml library in your project's code by using `const yaml = @import("yaml");`. 38 | 39 | For zig-0.14, please use any release after `0.1.0`. For pre-zig-0.14 (e.g., zig-0.13), use `0.0.1`. 40 | 41 | ## Basic usage 42 | 43 | The parser currently understands a few YAML primitives such as: 44 | * explicit documents (`---`, `...`) 45 | * mappings (`:`) 46 | * sequences (`-`, `[`, `]`) 47 | 48 | In fact, if you head over to `examples/` dir, you will find YAML examples that have been tested against this 49 | parser. You can also have a look at end-to-end test inputs in `test/` directory. 50 | 51 | If you want to use the parser as a library, add it as a package the usual way, and then: 52 | 53 | ```zig 54 | const std = @import("std"); 55 | const Yaml = @import("yaml").Yaml; 56 | const gpa = std.testing.allocator; 57 | 58 | const source = 59 | \\names: [ John Doe, MacIntosh, Jane Austin ] 60 | \\numbers: 61 | \\ - 10 62 | \\ - -8 63 | \\ - 6 64 | \\nested: 65 | \\ some: one 66 | \\ wick: john doe 67 | \\finally: [ 8.17, 68 | \\ 19.78 , 17 , 69 | \\ 21 ] 70 | ; 71 | 72 | var yaml: Yaml = .{ .source = source }; 73 | defer yaml.deinit(gpa); 74 | ``` 75 | 76 | 1. For untyped, raw representation of YAML, use `Yaml.load`: 77 | 78 | ```zig 79 | try yaml.load(gpa, source); 80 | 81 | try std.testing.expectEqual(untyped.docs.items.len, 1); 82 | 83 | const map = untyped.docs.items[0].map; 84 | try std.testing.expect(map.contains("names")); 85 | try std.testing.expectEqual(map.get("names").?.list.len, 3); 86 | ``` 87 | 88 | 2. For typed representation of YAML, use `Yaml.parse`: 89 | 90 | ```zig 91 | const Simple = struct { 92 | names: []const []const u8, 93 | numbers: []const i16, 94 | nested: struct { 95 | some: []const u8, 96 | wick: []const u8, 97 | }, 98 | finally: [4]f16, 99 | }; 100 | 101 | var arena = std.heap.ArenaAllocator.init(gpa); 102 | defer arena.deinit(); 103 | 104 | const simple = try yaml.parse(arena.allocator(), Simple); 105 | try std.testing.expectEqual(simple.names.len, 3); 106 | ``` 107 | 108 | 3. To convert `Yaml` structure back into text representation, use `Yaml.stringify`: 109 | 110 | ```zig 111 | try yaml.stringify(std.io.getStdOut().writer()); 112 | ``` 113 | 114 | which should write the following output to standard output when run: 115 | 116 | ```sh 117 | names: [ John Doe, MacIntosh, Jane Austin ] 118 | numbers: [ 10, -8, 6 ] 119 | nested: 120 | some: one 121 | wick: john doe 122 | finally: [ 8.17, 19.78, 17, 21 ] 123 | ``` 124 | 125 | ### Error handling 126 | 127 | The library currently reports user-friendly, more informative parsing errors only which are stored out-of-band. 128 | They can be accessed via `Yaml.parse_errors: std.zig.ErrorBundle`, but typically you would only hook them up to 129 | your error reporting setup. 130 | 131 | For example, `example/yaml.zig` binary does it like so: 132 | 133 | ```zig 134 | var yaml: Yaml = .{ .source = source }; 135 | defer yaml.deinit(allocator); 136 | 137 | yaml.load(allocator) catch |err| switch (err) { 138 | error.ParseFailure => { 139 | assert(yaml.parse_errors.errorMessageCount() > 0); 140 | yaml.parse_errors.renderToStdErr(.{ .ttyconf = std.io.tty.detectConfig(std.io.getStdErr()) }); 141 | return error.ParseFailure; 142 | }, 143 | else => return err, 144 | }; 145 | ``` 146 | 147 | ## Running YAML spec test suite 148 | 149 | Remember to clone the repo with submodules first 150 | 151 | ```sh 152 | git clone --recurse-submodules 153 | ``` 154 | 155 | Then, you can run the test suite as follows 156 | 157 | ```sh 158 | zig build test -Denable-spec-tests 159 | ``` 160 | 161 | See also [issue #48](https://github.com/kubkon/zig-yaml/issues/48) for a meta issue tracking failing spec tests. 162 | 163 | Any test that you think of working on and would like to include in the spec tests (that was previously skipped), can be removed from the skipped tests lists in https://github.com/kubkon/zig-yaml/blob/b3cc3a3319ab40fa466a4d5e9c8483267e6ffbee/test/spec.zig#L239-L562 164 | -------------------------------------------------------------------------------- /src/Tree.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const assert = std.debug.assert; 3 | const mem = std.mem; 4 | 5 | const Allocator = mem.Allocator; 6 | const Token = @import("Tokenizer.zig").Token; 7 | const Tree = @This(); 8 | 9 | source: []const u8, 10 | tokens: std.MultiArrayList(TokenWithLineCol).Slice, 11 | docs: []const Node.Index, 12 | nodes: std.MultiArrayList(Node).Slice, 13 | extra: []const u32, 14 | string_bytes: []const u8, 15 | 16 | pub fn deinit(self: *Tree, gpa: Allocator) void { 17 | self.tokens.deinit(gpa); 18 | self.nodes.deinit(gpa); 19 | gpa.free(self.docs); 20 | gpa.free(self.extra); 21 | gpa.free(self.string_bytes); 22 | self.* = undefined; 23 | } 24 | 25 | pub fn nodeTag(tree: Tree, node: Node.Index) Node.Tag { 26 | return tree.nodes.items(.tag)[@intFromEnum(node)]; 27 | } 28 | 29 | pub fn nodeData(tree: Tree, node: Node.Index) Node.Data { 30 | return tree.nodes.items(.data)[@intFromEnum(node)]; 31 | } 32 | 33 | pub fn nodeScope(tree: Tree, node: Node.Index) Node.Scope { 34 | return tree.nodes.items(.scope)[@intFromEnum(node)]; 35 | } 36 | 37 | /// Returns the requested data, as well as the new index which is at the start of the 38 | /// trailers for the object. 39 | pub fn extraData(tree: Tree, comptime T: type, index: Extra) struct { data: T, end: Extra } { 40 | const fields = std.meta.fields(T); 41 | var i = @intFromEnum(index); 42 | var result: T = undefined; 43 | inline for (fields) |field| { 44 | @field(result, field.name) = switch (field.type) { 45 | u32 => tree.extra[i], 46 | i32 => @bitCast(tree.extra[i]), 47 | Node.Index, Node.OptionalIndex, Token.Index => @enumFromInt(tree.extra[i]), 48 | else => @compileError("bad field type: " ++ @typeName(field.type)), 49 | }; 50 | i += 1; 51 | } 52 | return .{ 53 | .data = result, 54 | .end = @enumFromInt(i), 55 | }; 56 | } 57 | 58 | /// Returns the directive metadata if present. 59 | pub fn directive(self: Tree, node_index: Node.Index) ?[]const u8 { 60 | const tag = self.nodeTag(node_index); 61 | switch (tag) { 62 | .doc => return null, 63 | .doc_with_directive => { 64 | const data = self.nodeData(node_index).doc_with_directive; 65 | return self.rawString(data.directive, data.directive); 66 | }, 67 | else => unreachable, 68 | } 69 | } 70 | 71 | /// Returns the raw string such that it matches the range [start, end) in the Token stream. 72 | pub fn rawString(self: Tree, start: Token.Index, end: Token.Index) []const u8 { 73 | const start_token = self.token(start); 74 | const end_token = self.token(end); 75 | return self.source[start_token.loc.start..end_token.loc.end]; 76 | } 77 | 78 | pub fn token(self: Tree, index: Token.Index) Token { 79 | return self.tokens.items(.token)[@intFromEnum(index)]; 80 | } 81 | 82 | pub const Node = struct { 83 | tag: Tag, 84 | scope: Scope, 85 | data: Data, 86 | 87 | pub const Tag = enum(u8) { 88 | /// Document with no extra metadata. 89 | /// Comprises an index into another Node. 90 | /// Payload is maybe_value. 91 | doc, 92 | 93 | /// Document with directive. 94 | /// Payload is doc_with_directive. 95 | doc_with_directive, 96 | 97 | /// Map with a single key-value pair. 98 | /// Payload is map. 99 | map_single, 100 | 101 | /// Map with more than one key-value pair. 102 | /// Comprises an index into extras where payload is Map. 103 | /// Payload is extra. 104 | map_many, 105 | 106 | /// Empty list. 107 | /// Payload is unused. 108 | list_empty, 109 | 110 | /// List with one element. 111 | /// Payload is value. 112 | list_one, 113 | 114 | /// List with two elements. 115 | /// Payload is list. 116 | list_two, 117 | 118 | /// List of more than 2 elements. 119 | /// Comprises an index into extras where payload is List. 120 | /// Payload is extra. 121 | list_many, 122 | 123 | /// String that didn't require any preprocessing. 124 | /// Value is encoded directly as scope. 125 | /// Payload is unused. 126 | value, 127 | 128 | /// String that required preprocessing such as a quoted string. 129 | /// Payload is string. 130 | string_value, 131 | }; 132 | 133 | /// Describes the Token range that encapsulates this Node. 134 | pub const Scope = struct { 135 | start: Token.Index, 136 | end: Token.Index, 137 | 138 | pub fn rawString(scope: Scope, tree: Tree) []const u8 { 139 | return tree.rawString(scope.start, scope.end); 140 | } 141 | }; 142 | 143 | pub const Data = union { 144 | /// Node index. 145 | node: Index, 146 | 147 | /// Optional Node index. 148 | maybe_node: OptionalIndex, 149 | 150 | /// Document with a directive metadata. 151 | doc_with_directive: struct { 152 | maybe_node: OptionalIndex, 153 | directive: Token.Index, 154 | }, 155 | 156 | /// Map with exactly one key-value pair. 157 | map: struct { 158 | key: Token.Index, 159 | maybe_node: OptionalIndex, 160 | }, 161 | 162 | /// List with exactly two elements. 163 | list: struct { 164 | el1: Index, 165 | el2: Index, 166 | }, 167 | 168 | /// Index and length into the string table. 169 | string: String, 170 | 171 | /// Index into extra array. 172 | extra: Extra, 173 | }; 174 | 175 | pub const Index = enum(u32) { 176 | _, 177 | 178 | pub fn toOptional(ind: Index) OptionalIndex { 179 | const result: OptionalIndex = @enumFromInt(@intFromEnum(ind)); 180 | assert(result != .none); 181 | return result; 182 | } 183 | }; 184 | 185 | pub const OptionalIndex = enum(u32) { 186 | none = std.math.maxInt(u32), 187 | _, 188 | 189 | pub fn unwrap(opt: OptionalIndex) ?Index { 190 | if (opt == .none) return null; 191 | return @enumFromInt(@intFromEnum(opt)); 192 | } 193 | }; 194 | 195 | // Make sure we don't accidentally make nodes bigger than expected. 196 | // Note that in safety builds, Zig is allowed to insert a secret field for safety checks. 197 | comptime { 198 | if (!std.debug.runtime_safety) { 199 | assert(@sizeOf(Data) == 8); 200 | } 201 | } 202 | }; 203 | 204 | /// Index into extra array. 205 | pub const Extra = enum(u32) { 206 | _, 207 | }; 208 | 209 | /// Trailing is a list of MapEntries. 210 | pub const Map = struct { 211 | map_len: u32, 212 | 213 | pub const Entry = struct { 214 | key: Token.Index, 215 | maybe_node: Node.OptionalIndex, 216 | }; 217 | }; 218 | 219 | /// Trailing is a list of Node indexes. 220 | pub const List = struct { 221 | list_len: u32, 222 | 223 | pub const Entry = struct { 224 | node: Node.Index, 225 | }; 226 | }; 227 | 228 | /// Index and length into string table. 229 | pub const String = struct { 230 | index: Index, 231 | len: u32, 232 | 233 | pub const Index = enum(u32) { 234 | _, 235 | }; 236 | 237 | pub fn slice(str: String, tree: Tree) []const u8 { 238 | return tree.string_bytes[@intFromEnum(str.index)..][0..str.len]; 239 | } 240 | }; 241 | 242 | /// Tracked line-column information for each Token. 243 | pub const LineCol = struct { 244 | line: u32, 245 | col: u32, 246 | }; 247 | 248 | /// Token with line-column information. 249 | pub const TokenWithLineCol = struct { 250 | token: Token, 251 | line_col: LineCol, 252 | }; 253 | 254 | test { 255 | std.testing.refAllDecls(@This()); 256 | } 257 | -------------------------------------------------------------------------------- /test/test.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const mem = std.mem; 3 | const testing = std.testing; 4 | 5 | const Allocator = mem.Allocator; 6 | const Arena = std.heap.ArenaAllocator; 7 | const Yaml = @import("yaml").Yaml; 8 | 9 | const gpa = testing.allocator; 10 | 11 | fn loadFromFile(file_path: []const u8) !Yaml { 12 | const file = try std.fs.cwd().openFile(file_path, .{}); 13 | defer file.close(); 14 | 15 | const source = try file.readToEndAlloc(gpa, std.math.maxInt(u32)); 16 | defer gpa.free(source); 17 | 18 | var yaml: Yaml = .{ .source = source }; 19 | errdefer yaml.deinit(gpa); 20 | try yaml.load(gpa); 21 | return yaml; 22 | } 23 | 24 | test "simple" { 25 | const Simple = struct { 26 | names: []const []const u8, 27 | numbers: []const i16, 28 | nested: struct { 29 | some: []const u8, 30 | wick: []const u8, 31 | ok: bool, 32 | }, 33 | finally: [4]f16, 34 | isyaml: bool, 35 | hasBoolean: bool, 36 | 37 | pub fn eql(self: @This(), other: @This()) bool { 38 | if (self.names.len != other.names.len) return false; 39 | if (self.numbers.len != other.numbers.len) return false; 40 | if (self.finally.len != other.finally.len) return false; 41 | 42 | for (self.names, 0..) |lhs, i| { 43 | if (!mem.eql(u8, lhs, other.names[i])) return false; 44 | } 45 | 46 | for (self.numbers, 0..) |lhs, i| { 47 | if (lhs != other.numbers[i]) return false; 48 | } 49 | 50 | for (self.finally, 0..) |lhs, i| { 51 | if (lhs != other.finally[i]) return false; 52 | } 53 | 54 | if (!mem.eql(u8, self.nested.some, other.nested.some)) return false; 55 | if (!mem.eql(u8, self.nested.wick, other.nested.wick)) return false; 56 | 57 | return true; 58 | } 59 | }; 60 | 61 | var parsed = try loadFromFile("test/simple.yaml"); 62 | defer parsed.deinit(gpa); 63 | 64 | var arena = Arena.init(gpa); 65 | defer arena.deinit(); 66 | 67 | const result = try parsed.parse(arena.allocator(), Simple); 68 | const expected = Simple{ 69 | .names = &[_][]const u8{ "John Doe", "MacIntosh", "Jane Austin" }, 70 | .numbers = &[_]i16{ 10, -8, 6 }, 71 | .nested = .{ 72 | .some = "one", 73 | .wick = "john doe", 74 | .ok = true, 75 | }, 76 | .isyaml = false, 77 | .hasBoolean = false, 78 | .finally = [_]f16{ 8.17, 19.78, 17, 21 }, 79 | }; 80 | try testing.expect(result.eql(expected)); 81 | } 82 | 83 | const LibTbd = struct { 84 | tbd_version: u3, 85 | targets: []const []const u8, 86 | uuids: []const struct { 87 | target: []const u8, 88 | value: []const u8, 89 | }, 90 | install_name: []const u8, 91 | current_version: union(enum) { 92 | int: usize, 93 | string: []const u8, 94 | }, 95 | reexported_libraries: ?[]const struct { 96 | targets: []const []const u8, 97 | libraries: []const []const u8, 98 | }, 99 | parent_umbrella: ?[]const struct { 100 | targets: []const []const u8, 101 | umbrella: []const u8, 102 | }, 103 | exports: []const struct { 104 | targets: []const []const u8, 105 | symbols: []const []const u8, 106 | }, 107 | 108 | pub fn eql(self: LibTbd, other: LibTbd) bool { 109 | if (self.tbd_version != other.tbd_version) return false; 110 | if (self.targets.len != other.targets.len) return false; 111 | 112 | for (self.targets, 0..) |target, i| { 113 | if (!mem.eql(u8, target, other.targets[i])) return false; 114 | } 115 | 116 | if (!mem.eql(u8, self.install_name, other.install_name)) return false; 117 | 118 | switch (self.current_version) { 119 | .string => |string| { 120 | if (other.current_version != .string) return false; 121 | if (!mem.eql(u8, string, other.current_version.string)) return false; 122 | }, 123 | .int => |int| { 124 | if (other.current_version != .int) return false; 125 | if (int != other.current_version.int) return false; 126 | }, 127 | } 128 | 129 | if (self.reexported_libraries) |reexported_libraries| { 130 | const o_reexported_libraries = other.reexported_libraries orelse return false; 131 | 132 | if (reexported_libraries.len != o_reexported_libraries.len) return false; 133 | 134 | for (reexported_libraries, 0..) |reexport, i| { 135 | const o_reexport = o_reexported_libraries[i]; 136 | if (reexport.targets.len != o_reexport.targets.len) return false; 137 | if (reexport.libraries.len != o_reexport.libraries.len) return false; 138 | 139 | for (reexport.targets, 0..) |target, j| { 140 | const o_target = o_reexport.targets[j]; 141 | if (!mem.eql(u8, target, o_target)) return false; 142 | } 143 | 144 | for (reexport.libraries, 0..) |library, j| { 145 | const o_library = o_reexport.libraries[j]; 146 | if (!mem.eql(u8, library, o_library)) return false; 147 | } 148 | } 149 | } 150 | 151 | if (self.parent_umbrella) |parent_umbrella| { 152 | const o_parent_umbrella = other.parent_umbrella orelse return false; 153 | 154 | if (parent_umbrella.len != o_parent_umbrella.len) return false; 155 | 156 | for (parent_umbrella, 0..) |pumbrella, i| { 157 | const o_pumbrella = o_parent_umbrella[i]; 158 | if (pumbrella.targets.len != o_pumbrella.targets.len) return false; 159 | 160 | for (pumbrella.targets, 0..) |target, j| { 161 | const o_target = o_pumbrella.targets[j]; 162 | if (!mem.eql(u8, target, o_target)) return false; 163 | } 164 | 165 | if (!mem.eql(u8, pumbrella.umbrella, o_pumbrella.umbrella)) return false; 166 | } 167 | } 168 | 169 | if (self.exports.len != other.exports.len) return false; 170 | 171 | for (self.exports, 0..) |exp, i| { 172 | const o_exp = other.exports[i]; 173 | if (exp.targets.len != o_exp.targets.len) return false; 174 | if (exp.symbols.len != o_exp.symbols.len) return false; 175 | 176 | for (exp.targets, 0..) |target, j| { 177 | const o_target = o_exp.targets[j]; 178 | if (!mem.eql(u8, target, o_target)) return false; 179 | } 180 | 181 | for (exp.symbols, 0..) |symbol, j| { 182 | const o_symbol = o_exp.symbols[j]; 183 | if (!mem.eql(u8, symbol, o_symbol)) return false; 184 | } 185 | } 186 | 187 | return true; 188 | } 189 | }; 190 | 191 | test "single lib tbd" { 192 | var parsed = try loadFromFile("test/single_lib.tbd"); 193 | defer parsed.deinit(gpa); 194 | 195 | var arena = Arena.init(gpa); 196 | defer arena.deinit(); 197 | 198 | const result = try parsed.parse(arena.allocator(), LibTbd); 199 | const expected = LibTbd{ 200 | .tbd_version = 4, 201 | .targets = &[_][]const u8{ 202 | "x86_64-macos", 203 | "x86_64-maccatalyst", 204 | "arm64-macos", 205 | "arm64-maccatalyst", 206 | "arm64e-macos", 207 | "arm64e-maccatalyst", 208 | }, 209 | .uuids = &.{ 210 | .{ .target = "x86_64-macos", .value = "F86CC732-D5E4-30B5-AA7D-167DF5EC2708" }, 211 | .{ .target = "x86_64-maccatalyst", .value = "F86CC732-D5E4-30B5-AA7D-167DF5EC2708" }, 212 | .{ .target = "arm64-macos", .value = "00000000-0000-0000-0000-000000000000" }, 213 | .{ .target = "arm64-maccatalyst", .value = "00000000-0000-0000-0000-000000000000" }, 214 | .{ .target = "arm64e-macos", .value = "A17E8744-051E-356E-8619-66F2A6E89AD4" }, 215 | .{ .target = "arm64e-maccatalyst", .value = "A17E8744-051E-356E-8619-66F2A6E89AD4" }, 216 | }, 217 | .install_name = "/usr/lib/libSystem.B.dylib", 218 | .current_version = .{ .string = "1292.60.1" }, 219 | .reexported_libraries = &.{ 220 | .{ 221 | .targets = &.{ 222 | "x86_64-macos", 223 | "x86_64-maccatalyst", 224 | "arm64-macos", 225 | "arm64-maccatalyst", 226 | "arm64e-macos", 227 | "arm64e-maccatalyst", 228 | }, 229 | .libraries = &.{ 230 | "/usr/lib/system/libcache.dylib", "/usr/lib/system/libcommonCrypto.dylib", 231 | "/usr/lib/system/libcompiler_rt.dylib", "/usr/lib/system/libcopyfile.dylib", 232 | "/usr/lib/system/libxpc.dylib", 233 | }, 234 | }, 235 | }, 236 | .exports = &.{ 237 | .{ 238 | .targets = &.{ 239 | "x86_64-maccatalyst", 240 | "x86_64-macos", 241 | }, 242 | .symbols = &.{ 243 | "R8289209$_close", "R8289209$_fork", "R8289209$_fsync", "R8289209$_getattrlist", 244 | "R8289209$_write", 245 | }, 246 | }, 247 | .{ 248 | .targets = &.{ 249 | "x86_64-maccatalyst", 250 | "x86_64-macos", 251 | "arm64e-maccatalyst", 252 | "arm64e-macos", 253 | "arm64-macos", 254 | "arm64-maccatalyst", 255 | }, 256 | .symbols = &.{ 257 | "___crashreporter_info__", "_libSystem_atfork_child", "_libSystem_atfork_parent", 258 | "_libSystem_atfork_prepare", "_mach_init_routine", 259 | }, 260 | }, 261 | }, 262 | .parent_umbrella = null, 263 | }; 264 | try testing.expect(result.eql(expected)); 265 | } 266 | 267 | test "multi lib tbd" { 268 | var parsed = try loadFromFile("test/multi_lib.tbd"); 269 | defer parsed.deinit(gpa); 270 | 271 | var arena = Arena.init(gpa); 272 | defer arena.deinit(); 273 | 274 | const result = try parsed.parse(arena.allocator(), []LibTbd); 275 | const expected = &[_]LibTbd{ 276 | .{ 277 | .tbd_version = 4, 278 | .targets = &[_][]const u8{"x86_64-macos"}, 279 | .uuids = &.{ 280 | .{ .target = "x86_64-macos", .value = "F86CC732-D5E4-30B5-AA7D-167DF5EC2708" }, 281 | }, 282 | .install_name = "/usr/lib/libSystem.B.dylib", 283 | .current_version = .{ .string = "1292.60.1" }, 284 | .reexported_libraries = &.{ 285 | .{ 286 | .targets = &.{"x86_64-macos"}, 287 | .libraries = &.{"/usr/lib/system/libcache.dylib"}, 288 | }, 289 | }, 290 | .exports = &.{ 291 | .{ 292 | .targets = &.{"x86_64-macos"}, 293 | .symbols = &.{ "R8289209$_close", "R8289209$_fork" }, 294 | }, 295 | .{ 296 | .targets = &.{"x86_64-macos"}, 297 | .symbols = &.{ "___crashreporter_info__", "_libSystem_atfork_child" }, 298 | }, 299 | }, 300 | .parent_umbrella = null, 301 | }, 302 | .{ 303 | .tbd_version = 4, 304 | .targets = &[_][]const u8{"x86_64-macos"}, 305 | .uuids = &.{ 306 | .{ .target = "x86_64-macos", .value = "2F7F7303-DB23-359E-85CD-8B2F93223E2A" }, 307 | }, 308 | .install_name = "/usr/lib/system/libcache.dylib", 309 | .current_version = .{ .int = 83 }, 310 | .parent_umbrella = &.{ 311 | .{ 312 | .targets = &.{"x86_64-macos"}, 313 | .umbrella = "System", 314 | }, 315 | }, 316 | .exports = &.{ 317 | .{ 318 | .targets = &.{"x86_64-macos"}, 319 | .symbols = &.{ "_cache_create", "_cache_destroy" }, 320 | }, 321 | }, 322 | .reexported_libraries = null, 323 | }, 324 | }; 325 | 326 | for (result, 0..) |lib, i| { 327 | try testing.expect(lib.eql(expected[i])); 328 | } 329 | } 330 | -------------------------------------------------------------------------------- /src/Tokenizer.zig: -------------------------------------------------------------------------------- 1 | const Tokenizer = @This(); 2 | 3 | const std = @import("std"); 4 | const log = std.log.scoped(.tokenizer); 5 | const testing = std.testing; 6 | 7 | buffer: []const u8, 8 | index: usize = 0, 9 | in_flow: usize = 0, 10 | 11 | pub const Token = struct { 12 | id: Id, 13 | loc: Loc, 14 | 15 | pub const Loc = struct { 16 | start: usize, 17 | end: usize, 18 | }; 19 | 20 | pub const Id = enum { 21 | // zig fmt: off 22 | eof, 23 | 24 | new_line, 25 | doc_start, // --- 26 | doc_end, // ... 27 | seq_item_ind, // - 28 | map_value_ind, // : 29 | flow_map_start, // { 30 | flow_map_end, // } 31 | flow_seq_start, // [ 32 | flow_seq_end, // ] 33 | 34 | comma, 35 | space, 36 | tab, 37 | comment, // # 38 | alias, // * 39 | anchor, // & 40 | tag, // ! 41 | 42 | single_quoted, // '...' 43 | double_quoted, // "..." 44 | literal, 45 | // zig fmt: on 46 | }; 47 | 48 | pub const Index = enum(u32) { 49 | _, 50 | }; 51 | }; 52 | 53 | pub const TokenIterator = struct { 54 | buffer: []const Token, 55 | pos: Token.Index = @enumFromInt(0), 56 | 57 | pub fn next(self: *TokenIterator) ?Token { 58 | const token = self.peek() orelse return null; 59 | self.pos = @enumFromInt(@intFromEnum(self.pos) + 1); 60 | return token; 61 | } 62 | 63 | pub fn peek(self: TokenIterator) ?Token { 64 | const pos = @intFromEnum(self.pos); 65 | if (pos >= self.buffer.len) return null; 66 | return self.buffer[pos]; 67 | } 68 | 69 | pub fn reset(self: *TokenIterator) void { 70 | self.pos = @enumFromInt(0); 71 | } 72 | 73 | pub fn seekTo(self: *TokenIterator, pos: Token.Index) void { 74 | self.pos = pos; 75 | } 76 | 77 | pub fn seekBy(self: *TokenIterator, offset: isize) void { 78 | var pos = @intFromEnum(self.pos); 79 | if (offset < 0) { 80 | pos -|= @intCast(@abs(offset)); 81 | } else { 82 | pos +|= @intCast(@as(usize, @bitCast(offset))); 83 | } 84 | self.pos = @enumFromInt(pos); 85 | } 86 | }; 87 | 88 | fn stringMatchesPattern(comptime pattern: []const u8, slice: []const u8) bool { 89 | comptime var count: usize = 0; 90 | inline while (count < pattern.len) : (count += 1) { 91 | if (count >= slice.len) return false; 92 | const c = slice[count]; 93 | if (pattern[count] != c) return false; 94 | } 95 | return true; 96 | } 97 | 98 | fn matchesPattern(self: Tokenizer, comptime pattern: []const u8) bool { 99 | return stringMatchesPattern(pattern, self.buffer[self.index..]); 100 | } 101 | 102 | pub fn next(self: *Tokenizer) Token { 103 | var result = Token{ 104 | .id = .eof, 105 | .loc = .{ 106 | .start = self.index, 107 | .end = undefined, 108 | }, 109 | }; 110 | 111 | var state: enum { 112 | start, 113 | new_line, 114 | space, 115 | tab, 116 | comment, 117 | single_quoted, 118 | double_quoted, 119 | literal, 120 | } = .start; 121 | 122 | while (self.index < self.buffer.len) : (self.index += 1) { 123 | const c = self.buffer[self.index]; 124 | switch (state) { 125 | .start => switch (c) { 126 | ' ' => { 127 | state = .space; 128 | }, 129 | '\t' => { 130 | state = .tab; 131 | }, 132 | '\n' => { 133 | result.id = .new_line; 134 | self.index += 1; 135 | break; 136 | }, 137 | '\r' => { 138 | state = .new_line; 139 | }, 140 | 141 | '-' => if (self.matchesPattern("---")) { 142 | result.id = .doc_start; 143 | self.index += "---".len; 144 | break; 145 | } else if (self.matchesPattern("- ")) { 146 | result.id = .seq_item_ind; 147 | self.index += "- ".len; 148 | break; 149 | } else if (self.matchesPattern("-\n")) { 150 | result.id = .seq_item_ind; 151 | // we do not skip the newline 152 | self.index += "-".len; 153 | break; 154 | } else { 155 | state = .literal; 156 | }, 157 | 158 | '.' => if (self.matchesPattern("...")) { 159 | result.id = .doc_end; 160 | self.index += "...".len; 161 | break; 162 | } else { 163 | state = .literal; 164 | }, 165 | 166 | ',' => { 167 | result.id = .comma; 168 | self.index += 1; 169 | break; 170 | }, 171 | '#' => { 172 | result.id = .comment; 173 | state = .comment; 174 | }, 175 | '*' => { 176 | result.id = .alias; 177 | self.index += 1; 178 | break; 179 | }, 180 | '&' => { 181 | result.id = .anchor; 182 | self.index += 1; 183 | break; 184 | }, 185 | '!' => { 186 | result.id = .tag; 187 | self.index += 1; 188 | break; 189 | }, 190 | '[' => { 191 | result.id = .flow_seq_start; 192 | self.index += 1; 193 | self.in_flow += 1; 194 | break; 195 | }, 196 | ']' => { 197 | result.id = .flow_seq_end; 198 | self.index += 1; 199 | self.in_flow -|= 1; 200 | break; 201 | }, 202 | ':' => { 203 | result.id = .map_value_ind; 204 | self.index += 1; 205 | break; 206 | }, 207 | '{' => { 208 | result.id = .flow_map_start; 209 | self.index += 1; 210 | self.in_flow += 1; 211 | break; 212 | }, 213 | '}' => { 214 | result.id = .flow_map_end; 215 | self.index += 1; 216 | self.in_flow -|= 1; 217 | break; 218 | }, 219 | '\'' => { 220 | state = .single_quoted; 221 | }, 222 | '"' => { 223 | state = .double_quoted; 224 | }, 225 | else => { 226 | state = .literal; 227 | }, 228 | }, 229 | 230 | .comment => switch (c) { 231 | '\r', '\n' => { 232 | result.id = .comment; 233 | break; 234 | }, 235 | else => {}, 236 | }, 237 | 238 | .space => switch (c) { 239 | ' ' => {}, 240 | else => { 241 | result.id = .space; 242 | break; 243 | }, 244 | }, 245 | 246 | .tab => switch (c) { 247 | '\t' => {}, 248 | else => { 249 | result.id = .tab; 250 | break; 251 | }, 252 | }, 253 | 254 | .new_line => switch (c) { 255 | '\n' => { 256 | result.id = .new_line; 257 | self.index += 1; 258 | break; 259 | }, 260 | else => {}, // TODO this should be an error condition 261 | }, 262 | 263 | .single_quoted => switch (c) { 264 | '\'' => if (!self.matchesPattern("''")) { 265 | result.id = .single_quoted; 266 | self.index += 1; 267 | break; 268 | } else { 269 | self.index += "''".len - 1; 270 | }, 271 | else => {}, 272 | }, 273 | 274 | .double_quoted => switch (c) { 275 | '"' => { 276 | if (stringMatchesPattern("\\", self.buffer[self.index - 1 ..])) { 277 | self.index += 1; 278 | } else { 279 | result.id = .double_quoted; 280 | self.index += 1; 281 | break; 282 | } 283 | }, 284 | else => {}, 285 | }, 286 | 287 | .literal => switch (c) { 288 | '\r', '\n', ' ', '\'', '"', ']', '}' => { 289 | result.id = .literal; 290 | break; 291 | }, 292 | ',', '[', '{' => { 293 | result.id = .literal; 294 | if (self.in_flow > 0) { 295 | break; 296 | } 297 | }, 298 | ':' => { 299 | result.id = .literal; 300 | if (self.matchesPattern(": ") or self.matchesPattern(":\n") or self.matchesPattern(":\r")) { 301 | break; 302 | } 303 | }, 304 | else => { 305 | result.id = .literal; 306 | }, 307 | }, 308 | } 309 | } 310 | 311 | if (self.index >= self.buffer.len) { 312 | switch (state) { 313 | .literal => { 314 | result.id = .literal; 315 | }, 316 | else => {}, 317 | } 318 | } 319 | 320 | result.loc.end = self.index; 321 | 322 | log.debug("{any}", .{result}); 323 | log.debug(" | {s}", .{self.buffer[result.loc.start..result.loc.end]}); 324 | 325 | return result; 326 | } 327 | 328 | fn testExpected(source: []const u8, expected: []const Token.Id) !void { 329 | var tokenizer = Tokenizer{ 330 | .buffer = source, 331 | }; 332 | 333 | var given = std.array_list.Managed(Token.Id).init(testing.allocator); 334 | defer given.deinit(); 335 | 336 | while (true) { 337 | const token = tokenizer.next(); 338 | try given.append(token.id); 339 | if (token.id == .eof) break; 340 | } 341 | 342 | try testing.expectEqualSlices(Token.Id, expected, given.items); 343 | } 344 | 345 | test { 346 | std.testing.refAllDecls(@This()); 347 | } 348 | 349 | test "empty doc" { 350 | try testExpected("", &[_]Token.Id{.eof}); 351 | } 352 | 353 | test "empty doc with explicit markers" { 354 | try testExpected( 355 | \\--- 356 | \\... 357 | , &[_]Token.Id{ 358 | .doc_start, .new_line, .doc_end, .eof, 359 | }); 360 | } 361 | 362 | test "empty doc with explicit markers and a directive" { 363 | try testExpected( 364 | \\--- !tbd-v1 365 | \\... 366 | , &[_]Token.Id{ 367 | .doc_start, 368 | .space, 369 | .tag, 370 | .literal, 371 | .new_line, 372 | .doc_end, 373 | .eof, 374 | }); 375 | } 376 | 377 | test "sequence of values" { 378 | try testExpected( 379 | \\- 0 380 | \\- 1 381 | \\- 2 382 | , &[_]Token.Id{ 383 | .seq_item_ind, 384 | .literal, 385 | .new_line, 386 | .seq_item_ind, 387 | .literal, 388 | .new_line, 389 | .seq_item_ind, 390 | .literal, 391 | .eof, 392 | }); 393 | } 394 | 395 | test "sequence of sequences" { 396 | try testExpected( 397 | \\- [ val1, val2] 398 | \\- [val3, val4 ] 399 | , &[_]Token.Id{ 400 | .seq_item_ind, 401 | .flow_seq_start, 402 | .space, 403 | .literal, 404 | .comma, 405 | .space, 406 | .literal, 407 | .flow_seq_end, 408 | .new_line, 409 | .seq_item_ind, 410 | .flow_seq_start, 411 | .literal, 412 | .comma, 413 | .space, 414 | .literal, 415 | .space, 416 | .flow_seq_end, 417 | .eof, 418 | }); 419 | } 420 | 421 | test "mappings" { 422 | try testExpected( 423 | \\key1: value1 424 | \\key2: value2 425 | , &[_]Token.Id{ 426 | .literal, 427 | .map_value_ind, 428 | .space, 429 | .literal, 430 | .new_line, 431 | .literal, 432 | .map_value_ind, 433 | .space, 434 | .literal, 435 | .eof, 436 | }); 437 | } 438 | 439 | test "inline mapped sequence of values" { 440 | try testExpected( 441 | \\key : [ val1, 442 | \\ val2 ] 443 | , &[_]Token.Id{ 444 | .literal, 445 | .space, 446 | .map_value_ind, 447 | .space, 448 | .flow_seq_start, 449 | .space, 450 | .literal, 451 | .comma, 452 | .space, 453 | .new_line, 454 | .space, 455 | .literal, 456 | .space, 457 | .flow_seq_end, 458 | .eof, 459 | }); 460 | } 461 | 462 | test "part of tbd" { 463 | try testExpected( 464 | \\--- !tapi-tbd 465 | \\tbd-version: 4 466 | \\targets: [ x86_64-macos ] 467 | \\ 468 | \\uuids: 469 | \\ - target: x86_64-macos 470 | \\ value: F86CC732-D5E4-30B5-AA7D-167DF5EC2708 471 | \\ 472 | \\install-name: '/usr/lib/libSystem.B.dylib' 473 | \\... 474 | , &[_]Token.Id{ 475 | .doc_start, 476 | .space, 477 | .tag, 478 | .literal, 479 | .new_line, 480 | .literal, 481 | .map_value_ind, 482 | .space, 483 | .literal, 484 | .new_line, 485 | .literal, 486 | .map_value_ind, 487 | .space, 488 | .flow_seq_start, 489 | .space, 490 | .literal, 491 | .space, 492 | .flow_seq_end, 493 | .new_line, 494 | .new_line, 495 | .literal, 496 | .map_value_ind, 497 | .new_line, 498 | .space, 499 | .seq_item_ind, 500 | .literal, 501 | .map_value_ind, 502 | .space, 503 | .literal, 504 | .new_line, 505 | .space, 506 | .literal, 507 | .map_value_ind, 508 | .space, 509 | .literal, 510 | .new_line, 511 | .new_line, 512 | .literal, 513 | .map_value_ind, 514 | .space, 515 | .single_quoted, 516 | .new_line, 517 | .doc_end, 518 | .eof, 519 | }); 520 | } 521 | 522 | test "Unindented list" { 523 | try testExpected( 524 | \\b: 525 | \\- foo: 1 526 | \\c: 1 527 | , &[_]Token.Id{ 528 | .literal, 529 | .map_value_ind, 530 | .new_line, 531 | .seq_item_ind, 532 | .literal, 533 | .map_value_ind, 534 | .space, 535 | .literal, 536 | .new_line, 537 | .literal, 538 | .map_value_ind, 539 | .space, 540 | .literal, 541 | .eof, 542 | }); 543 | } 544 | 545 | test "escape sequences" { 546 | try testExpected( 547 | \\a: 'here''s an apostrophe' 548 | \\b: "a newline\nand a\ttab" 549 | \\c: "\"here\" and there" 550 | , &[_]Token.Id{ 551 | .literal, 552 | .map_value_ind, 553 | .space, 554 | .single_quoted, 555 | .new_line, 556 | .literal, 557 | .map_value_ind, 558 | .space, 559 | .double_quoted, 560 | .new_line, 561 | .literal, 562 | .map_value_ind, 563 | .space, 564 | .double_quoted, 565 | .eof, 566 | }); 567 | } 568 | 569 | test "comments" { 570 | try testExpected( 571 | \\key: # some comment about the key 572 | \\# first value 573 | \\- val1 574 | \\# second value 575 | \\- val2 576 | , &[_]Token.Id{ 577 | .literal, 578 | .map_value_ind, 579 | .space, 580 | .comment, 581 | .new_line, 582 | .comment, 583 | .new_line, 584 | .seq_item_ind, 585 | .literal, 586 | .new_line, 587 | .comment, 588 | .new_line, 589 | .seq_item_ind, 590 | .literal, 591 | .eof, 592 | }); 593 | } 594 | 595 | test "quoted literals" { 596 | try testExpected( 597 | \\'#000000' 598 | \\'[000000' 599 | \\"&someString" 600 | , &[_]Token.Id{ 601 | .single_quoted, 602 | .new_line, 603 | .single_quoted, 604 | .new_line, 605 | .double_quoted, 606 | .eof, 607 | }); 608 | } 609 | 610 | test "unquoted literals" { 611 | try testExpected( 612 | \\key1: helloWorld 613 | \\key2: hello,world 614 | \\key3: [hello,world] 615 | , &[_]Token.Id{ 616 | // key1 617 | .literal, 618 | .map_value_ind, 619 | .space, 620 | .literal, // helloWorld 621 | .new_line, 622 | // key2 623 | .literal, 624 | .map_value_ind, 625 | .space, 626 | .literal, // hello,world 627 | .new_line, 628 | // key3 629 | .literal, 630 | .map_value_ind, 631 | .space, 632 | .flow_seq_start, 633 | .literal, // hello 634 | .comma, 635 | .literal, // world 636 | .flow_seq_end, 637 | .eof, 638 | }); 639 | } 640 | 641 | test "unquoted literal containing colon" { 642 | try testExpected( 643 | \\key1: val:ue 644 | \\key2: val::ue 645 | , &[_]Token.Id{ 646 | // key1 647 | .literal, 648 | .map_value_ind, 649 | .space, 650 | .literal, // val:ue 651 | .new_line, 652 | // key2 653 | .literal, 654 | .map_value_ind, 655 | .space, 656 | .literal, // val::ue 657 | .eof, 658 | }); 659 | } 660 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "flake-compat": { 4 | "flake": false, 5 | "locked": { 6 | "lastModified": 1747046372, 7 | "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=", 8 | "owner": "edolstra", 9 | "repo": "flake-compat", 10 | "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885", 11 | "type": "github" 12 | }, 13 | "original": { 14 | "owner": "edolstra", 15 | "repo": "flake-compat", 16 | "type": "github" 17 | } 18 | }, 19 | "flake-compat_2": { 20 | "flake": false, 21 | "locked": { 22 | "lastModified": 1733328505, 23 | "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", 24 | "owner": "edolstra", 25 | "repo": "flake-compat", 26 | "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", 27 | "type": "github" 28 | }, 29 | "original": { 30 | "owner": "edolstra", 31 | "repo": "flake-compat", 32 | "type": "github" 33 | } 34 | }, 35 | "flake-compat_3": { 36 | "flake": false, 37 | "locked": { 38 | "lastModified": 1696426674, 39 | "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", 40 | "owner": "edolstra", 41 | "repo": "flake-compat", 42 | "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", 43 | "type": "github" 44 | }, 45 | "original": { 46 | "owner": "edolstra", 47 | "repo": "flake-compat", 48 | "type": "github" 49 | } 50 | }, 51 | "flake-compat_4": { 52 | "flake": false, 53 | "locked": { 54 | "lastModified": 1696426674, 55 | "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", 56 | "owner": "edolstra", 57 | "repo": "flake-compat", 58 | "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", 59 | "type": "github" 60 | }, 61 | "original": { 62 | "owner": "edolstra", 63 | "repo": "flake-compat", 64 | "type": "github" 65 | } 66 | }, 67 | "flake-compat_5": { 68 | "flake": false, 69 | "locked": { 70 | "lastModified": 1696426674, 71 | "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", 72 | "owner": "edolstra", 73 | "repo": "flake-compat", 74 | "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", 75 | "type": "github" 76 | }, 77 | "original": { 78 | "owner": "edolstra", 79 | "repo": "flake-compat", 80 | "type": "github" 81 | } 82 | }, 83 | "flake-compat_6": { 84 | "flake": false, 85 | "locked": { 86 | "lastModified": 1696426674, 87 | "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", 88 | "owner": "edolstra", 89 | "repo": "flake-compat", 90 | "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", 91 | "type": "github" 92 | }, 93 | "original": { 94 | "owner": "edolstra", 95 | "repo": "flake-compat", 96 | "type": "github" 97 | } 98 | }, 99 | "flake-utils": { 100 | "inputs": { 101 | "systems": "systems" 102 | }, 103 | "locked": { 104 | "lastModified": 1731533236, 105 | "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", 106 | "owner": "numtide", 107 | "repo": "flake-utils", 108 | "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", 109 | "type": "github" 110 | }, 111 | "original": { 112 | "owner": "numtide", 113 | "repo": "flake-utils", 114 | "type": "github" 115 | } 116 | }, 117 | "flake-utils_2": { 118 | "inputs": { 119 | "systems": "systems_2" 120 | }, 121 | "locked": { 122 | "lastModified": 1731533236, 123 | "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", 124 | "owner": "numtide", 125 | "repo": "flake-utils", 126 | "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", 127 | "type": "github" 128 | }, 129 | "original": { 130 | "owner": "numtide", 131 | "repo": "flake-utils", 132 | "type": "github" 133 | } 134 | }, 135 | "flake-utils_3": { 136 | "inputs": { 137 | "systems": "systems_3" 138 | }, 139 | "locked": { 140 | "lastModified": 1705309234, 141 | "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", 142 | "owner": "numtide", 143 | "repo": "flake-utils", 144 | "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", 145 | "type": "github" 146 | }, 147 | "original": { 148 | "owner": "numtide", 149 | "repo": "flake-utils", 150 | "type": "github" 151 | } 152 | }, 153 | "flake-utils_4": { 154 | "inputs": { 155 | "systems": "systems_4" 156 | }, 157 | "locked": { 158 | "lastModified": 1705309234, 159 | "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", 160 | "owner": "numtide", 161 | "repo": "flake-utils", 162 | "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", 163 | "type": "github" 164 | }, 165 | "original": { 166 | "owner": "numtide", 167 | "repo": "flake-utils", 168 | "type": "github" 169 | } 170 | }, 171 | "flake-utils_5": { 172 | "inputs": { 173 | "systems": "systems_5" 174 | }, 175 | "locked": { 176 | "lastModified": 1705309234, 177 | "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", 178 | "owner": "numtide", 179 | "repo": "flake-utils", 180 | "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", 181 | "type": "github" 182 | }, 183 | "original": { 184 | "owner": "numtide", 185 | "repo": "flake-utils", 186 | "type": "github" 187 | } 188 | }, 189 | "flake-utils_6": { 190 | "inputs": { 191 | "systems": "systems_6" 192 | }, 193 | "locked": { 194 | "lastModified": 1705309234, 195 | "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", 196 | "owner": "numtide", 197 | "repo": "flake-utils", 198 | "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", 199 | "type": "github" 200 | }, 201 | "original": { 202 | "owner": "numtide", 203 | "repo": "flake-utils", 204 | "type": "github" 205 | } 206 | }, 207 | "gitignore": { 208 | "inputs": { 209 | "nixpkgs": [ 210 | "poop", 211 | "zls", 212 | "nixpkgs" 213 | ] 214 | }, 215 | "locked": { 216 | "lastModified": 1709087332, 217 | "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", 218 | "owner": "hercules-ci", 219 | "repo": "gitignore.nix", 220 | "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", 221 | "type": "github" 222 | }, 223 | "original": { 224 | "owner": "hercules-ci", 225 | "repo": "gitignore.nix", 226 | "type": "github" 227 | } 228 | }, 229 | "gitignore_2": { 230 | "inputs": { 231 | "nixpkgs": [ 232 | "zls", 233 | "nixpkgs" 234 | ] 235 | }, 236 | "locked": { 237 | "lastModified": 1709087332, 238 | "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", 239 | "owner": "hercules-ci", 240 | "repo": "gitignore.nix", 241 | "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", 242 | "type": "github" 243 | }, 244 | "original": { 245 | "owner": "hercules-ci", 246 | "repo": "gitignore.nix", 247 | "type": "github" 248 | } 249 | }, 250 | "nixpkgs": { 251 | "locked": { 252 | "lastModified": 1751274312, 253 | "narHash": "sha256-/bVBlRpECLVzjV19t5KMdMFWSwKLtb5RyXdjz3LJT+g=", 254 | "owner": "nixos", 255 | "repo": "nixpkgs", 256 | "rev": "50ab793786d9de88ee30ec4e4c24fb4236fc2674", 257 | "type": "github" 258 | }, 259 | "original": { 260 | "owner": "nixos", 261 | "ref": "nixos-24.11", 262 | "repo": "nixpkgs", 263 | "type": "github" 264 | } 265 | }, 266 | "nixpkgs_2": { 267 | "locked": { 268 | "lastModified": 1739758141, 269 | "narHash": "sha256-uq6A2L7o1/tR6VfmYhZWoVAwb3gTy7j4Jx30MIrH0rE=", 270 | "owner": "nixos", 271 | "repo": "nixpkgs", 272 | "rev": "c618e28f70257593de75a7044438efc1c1fc0791", 273 | "type": "github" 274 | }, 275 | "original": { 276 | "owner": "nixos", 277 | "ref": "nixos-24.11", 278 | "repo": "nixpkgs", 279 | "type": "github" 280 | } 281 | }, 282 | "nixpkgs_3": { 283 | "locked": { 284 | "lastModified": 1708161998, 285 | "narHash": "sha256-6KnemmUorCvlcAvGziFosAVkrlWZGIc6UNT9GUYr0jQ=", 286 | "owner": "NixOS", 287 | "repo": "nixpkgs", 288 | "rev": "84d981bae8b5e783b3b548de505b22880559515f", 289 | "type": "github" 290 | }, 291 | "original": { 292 | "owner": "NixOS", 293 | "ref": "nixos-23.11", 294 | "repo": "nixpkgs", 295 | "type": "github" 296 | } 297 | }, 298 | "nixpkgs_4": { 299 | "locked": { 300 | "lastModified": 1739206421, 301 | "narHash": "sha256-PwQASeL2cGVmrtQYlrBur0U20Xy07uSWVnFup2PHnDs=", 302 | "owner": "NixOS", 303 | "repo": "nixpkgs", 304 | "rev": "44534bc021b85c8d78e465021e21f33b856e2540", 305 | "type": "github" 306 | }, 307 | "original": { 308 | "owner": "NixOS", 309 | "ref": "nixos-24.11", 310 | "repo": "nixpkgs", 311 | "type": "github" 312 | } 313 | }, 314 | "nixpkgs_5": { 315 | "locked": { 316 | "lastModified": 1708161998, 317 | "narHash": "sha256-6KnemmUorCvlcAvGziFosAVkrlWZGIc6UNT9GUYr0jQ=", 318 | "owner": "NixOS", 319 | "repo": "nixpkgs", 320 | "rev": "84d981bae8b5e783b3b548de505b22880559515f", 321 | "type": "github" 322 | }, 323 | "original": { 324 | "owner": "NixOS", 325 | "ref": "nixos-23.11", 326 | "repo": "nixpkgs", 327 | "type": "github" 328 | } 329 | }, 330 | "nixpkgs_6": { 331 | "locked": { 332 | "lastModified": 1752162966, 333 | "narHash": "sha256-3MxxkU8ZXMHXcbFz7UE4M6qnIPTYGcE/7EMqlZNnVDE=", 334 | "owner": "NixOS", 335 | "repo": "nixpkgs", 336 | "rev": "10e687235226880ed5e9f33f1ffa71fe60f2638a", 337 | "type": "github" 338 | }, 339 | "original": { 340 | "owner": "NixOS", 341 | "ref": "nixos-25.05", 342 | "repo": "nixpkgs", 343 | "type": "github" 344 | } 345 | }, 346 | "poop": { 347 | "inputs": { 348 | "flake-compat": "flake-compat_2", 349 | "flake-utils": "flake-utils_2", 350 | "nixpkgs": "nixpkgs_2", 351 | "zig": "zig", 352 | "zls": "zls" 353 | }, 354 | "locked": { 355 | "lastModified": 1739946927, 356 | "narHash": "sha256-07xFvrm46RsQL81Mn2VMRRuU2//4VKs/V4G6n203OUQ=", 357 | "owner": "kubkon", 358 | "repo": "poop", 359 | "rev": "2007a7e1dcc9c08b878d513bba27cdfb91991622", 360 | "type": "github" 361 | }, 362 | "original": { 363 | "owner": "kubkon", 364 | "ref": "nix", 365 | "repo": "poop", 366 | "type": "github" 367 | } 368 | }, 369 | "root": { 370 | "inputs": { 371 | "flake-compat": "flake-compat", 372 | "flake-utils": "flake-utils", 373 | "nixpkgs": "nixpkgs", 374 | "poop": "poop", 375 | "zig": "zig_2", 376 | "zls": "zls_2" 377 | } 378 | }, 379 | "systems": { 380 | "locked": { 381 | "lastModified": 1681028828, 382 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 383 | "owner": "nix-systems", 384 | "repo": "default", 385 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 386 | "type": "github" 387 | }, 388 | "original": { 389 | "owner": "nix-systems", 390 | "repo": "default", 391 | "type": "github" 392 | } 393 | }, 394 | "systems_2": { 395 | "locked": { 396 | "lastModified": 1681028828, 397 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 398 | "owner": "nix-systems", 399 | "repo": "default", 400 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 401 | "type": "github" 402 | }, 403 | "original": { 404 | "owner": "nix-systems", 405 | "repo": "default", 406 | "type": "github" 407 | } 408 | }, 409 | "systems_3": { 410 | "locked": { 411 | "lastModified": 1681028828, 412 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 413 | "owner": "nix-systems", 414 | "repo": "default", 415 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 416 | "type": "github" 417 | }, 418 | "original": { 419 | "owner": "nix-systems", 420 | "repo": "default", 421 | "type": "github" 422 | } 423 | }, 424 | "systems_4": { 425 | "locked": { 426 | "lastModified": 1681028828, 427 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 428 | "owner": "nix-systems", 429 | "repo": "default", 430 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 431 | "type": "github" 432 | }, 433 | "original": { 434 | "owner": "nix-systems", 435 | "repo": "default", 436 | "type": "github" 437 | } 438 | }, 439 | "systems_5": { 440 | "locked": { 441 | "lastModified": 1681028828, 442 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 443 | "owner": "nix-systems", 444 | "repo": "default", 445 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 446 | "type": "github" 447 | }, 448 | "original": { 449 | "owner": "nix-systems", 450 | "repo": "default", 451 | "type": "github" 452 | } 453 | }, 454 | "systems_6": { 455 | "locked": { 456 | "lastModified": 1681028828, 457 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 458 | "owner": "nix-systems", 459 | "repo": "default", 460 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 461 | "type": "github" 462 | }, 463 | "original": { 464 | "owner": "nix-systems", 465 | "repo": "default", 466 | "type": "github" 467 | } 468 | }, 469 | "zig": { 470 | "inputs": { 471 | "flake-compat": "flake-compat_3", 472 | "flake-utils": "flake-utils_3", 473 | "nixpkgs": "nixpkgs_3" 474 | }, 475 | "locked": { 476 | "lastModified": 1739924999, 477 | "narHash": "sha256-EHOOMs8bnmRO+tCmZ+ZeDi6VbQykISfpkknxkGPpOjA=", 478 | "owner": "mitchellh", 479 | "repo": "zig-overlay", 480 | "rev": "6a43a4acb17ddcb72d222e1dbf8732a8e46e9b41", 481 | "type": "github" 482 | }, 483 | "original": { 484 | "owner": "mitchellh", 485 | "repo": "zig-overlay", 486 | "type": "github" 487 | } 488 | }, 489 | "zig-overlay": { 490 | "inputs": { 491 | "flake-compat": "flake-compat_4", 492 | "flake-utils": "flake-utils_4", 493 | "nixpkgs": [ 494 | "poop", 495 | "zls", 496 | "nixpkgs" 497 | ] 498 | }, 499 | "locked": { 500 | "lastModified": 1739275930, 501 | "narHash": "sha256-Tc8LiHKWpO0VHwoUb3aLf6Fp1exjGbqK0RdbUmCYw58=", 502 | "owner": "mitchellh", 503 | "repo": "zig-overlay", 504 | "rev": "163ae88f737f998b272e19c98ca6ce9a2aa02441", 505 | "type": "github" 506 | }, 507 | "original": { 508 | "owner": "mitchellh", 509 | "repo": "zig-overlay", 510 | "type": "github" 511 | } 512 | }, 513 | "zig-overlay_2": { 514 | "inputs": { 515 | "flake-compat": "flake-compat_6", 516 | "flake-utils": "flake-utils_6", 517 | "nixpkgs": [ 518 | "zls", 519 | "nixpkgs" 520 | ] 521 | }, 522 | "locked": { 523 | "lastModified": 1752280594, 524 | "narHash": "sha256-fdhSY2g5O05QAjlxgh6aZYGyghQlGslPA/f6aPed1c0=", 525 | "owner": "mitchellh", 526 | "repo": "zig-overlay", 527 | "rev": "ef606ab86e0ac5aa4c6105ed4c446628989028a6", 528 | "type": "github" 529 | }, 530 | "original": { 531 | "owner": "mitchellh", 532 | "repo": "zig-overlay", 533 | "type": "github" 534 | } 535 | }, 536 | "zig_2": { 537 | "inputs": { 538 | "flake-compat": "flake-compat_5", 539 | "flake-utils": "flake-utils_5", 540 | "nixpkgs": "nixpkgs_5" 541 | }, 542 | "locked": { 543 | "lastModified": 1752453549, 544 | "narHash": "sha256-3c6/4K+0I/zGC86mC+4FrWNl61/9D/HUYxrFb6nd7u4=", 545 | "owner": "mitchellh", 546 | "repo": "zig-overlay", 547 | "rev": "fff2cca28feb0dc52cdcf0ba7d142f8dad435324", 548 | "type": "github" 549 | }, 550 | "original": { 551 | "owner": "mitchellh", 552 | "repo": "zig-overlay", 553 | "type": "github" 554 | } 555 | }, 556 | "zls": { 557 | "inputs": { 558 | "gitignore": "gitignore", 559 | "nixpkgs": "nixpkgs_4", 560 | "zig-overlay": "zig-overlay" 561 | }, 562 | "locked": { 563 | "lastModified": 1739844671, 564 | "narHash": "sha256-pD/4a2xN4sO3ypVc4A2dL9Su0yIBuu9FCO2nt2xVGOk=", 565 | "owner": "zigtools", 566 | "repo": "zls", 567 | "rev": "7f367a64106d4eb2dc3656e24a1a4370358080ec", 568 | "type": "github" 569 | }, 570 | "original": { 571 | "owner": "zigtools", 572 | "repo": "zls", 573 | "type": "github" 574 | } 575 | }, 576 | "zls_2": { 577 | "inputs": { 578 | "gitignore": "gitignore_2", 579 | "nixpkgs": "nixpkgs_6", 580 | "zig-overlay": "zig-overlay_2" 581 | }, 582 | "locked": { 583 | "lastModified": 1752512596, 584 | "narHash": "sha256-SqOxj5kDoIol1l5SvoiTzccCaEHz37i5Lo4vTjqFBCQ=", 585 | "owner": "zigtools", 586 | "repo": "zls", 587 | "rev": "2057e00911908b80995950908baf2f53f9cc33ef", 588 | "type": "github" 589 | }, 590 | "original": { 591 | "owner": "zigtools", 592 | "repo": "zls", 593 | "type": "github" 594 | } 595 | } 596 | }, 597 | "root": "root", 598 | "version": 7 599 | } 600 | -------------------------------------------------------------------------------- /src/Yaml.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const assert = std.debug.assert; 3 | const math = std.math; 4 | const mem = std.mem; 5 | const log = std.log.scoped(.yaml); 6 | 7 | const Allocator = mem.Allocator; 8 | const ArenaAllocator = std.heap.ArenaAllocator; 9 | const ErrorBundle = std.zig.ErrorBundle; 10 | const Node = Tree.Node; 11 | const Parser = @import("Parser.zig"); 12 | const ParseError = Parser.ParseError; 13 | const Tokenizer = @import("Tokenizer.zig"); 14 | const Token = Tokenizer.Token; 15 | const Tree = @import("Tree.zig"); 16 | const Yaml = @This(); 17 | 18 | source: []const u8, 19 | docs: std.ArrayListUnmanaged(Value) = .empty, 20 | tree: ?Tree = null, 21 | parse_errors: ErrorBundle = .empty, 22 | 23 | pub fn deinit(self: *Yaml, gpa: Allocator) void { 24 | for (self.docs.items) |*value| { 25 | value.deinit(gpa); 26 | } 27 | self.docs.deinit(gpa); 28 | if (self.tree) |*tree| { 29 | tree.deinit(gpa); 30 | } 31 | self.parse_errors.deinit(gpa); 32 | self.* = undefined; 33 | } 34 | 35 | pub fn load(self: *Yaml, gpa: Allocator) !void { 36 | var parser = try Parser.init(gpa, self.source); 37 | defer parser.deinit(gpa); 38 | 39 | parser.parse(gpa) catch |err| switch (err) { 40 | error.ParseFailure => { 41 | self.parse_errors = try parser.errors.toOwnedBundle(""); 42 | return error.ParseFailure; 43 | }, 44 | else => return err, 45 | }; 46 | 47 | self.tree = try parser.toOwnedTree(gpa); 48 | 49 | try self.docs.ensureTotalCapacityPrecise(gpa, self.tree.?.docs.len); 50 | 51 | for (self.tree.?.docs) |node| { 52 | const value = try Value.fromNode(gpa, self.tree.?, node); 53 | self.docs.appendAssumeCapacity(value); 54 | } 55 | } 56 | 57 | pub fn parse(self: Yaml, arena: Allocator, comptime T: type) Error!T { 58 | if (self.docs.items.len == 0) { 59 | if (@typeInfo(T) == .void) return {}; 60 | return error.TypeMismatch; 61 | } 62 | 63 | if (self.docs.items.len == 1) { 64 | return self.parseValue(arena, T, self.docs.items[0]); 65 | } 66 | 67 | switch (@typeInfo(T)) { 68 | .array => |info| { 69 | var parsed: T = undefined; 70 | for (self.docs.items, 0..) |doc, i| { 71 | parsed[i] = try self.parseValue(arena, info.child, doc); 72 | } 73 | return parsed; 74 | }, 75 | .pointer => |info| { 76 | switch (info.size) { 77 | .slice => { 78 | var parsed = try arena.alloc(info.child, self.docs.items.len); 79 | for (self.docs.items, 0..) |doc, i| { 80 | parsed[i] = try self.parseValue(arena, info.child, doc); 81 | } 82 | return parsed; 83 | }, 84 | else => return error.TypeMismatch, 85 | } 86 | }, 87 | .@"union" => return error.Unimplemented, 88 | else => return error.TypeMismatch, 89 | } 90 | } 91 | 92 | fn parseValue(self: Yaml, arena: Allocator, comptime T: type, value: Value) Error!T { 93 | return switch (@typeInfo(T)) { 94 | .int => self.parseInt(T, value), 95 | .bool => self.parseBoolean(bool, value), 96 | .float => self.parseFloat(T, value), 97 | .@"struct" => if (value.asMap()) |map| { 98 | return self.parseStruct(arena, T, map); 99 | } else return error.TypeMismatch, 100 | .@"union" => self.parseUnion(arena, T, value), 101 | .array => if (value.asList()) |list| { 102 | return self.parseArray(arena, T, list); 103 | } else return error.TypeMismatch, 104 | .pointer => if (value.asList()) |list| { 105 | return self.parsePointer(arena, T, .{ .list = list }); 106 | } else if (value.asScalar()) |scalar| { 107 | return self.parsePointer(arena, T, .{ .scalar = try arena.dupe(u8, scalar) }); 108 | } else if (value.asMap()) |map| { 109 | return self.parsePointer(arena, T, .{ .map = map }); 110 | } else return error.TypeMismatch, 111 | .@"enum" => self.parseEnum(T, value), 112 | .void => error.TypeMismatch, 113 | .optional => unreachable, 114 | else => error.Unimplemented, 115 | }; 116 | } 117 | 118 | fn parseInt(self: Yaml, comptime T: type, value: Value) Error!T { 119 | _ = self; 120 | const scalar = value.asScalar() orelse return error.TypeMismatch; 121 | return try std.fmt.parseInt(T, scalar, 0); 122 | } 123 | 124 | fn parseFloat(self: Yaml, comptime T: type, value: Value) Error!T { 125 | _ = self; 126 | const scalar = value.asScalar() orelse return error.TypeMismatch; 127 | return try std.fmt.parseFloat(T, scalar); 128 | } 129 | 130 | fn parseBoolean(self: Yaml, comptime T: type, value: Value) Error!T { 131 | _ = self; 132 | const raw = value.asScalar() orelse return error.TypeMismatch; 133 | 134 | if (raw.len > 0 and raw.len <= longestBooleanValueString) { 135 | var buffer: [longestBooleanValueString]u8 = undefined; 136 | const lower_raw = std.ascii.lowerString(&buffer, raw); 137 | 138 | for (supportedTruthyBooleanValue) |v| { 139 | if (std.mem.eql(u8, v, lower_raw)) { 140 | return true; 141 | } 142 | } 143 | 144 | for (supportedFalsyBooleanValue) |v| { 145 | if (std.mem.eql(u8, v, lower_raw)) { 146 | return false; 147 | } 148 | } 149 | } 150 | 151 | return error.TypeMismatch; 152 | } 153 | 154 | fn parseUnion(self: Yaml, arena: Allocator, comptime T: type, value: Value) Error!T { 155 | const union_info = @typeInfo(T).@"union"; 156 | 157 | if (union_info.tag_type) |_| { 158 | inline for (union_info.fields) |field| { 159 | if (self.parseValue(arena, field.type, value)) |u_value| { 160 | return @unionInit(T, field.name, u_value); 161 | } else |err| switch (err) { 162 | error.InvalidCharacter => {}, 163 | error.TypeMismatch => {}, 164 | error.StructFieldMissing => {}, 165 | else => return err, 166 | } 167 | } 168 | } else return error.UntaggedUnion; 169 | 170 | return error.UnionTagMissing; 171 | } 172 | 173 | fn parseOptional(self: Yaml, arena: Allocator, comptime T: type, value: ?Value) Error!T { 174 | const unwrapped = value orelse return null; 175 | const opt_info = @typeInfo(T).optional; 176 | return @as(T, try self.parseValue(arena, opt_info.child, unwrapped)); 177 | } 178 | 179 | fn parseStruct(self: Yaml, arena: Allocator, comptime T: type, map: Map) Error!T { 180 | const struct_info = @typeInfo(T).@"struct"; 181 | var parsed: T = undefined; 182 | 183 | inline for (struct_info.fields) |field| { 184 | var value: ?Value = map.get(field.name) orelse blk: { 185 | const field_name = try mem.replaceOwned(u8, arena, field.name, "_", "-"); 186 | break :blk map.get(field_name); 187 | }; 188 | 189 | if (@typeInfo(field.type) == .optional) { 190 | if (value == null) blk: { 191 | const maybe_default_value = field.defaultValue() orelse break :blk; 192 | value = Value.encode(arena, maybe_default_value) catch break :blk; 193 | } 194 | @field(parsed, field.name) = try self.parseOptional(arena, field.type, value); 195 | continue; 196 | } 197 | 198 | if (field.defaultValue()) |default_value| { 199 | if (value == null) blk: { 200 | value = Value.encode(arena, default_value) catch break :blk; 201 | } 202 | } 203 | 204 | const unwrapped = value orelse { 205 | log.debug("missing struct field: {s}: {s}", .{ field.name, @typeName(field.type) }); 206 | return error.StructFieldMissing; 207 | }; 208 | @field(parsed, field.name) = try self.parseValue(arena, field.type, unwrapped); 209 | } 210 | 211 | return parsed; 212 | } 213 | 214 | fn parsePointer(self: Yaml, arena: Allocator, comptime T: type, value: Value) Error!T { 215 | const ptr_info = @typeInfo(T).pointer; 216 | 217 | switch (ptr_info.size) { 218 | .slice => { 219 | if (ptr_info.child == u8) { 220 | const scalar = value.asScalar() orelse return error.TypeMismatch; 221 | return try arena.dupe(u8, scalar); 222 | } 223 | 224 | if (value.asList()) |list| { 225 | var parsed = try arena.alloc(ptr_info.child, list.len); 226 | for (list, 0..) |elem, i| { 227 | parsed[i] = try self.parseValue(arena, ptr_info.child, elem); 228 | } 229 | return parsed; 230 | } 231 | 232 | return error.TypeMismatch; 233 | }, 234 | .one => { 235 | const parsed = try arena.create(ptr_info.child); 236 | parsed.* = try self.parseValue(arena, ptr_info.child, value); 237 | return parsed; 238 | }, 239 | else => return error.Unimplemented, 240 | } 241 | } 242 | 243 | fn parseArray(self: Yaml, arena: Allocator, comptime T: type, list: List) Error!T { 244 | const array_info = @typeInfo(T).array; 245 | if (array_info.len != list.len) return error.ArraySizeMismatch; 246 | 247 | var parsed: T = undefined; 248 | for (list, 0..) |elem, i| { 249 | parsed[i] = try self.parseValue(arena, array_info.child, elem); 250 | } 251 | 252 | return parsed; 253 | } 254 | 255 | fn parseEnum(self: Yaml, comptime T: type, value: Value) Error!T { 256 | _ = self; 257 | 258 | const scalar = value.asScalar() orelse return error.TypeMismatch; 259 | return std.meta.stringToEnum(T, scalar) orelse error.InvalidEnum; 260 | } 261 | 262 | pub fn stringify(self: Yaml, writer: *std.Io.Writer) !void { 263 | for (self.docs.items, self.tree.?.docs) |doc, node| { 264 | try writer.writeAll("---"); 265 | if (self.tree.?.directive(node)) |directive| { 266 | try writer.print(" !{s}", .{directive}); 267 | } 268 | try writer.writeByte('\n'); 269 | try doc.stringify(writer, .{}); 270 | try writer.writeByte('\n'); 271 | } 272 | try writer.writeAll("...\n"); 273 | } 274 | 275 | const supportedTruthyBooleanValue: [4][]const u8 = .{ "y", "yes", "on", "true" }; 276 | const supportedFalsyBooleanValue: [4][]const u8 = .{ "n", "no", "off", "false" }; 277 | 278 | const longestBooleanValueString = blk: { 279 | var lengths: [supportedTruthyBooleanValue.len + supportedFalsyBooleanValue.len]usize = undefined; 280 | for (supportedTruthyBooleanValue, 0..) |v, i| { 281 | lengths[i] = v.len; 282 | } 283 | for (supportedFalsyBooleanValue, supportedTruthyBooleanValue.len..) |v, i| { 284 | lengths[i] = v.len; 285 | } 286 | break :blk mem.max(usize, &lengths); 287 | }; 288 | 289 | pub const Error = error{ 290 | InvalidCharacter, 291 | Unimplemented, 292 | TypeMismatch, 293 | StructFieldMissing, 294 | ArraySizeMismatch, 295 | UntaggedUnion, 296 | UnionTagMissing, 297 | Overflow, 298 | OutOfMemory, 299 | InvalidEnum, 300 | }; 301 | 302 | pub const YamlError = error{ 303 | UnexpectedNodeType, 304 | DuplicateMapKey, 305 | OutOfMemory, 306 | CannotEncodeValue, 307 | } || ParseError || std.fmt.ParseIntError; 308 | 309 | pub const StringifyError = error{ 310 | OutOfMemory, 311 | } || YamlError || std.fs.File.WriteError || std.Io.Writer.Error; 312 | 313 | pub const List = []Value; 314 | pub const Map = std.StringArrayHashMapUnmanaged(Value); 315 | 316 | pub const Value = union(enum) { 317 | empty, 318 | scalar: []const u8, 319 | list: List, 320 | map: Map, 321 | boolean: bool, 322 | 323 | pub fn deinit(self: *Value, gpa: Allocator) void { 324 | switch (self.*) { 325 | .scalar => |scalar| gpa.free(scalar), 326 | .list => |list| { 327 | for (list) |*value| { 328 | value.deinit(gpa); 329 | } 330 | gpa.free(list); 331 | }, 332 | .map => |*map| { 333 | for (map.keys(), map.values()) |key, *value| { 334 | gpa.free(key); 335 | value.deinit(gpa); 336 | } 337 | map.deinit(gpa); 338 | }, 339 | .empty, .boolean => {}, 340 | } 341 | } 342 | 343 | pub fn asScalar(self: Value) ?[]const u8 { 344 | if (self != .scalar) return null; 345 | return self.scalar; 346 | } 347 | 348 | pub fn asList(self: Value) ?List { 349 | if (self != .list) return null; 350 | return self.list; 351 | } 352 | 353 | pub fn asMap(self: Value) ?Map { 354 | if (self != .map) return null; 355 | return self.map; 356 | } 357 | 358 | const StringifyArgs = struct { 359 | indentation: usize = 0, 360 | should_inline_first_key: bool = false, 361 | }; 362 | 363 | pub fn stringify(self: Value, writer: *std.Io.Writer, args: StringifyArgs) StringifyError!void { 364 | switch (self) { 365 | .empty => return, 366 | .scalar => |scalar| return writer.print("{s}", .{scalar}), 367 | .list => |list| { 368 | const len = list.len; 369 | if (len == 0) return; 370 | 371 | const first = list[0]; 372 | if (first.isCompound()) { 373 | for (list, 0..) |elem, i| { 374 | const indentation = try writer.writableSlice(args.indentation); 375 | @memset(indentation, ' '); 376 | try writer.writeAll("- "); 377 | try elem.stringify(writer, .{ 378 | .indentation = args.indentation + 2, 379 | .should_inline_first_key = true, 380 | }); 381 | if (i < len - 1) { 382 | try writer.writeByte('\n'); 383 | } 384 | } 385 | return; 386 | } 387 | 388 | try writer.writeAll("[ "); 389 | for (list, 0..) |elem, i| { 390 | try elem.stringify(writer, args); 391 | if (i < len - 1) { 392 | try writer.writeAll(", "); 393 | } 394 | } 395 | try writer.writeAll(" ]"); 396 | }, 397 | .map => |map| { 398 | const len = map.count(); 399 | if (len == 0) return; 400 | 401 | var i: usize = 0; 402 | for (map.keys(), map.values()) |key, value| { 403 | if (!args.should_inline_first_key or i != 0) { 404 | const indentation = try writer.writableSlice(args.indentation); 405 | @memset(indentation, ' '); 406 | } 407 | try writer.print("{s}: ", .{key}); 408 | 409 | const should_inline = blk: { 410 | if (!value.isCompound()) break :blk true; 411 | if (value == .list and value.list.len > 0 and !value.list[0].isCompound()) break :blk true; 412 | break :blk false; 413 | }; 414 | 415 | if (should_inline) { 416 | try value.stringify(writer, args); 417 | } else { 418 | try writer.writeByte('\n'); 419 | try value.stringify(writer, .{ 420 | .indentation = args.indentation + 4, 421 | }); 422 | } 423 | 424 | if (i < len - 1) { 425 | try writer.writeByte('\n'); 426 | } 427 | 428 | i += 1; 429 | } 430 | }, 431 | .boolean => |value| return writer.writeAll(if (value) "true" else "false"), 432 | } 433 | } 434 | 435 | fn isCompound(self: Value) bool { 436 | return switch (self) { 437 | .list, .map => true, 438 | else => false, 439 | }; 440 | } 441 | 442 | fn fromNode(gpa: Allocator, tree: Tree, node_index: Node.Index) YamlError!Value { 443 | const tag = tree.nodeTag(node_index); 444 | switch (tag) { 445 | .doc => { 446 | const inner = tree.nodeData(node_index).maybe_node.unwrap() orelse return .empty; 447 | return Value.fromNode(gpa, tree, inner); 448 | }, 449 | .doc_with_directive => { 450 | const inner = tree.nodeData(node_index).doc_with_directive.maybe_node.unwrap() orelse return .empty; 451 | return Value.fromNode(gpa, tree, inner); 452 | }, 453 | .map_single => { 454 | const entry = tree.nodeData(node_index).map; 455 | 456 | // TODO use ContextAdapted HashMap and do not duplicate keys, intern 457 | // in a contiguous string buffer. 458 | var out_map: Map = .empty; 459 | errdefer out_map.deinit(gpa); 460 | try out_map.ensureTotalCapacity(gpa, 1); 461 | 462 | const key = try gpa.dupe(u8, tree.rawString(entry.key, entry.key)); 463 | errdefer gpa.free(key); 464 | 465 | const gop = out_map.getOrPutAssumeCapacity(key); 466 | if (gop.found_existing) return error.DuplicateMapKey; 467 | 468 | gop.value_ptr.* = if (entry.maybe_node.unwrap()) |value| 469 | try Value.fromNode(gpa, tree, value) 470 | else 471 | .empty; 472 | 473 | return Value{ .map = out_map }; 474 | }, 475 | .map_many => { 476 | const extra_index = tree.nodeData(node_index).extra; 477 | const map = tree.extraData(Tree.Map, extra_index); 478 | 479 | // TODO use ContextAdapted HashMap and do not duplicate keys, intern 480 | // in a contiguous string buffer. 481 | var out_map: Map = .empty; 482 | errdefer { 483 | for (out_map.keys(), out_map.values()) |key, *value| { 484 | gpa.free(key); 485 | value.deinit(gpa); 486 | } 487 | out_map.deinit(gpa); 488 | } 489 | try out_map.ensureTotalCapacity(gpa, map.data.map_len); 490 | 491 | var extra_end = map.end; 492 | for (0..map.data.map_len) |_| { 493 | const entry = tree.extraData(Tree.Map.Entry, extra_end); 494 | extra_end = entry.end; 495 | 496 | const key = try gpa.dupe(u8, tree.rawString(entry.data.key, entry.data.key)); 497 | errdefer gpa.free(key); 498 | 499 | const gop = out_map.getOrPutAssumeCapacity(key); 500 | if (gop.found_existing) return error.DuplicateMapKey; 501 | 502 | gop.value_ptr.* = if (entry.data.maybe_node.unwrap()) |value| 503 | try Value.fromNode(gpa, tree, value) 504 | else 505 | .empty; 506 | } 507 | 508 | return Value{ .map = out_map }; 509 | }, 510 | .list_empty => { 511 | return Value{ .list = &.{} }; 512 | }, 513 | .list_one => { 514 | const value_index = tree.nodeData(node_index).node; 515 | const out_list = try gpa.alloc(Value, 1); 516 | errdefer gpa.free(out_list); 517 | const value = try Value.fromNode(gpa, tree, value_index); 518 | out_list[0] = value; 519 | return Value{ .list = out_list }; 520 | }, 521 | .list_two => { 522 | const list = tree.nodeData(node_index).list; 523 | const out_list = try gpa.alloc(Value, 2); 524 | errdefer { 525 | for (out_list) |*value| { 526 | value.deinit(gpa); 527 | } 528 | gpa.free(out_list); 529 | } 530 | for (out_list, &[2]Node.Index{ list.el1, list.el2 }) |*out, value_index| { 531 | out.* = try Value.fromNode(gpa, tree, value_index); 532 | } 533 | return Value{ .list = out_list }; 534 | }, 535 | .list_many => { 536 | const extra_index = tree.nodeData(node_index).extra; 537 | const list = tree.extraData(Tree.List, extra_index); 538 | 539 | var out_list: std.ArrayListUnmanaged(Value) = .empty; 540 | errdefer for (out_list.items) |*value| { 541 | value.deinit(gpa); 542 | }; 543 | defer out_list.deinit(gpa); 544 | try out_list.ensureTotalCapacityPrecise(gpa, list.data.list_len); 545 | 546 | var extra_end = list.end; 547 | for (0..list.data.list_len) |_| { 548 | const elem = tree.extraData(Tree.List.Entry, extra_end); 549 | extra_end = elem.end; 550 | 551 | const value = try Value.fromNode(gpa, tree, elem.data.node); 552 | out_list.appendAssumeCapacity(value); 553 | } 554 | 555 | return Value{ .list = try out_list.toOwnedSlice(gpa) }; 556 | }, 557 | .string_value => { 558 | const raw = tree.nodeData(node_index).string.slice(tree); 559 | return Value{ .scalar = try gpa.dupe(u8, raw) }; 560 | }, 561 | .value => { 562 | const raw = tree.nodeScope(node_index).rawString(tree); 563 | return Value{ .scalar = try gpa.dupe(u8, raw) }; 564 | }, 565 | } 566 | } 567 | 568 | pub fn encode(arena: Allocator, input: anytype) YamlError!?Value { 569 | switch (@typeInfo(@TypeOf(input))) { 570 | .comptime_int, 571 | .int, 572 | .comptime_float, 573 | .float, 574 | => return Value{ .scalar = try std.fmt.allocPrint(arena, "{d}", .{input}) }, 575 | 576 | .@"struct" => |info| if (info.is_tuple) { 577 | var list: std.ArrayListUnmanaged(Value) = .empty; 578 | try list.ensureTotalCapacityPrecise(arena, info.fields.len); 579 | 580 | inline for (info.fields) |field| { 581 | if (try encode(arena, @field(input, field.name))) |value| { 582 | list.appendAssumeCapacity(value); 583 | } 584 | } 585 | 586 | return Value{ .list = try list.toOwnedSlice(arena) }; 587 | } else { 588 | var map: Map = .empty; 589 | try map.ensureTotalCapacity(arena, info.fields.len); 590 | 591 | inline for (info.fields) |field| { 592 | if (try encode(arena, @field(input, field.name))) |value| { 593 | const key = try arena.dupe(u8, field.name); 594 | map.putAssumeCapacityNoClobber(key, value); 595 | } 596 | } 597 | 598 | return Value{ .map = map }; 599 | }, 600 | 601 | .@"union" => |info| if (info.tag_type) |tag_type| { 602 | inline for (info.fields) |field| { 603 | if (@field(tag_type, field.name) == input) { 604 | return try encode(arena, @field(input, field.name)); 605 | } 606 | } else unreachable; 607 | } else return error.UntaggedUnion, 608 | 609 | .array => return encode(arena, &input), 610 | 611 | .pointer => |info| switch (info.size) { 612 | .one => switch (@typeInfo(info.child)) { 613 | .array => |child_info| { 614 | const Slice = []const child_info.child; 615 | return encode(arena, @as(Slice, input)); 616 | }, 617 | else => { 618 | return encode(arena, input); 619 | }, 620 | }, 621 | .slice => { 622 | if (info.child == u8) { 623 | return Value{ .scalar = try arena.dupe(u8, input) }; 624 | } 625 | 626 | var list: std.ArrayListUnmanaged(Value) = .empty; 627 | try list.ensureTotalCapacityPrecise(arena, input.len); 628 | 629 | for (input) |elem| { 630 | if (try encode(arena, elem)) |value| { 631 | list.appendAssumeCapacity(value); 632 | } else { 633 | log.debug("Could not encode value in a list: {any}", .{elem}); 634 | return error.CannotEncodeValue; 635 | } 636 | } 637 | 638 | return Value{ .list = try list.toOwnedSlice(arena) }; 639 | }, 640 | else => { 641 | @compileError("Unhandled type: " ++ @typeName(@TypeOf(input))); 642 | }, 643 | }, 644 | 645 | // TODO we should probably have an option to encode `null` and also 646 | // allow for some default value too. 647 | .optional => return if (input) |val| encode(arena, val) else null, 648 | 649 | .null => return null, 650 | .bool => return Value{ .boolean = input }, 651 | .@"enum" => return Value{ .scalar = try arena.dupe(u8, @tagName(input)) }, 652 | 653 | else => { 654 | @compileError("Unhandled type: " ++ @typeName(@TypeOf(input))); 655 | }, 656 | } 657 | } 658 | }; 659 | 660 | pub const ErrorMsg = struct { 661 | msg: []const u8, 662 | line_col: Tree.LineCol, 663 | 664 | pub fn deinit(err: *ErrorMsg, gpa: Allocator) void { 665 | gpa.free(err.msg); 666 | } 667 | }; 668 | 669 | test { 670 | _ = @import("Yaml/test.zig"); 671 | } 672 | -------------------------------------------------------------------------------- /test/spec.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const fs = std.fs; 3 | const mem = std.mem; 4 | 5 | const Allocator = mem.Allocator; 6 | const Step = std.Build.Step; 7 | const SpecTest = @This(); 8 | 9 | pub const base_id: Step.Id = .custom; 10 | 11 | step: Step, 12 | output_file: std.Build.GeneratedFile, 13 | 14 | const test_filename = "yaml_test_suite.zig"; 15 | 16 | const preamble = 17 | \\// This file is generated from the YAML 1.2 test database. 18 | \\ 19 | \\const std = @import("std"); 20 | \\const testing = std.testing; 21 | \\ 22 | \\const Yaml = @import("yaml").Yaml; 23 | \\ 24 | \\const alloc = testing.allocator; 25 | \\ 26 | \\fn loadFromFile(file_path: []const u8) !Yaml { 27 | \\ const file = try std.fs.openFileAbsolute(file_path, .{}); 28 | \\ defer file.close(); 29 | \\ 30 | \\ const source = try file.readToEndAlloc(alloc, std.math.maxInt(u32)); 31 | \\ defer alloc.free(source); 32 | \\ 33 | \\ var yaml: Yaml = .{ .source = source }; 34 | \\ errdefer yaml.deinit(alloc); 35 | \\ try yaml.load(alloc); 36 | \\ return yaml; 37 | \\} 38 | \\ 39 | \\fn loadFileString(file_path: []const u8) ![]u8 { 40 | \\ const file = try std.fs.openFileAbsolute(file_path, .{}); 41 | \\ defer file.close(); 42 | \\ 43 | \\ const source = try file.readToEndAlloc(alloc, std.math.maxInt(u32)); 44 | \\ return source; 45 | \\} 46 | \\ 47 | ; 48 | 49 | pub fn create(owner: *std.Build) *SpecTest { 50 | const spec_test = owner.allocator.create(SpecTest) catch @panic("OOM"); 51 | 52 | spec_test.* = .{ 53 | .step = Step.init(.{ .id = base_id, .name = "yaml-test-generate", .owner = owner, .makeFn = make }), 54 | .output_file = std.Build.GeneratedFile{ .step = &spec_test.step }, 55 | }; 56 | return spec_test; 57 | } 58 | 59 | pub fn path(spec_test: *SpecTest) std.Build.LazyPath { 60 | return std.Build.LazyPath{ .generated = .{ .file = &spec_test.output_file } }; 61 | } 62 | 63 | const Testcase = struct { 64 | name: []const u8, 65 | path: []const u8, 66 | result: union(enum) { 67 | expected_output_path: []const u8, 68 | error_expected, 69 | none, 70 | skip, 71 | }, 72 | tags: std.BufSet, 73 | }; 74 | 75 | fn make(step: *Step, make_options: Step.MakeOptions) !void { 76 | _ = make_options; 77 | 78 | const spec_test: *SpecTest = @fieldParentPtr("step", step); 79 | const b = step.owner; 80 | 81 | const cwd = std.fs.cwd(); 82 | cwd.access("test/yaml-test-suite/tags", .{}) catch { 83 | return spec_test.step.fail("Testfiles not found, make sure you have loaded the submodule.", .{}); 84 | }; 85 | if (b.graph.host.result.os.tag == .windows) { 86 | return spec_test.step.fail("Windows does not support symlinks in git properly, can't run testsuite.", .{}); 87 | } 88 | 89 | var arena_allocator = std.heap.ArenaAllocator.init(b.allocator); 90 | defer arena_allocator.deinit(); 91 | const arena = arena_allocator.allocator(); 92 | 93 | var testcases = std.StringArrayHashMap(Testcase).init(arena); 94 | 95 | const root_data_path = try fs.path.join(arena, &[_][]const u8{ 96 | b.build_root.path.?, 97 | "test/yaml-test-suite", 98 | }); 99 | 100 | const root_data_dir = try std.fs.openDirAbsolute(root_data_path, .{}); 101 | 102 | var itdir = try root_data_dir.openDir("tags", .{ 103 | .iterate = true, 104 | .access_sub_paths = true, 105 | }); 106 | 107 | var walker = try itdir.walk(arena); 108 | defer walker.deinit(); 109 | 110 | loop: { 111 | while (walker.next()) |maybe_entry| { 112 | if (maybe_entry) |entry| { 113 | if (entry.kind != .sym_link) continue; 114 | collectTest(arena, entry, &testcases) catch |err| switch (err) { 115 | error.OutOfMemory => @panic("OOM"), 116 | else => |e| return e, 117 | }; 118 | } else { 119 | break :loop; 120 | } 121 | } else |err| { 122 | std.debug.print("err: {}", .{err}); 123 | break :loop; 124 | } 125 | } 126 | 127 | var output = std.array_list.Managed(u8).init(arena); 128 | const writer = output.writer(); 129 | try writer.writeAll(preamble); 130 | 131 | while (testcases.pop()) |kv| { 132 | emitTest(arena, &output, kv.value) catch |err| switch (err) { 133 | error.OutOfMemory => @panic("OOM"), 134 | else => |e| return e, 135 | }; 136 | } 137 | 138 | var man = b.graph.cache.obtain(); 139 | defer man.deinit(); 140 | 141 | man.hash.addBytes(output.items); 142 | 143 | if (try step.cacheHit(&man)) { 144 | const digest = man.final(); 145 | spec_test.output_file.path = try b.cache_root.join(b.allocator, &.{ 146 | &digest, test_filename, 147 | }); 148 | return; 149 | } 150 | const digest = man.final(); 151 | 152 | const sub_path = b.pathJoin(&.{ &digest, test_filename }); 153 | const sub_path_dirname = fs.path.dirname(sub_path).?; 154 | 155 | b.cache_root.handle.makePath(sub_path_dirname) catch |err| { 156 | return step.fail("unable to make path '{?s}{s}': {any}", .{ b.cache_root.path, sub_path_dirname, err }); 157 | }; 158 | 159 | b.cache_root.handle.writeFile(.{ .sub_path = sub_path, .data = output.items }) catch |err| { 160 | return step.fail("unable to write file: {}", .{err}); 161 | }; 162 | spec_test.output_file.path = try b.cache_root.join(b.allocator, &.{sub_path}); 163 | try man.writeManifest(); 164 | } 165 | 166 | fn collectTest(arena: Allocator, entry: fs.Dir.Walker.Entry, testcases: *std.StringArrayHashMap(Testcase)) !void { 167 | var path_components_it = try std.fs.path.componentIterator(entry.path); 168 | const first_path = path_components_it.first().?; 169 | 170 | var path_components = std.array_list.Managed([]const u8).init(arena); 171 | while (path_components_it.next()) |component| { 172 | try path_components.append(component.name); 173 | } 174 | 175 | const remaining_path = try fs.path.join(arena, path_components.items); 176 | const result = try testcases.getOrPut(remaining_path); 177 | 178 | if (!result.found_existing) { 179 | result.key_ptr.* = remaining_path; 180 | 181 | const in_path = try fs.path.join(arena, &[_][]const u8{ 182 | entry.basename, 183 | "in.yaml", 184 | }); 185 | const real_in_path = try entry.dir.realpathAlloc(arena, in_path); 186 | 187 | const name_file_path = try fs.path.join(arena, &[_][]const u8{ 188 | entry.basename, 189 | "===", 190 | }); 191 | const name_file = try entry.dir.openFile(name_file_path, .{}); 192 | defer name_file.close(); 193 | const name = try name_file.readToEndAlloc(arena, std.math.maxInt(u32)); 194 | 195 | var tag_set = std.BufSet.init(arena); 196 | try tag_set.insert(first_path.name); 197 | 198 | const full_name = try std.fmt.allocPrint(arena, "{s} - {s}", .{ 199 | remaining_path, 200 | name[0 .. name.len - 1], 201 | }); 202 | 203 | result.value_ptr.* = .{ 204 | .name = full_name, 205 | .path = real_in_path, 206 | .result = .{ .none = {} }, 207 | .tags = tag_set, 208 | }; 209 | 210 | if (skipTest(full_name)) { 211 | result.value_ptr.result = .skip; 212 | return; 213 | } 214 | 215 | const out_path = try fs.path.join(arena, &[_][]const u8{ 216 | entry.basename, 217 | "out.yaml", 218 | }); 219 | const err_path = try fs.path.join(arena, &[_][]const u8{ 220 | entry.basename, 221 | "error", 222 | }); 223 | 224 | if (canAccess(entry.dir, out_path)) { 225 | const real_out_path = try entry.dir.realpathAlloc(arena, out_path); 226 | result.value_ptr.result = .{ .expected_output_path = real_out_path }; 227 | } else if (canAccess(entry.dir, err_path)) { 228 | result.value_ptr.result = .{ .error_expected = {} }; 229 | } 230 | } else { 231 | try result.value_ptr.tags.insert(first_path.name); 232 | } 233 | } 234 | 235 | fn skipTest(name: []const u8) bool { 236 | for (skipped_tests) |skipped_name| { 237 | if (mem.eql(u8, name, skipped_name)) return true; 238 | } 239 | return false; 240 | } 241 | 242 | const skipped_tests = &[_][]const u8{ 243 | "JR7V - Question marks in scalars", 244 | "UDM2 - Plain URL in flow mapping", 245 | "8MK2 - Explicit Non-Specific Tag", 246 | "6XDY - Two document start markers", 247 | "652Z - Question mark at start of flow key", 248 | "PUW8 - Document start on last line", 249 | "FBC9 - Allowed characters in plain scalars", 250 | "5TRB - Invalid document-start marker in doublequoted tring", 251 | "9MQT/01 - Scalar doc with '...' in content", 252 | "9MQT/00 - Scalar doc with '...' in content", 253 | "CPZ3 - Doublequoted scalar starting with a tab", 254 | "8XYN - Anchor with unicode character", 255 | "Y2GN - Anchor with colon in the middle", 256 | "KSS4 - Scalars on --- line", 257 | "FTA2 - Single block sequence with anchor and explicit document start", 258 | "3R3P - Single block sequence with anchor", 259 | "F2C7 - Anchors and Tags", 260 | "TS54 - Folded Block Scalar", 261 | "MZX3 - Non-Specific Tags on Scalars", 262 | "AB8U - Sequence entry that looks like two with wrong indentation", 263 | "9MAG - Flow sequence with invalid comma at the beginning", 264 | "YJV2 - Dash in flow sequence", 265 | "FUP4 - Flow Sequence in Flow Sequence", 266 | "33X3 - Three explicit integers in a block sequence", 267 | "2AUY - Tags in Block Sequence", 268 | "SM9W/00 - Single character streams", 269 | "G5U8 - Plain dashes in flow sequence", 270 | "DHP8 - Flow Sequence", 271 | "3MYT - Plain Scalar looking like key, comment, anchor and tag", 272 | "A984 - Multiline Scalar in Mapping", 273 | "S7BG - Colon followed by comma", 274 | "HM87/00 - Scalars in flow start with syntax char", 275 | "HM87/01 - Scalars in flow start with syntax char", 276 | "4V8U - Plain scalar with backslashes", 277 | "H3Z8 - Literal unicode", 278 | "82AN - Three dashes and content without space", 279 | "BS4K - Comment between plain scalar lines", 280 | "FH7J - Tags on Empty Scalars", 281 | "CQ3W - Double quoted string without closing quote", 282 | "Y79Y/001 - Tabs in various contexts", 283 | "Y79Y/006 - Tabs in various contexts", 284 | "Y79Y/010 - Tabs in various contexts", 285 | "Y79Y/003 - Tabs in various contexts", 286 | "Y79Y/004 - Tabs in various contexts", 287 | "Y79Y/005 - Tabs in various contexts", 288 | "Y79Y/002 - Tabs in various contexts", 289 | "9YRD - Multiline Scalar at Top Level", 290 | "CFD4 - Empty implicit key in single pair flow sequences", 291 | "3UYS - Escaped slash in double quotes", 292 | "Y79Y/008 - Tabs in various contexts", 293 | "UV7Q - Legal tab after indentation", 294 | "SKE5 - Anchor before zero indented sequence", 295 | "EW3V - Wrong indendation in mapping", 296 | "DK95/03 - Tabs that look like indentation", 297 | "DK95/04 - Tabs that look like indentation", 298 | "DK95/05 - Tabs that look like indentation", 299 | "DK95/07 - Tabs that look like indentation", 300 | "DK95/00 - Tabs that look like indentation", 301 | "DK95/01 - Tabs that look like indentation", 302 | "DK95/06 - Tabs that look like indentation", 303 | "ZVH3 - Wrong indented sequence item", 304 | "96NN/00 - Leading tab content in literals", 305 | "96NN/01 - Leading tab content in literals", 306 | "F6MC - More indented lines at the beginning of folded block scalars", 307 | "Y79Y/009 - Tabs in various contexts", 308 | "Y79Y/000 - Tabs in various contexts", 309 | "Y79Y/007 - Tabs in various contexts", 310 | "KH5V/01 - Inline tabs in double quoted", 311 | "KH5V/02 - Inline tabs in double quoted", 312 | "Q5MG - Tab at beginning of line followed by a flow mapping", 313 | "4RWC - Trailing spaces after flow collection", 314 | "LP6E - Whitespace After Scalars in Flow", 315 | "NHX8 - Empty Lines at End of Document", 316 | "NB6Z - Multiline plain value with tabs on empty lines", 317 | "DE56/01 - Trailing tabs in double quoted", 318 | "DE56/00 - Trailing tabs in double quoted", 319 | "DE56/02 - Trailing tabs in double quoted", 320 | "DE56/05 - Trailing tabs in double quoted", 321 | "DE56/04 - Trailing tabs in double quoted", 322 | "DE56/03 - Trailing tabs in double quoted", 323 | "L24T/01 - Trailing line of spaces", 324 | "L24T/00 - Trailing line of spaces", 325 | "3RLN/01 - Leading tabs in double quoted", 326 | "3RLN/04 - Leading tabs in double quoted", 327 | "9MMA - Directive by itself with no document", 328 | "MUS6/06 - Directive variants", 329 | "MUS6/02 - Directive variants", 330 | "MUS6/05 - Directive variants", 331 | "MUS6/04 - Directive variants", 332 | "MUS6/03 - Directive variants", 333 | "XLQ9 - Multiline scalar that looks like a YAML directive", 334 | "M2N8/01 - Question mark edge cases", 335 | "M2N8/00 - Question mark edge cases", 336 | "UKK6/01 - Syntax character edge cases", 337 | "UKK6/00 - Syntax character edge cases", 338 | "UKK6/02 - Syntax character edge cases", 339 | "6H3V - Backslashes in singlequotes", 340 | "U3C3 - Spec Example 6.16. “TAG” directive", 341 | "DBG4 - Spec Example 7.10. Plain Characters", 342 | "MJS9 - Spec Example 6.7. Block Folding", 343 | "96L6 - Spec Example 2.14. In the folded scalars, newlines become spaces", 344 | "4CQQ - Spec Example 2.18. Multi-line Flow Scalars", 345 | "6CK3 - Spec Example 6.26. Tag Shorthands", 346 | "BEC7 - Spec Example 6.14. “YAML” directive", 347 | "WZ62 - Spec Example 7.2. Empty Content", 348 | "5TYM - Spec Example 6.21. Local Tag Prefix", 349 | "27NA - Spec Example 5.9. Directive Indicator", 350 | "JHB9 - Spec Example 2.7. Two Documents in a Stream", 351 | "LQZ7 - Spec Example 7.4. Double Quoted Implicit Keys", 352 | "S4JQ - Spec Example 6.28. Non-Specific Tags", 353 | "G992 - Spec Example 8.9. Folded Scalar", 354 | "YD5X - Spec Example 2.5. Sequence of Sequences", 355 | "8UDB - Spec Example 7.14. Flow Sequence Entries", 356 | "6ZKB - Spec Example 9.6. Stream", 357 | "G4RS - Spec Example 2.17. Quoted Scalars", 358 | "6LVF - Spec Example 6.13. Reserved Directives", 359 | "5KJE - Spec Example 7.13. Flow Sequence", 360 | "6VJK - Spec Example 2.15. Folded newlines are preserved for \"more indented\" and blank lines", 361 | "K527 - Spec Example 6.6. Line Folding", 362 | "SU5Z - Comment without whitespace after doublequoted scalar", 363 | "L383 - Two scalar docs with trailing comments", 364 | "DC7X - Various trailing tabs", 365 | "U3XV - Node and Mapping Key Anchors", 366 | "Q9WF - Spec Example 6.12. Separation Spaces", 367 | "7T8X - Spec Example 8.10. Folded Lines - 8.13. Final Empty Lines", 368 | "CML9 - Missing comma in flow", 369 | "P94K - Spec Example 6.11. Multi-Line Comments", 370 | "7TMG - Comment in flow sequence before comma", 371 | "DK3J - Zero indented block scalar with line that looks like a comment", 372 | "SYW4 - Spec Example 2.2. Mapping Scalars to Scalars", 373 | "735Y - Spec Example 8.20. Block Node Types", 374 | "B3HG - Spec Example 8.9. Folded Scalar [1.3]", 375 | "6WLZ - Spec Example 6.18. Primary Tag Handle [1.3]", 376 | "EX5H - Multiline Scalar at Top Level [1.3]", 377 | "4Q9F - Folded Block Scalar [1.3]", 378 | "Q8AD - Spec Example 7.5. Double Quoted Line Breaks [1.3]", 379 | "6WPF - Spec Example 6.8. Flow Folding [1.3]", 380 | "SSW6 - Spec Example 7.7. Single Quoted Characters [1.3]", 381 | "9DXL - Spec Example 9.6. Stream [1.3]", 382 | "EXG3 - Three dashes and content without space [1.3]", 383 | "T4YY - Spec Example 7.9. Single Quoted Lines [1.3]", 384 | "9TFX - Spec Example 7.6. Double Quoted Lines [1.3]", 385 | "93WF - Spec Example 6.6. Line Folding [1.3]", 386 | "52DL - Explicit Non-Specific Tag [1.3]", 387 | "2LFX - Spec Example 6.13. Reserved Directives [1.3]", 388 | "PW8X - Anchors on Empty Scalars", 389 | "XW4D - Various Trailing Comments", 390 | "NP9H - Spec Example 7.5. Double Quoted Line Breaks", 391 | "HS5T - Spec Example 7.12. Plain Lines", 392 | "J3BT - Spec Example 5.12. Tabs and Spaces", 393 | "PRH3 - Spec Example 7.9. Single Quoted Lines", 394 | "7A4E - Spec Example 7.6. Double Quoted Lines", 395 | "TL85 - Spec Example 6.8. Flow Folding", 396 | "8G76 - Spec Example 6.10. Comment Lines", 397 | "98YD - Spec Example 5.5. Comment Indicator", 398 | "M29M - Literal Block Scalar", 399 | "P2AD - Spec Example 8.1. Block Scalar Header", 400 | "T26H - Spec Example 8.8. Literal Content [1.3]", 401 | "W42U - Spec Example 8.15. Block Sequence Entry Types", 402 | "XV9V - Spec Example 6.5. Empty Lines [1.3]", 403 | "5GBF - Spec Example 6.5. Empty Lines", 404 | "JEF9/01 - Trailing whitespace in streams", 405 | "JEF9/00 - Trailing whitespace in streams", 406 | "JEF9/02 - Trailing whitespace in streams", 407 | "A6F9 - Spec Example 8.4. Chomping Final Line Break", 408 | "4ZYM - Spec Example 6.4. Line Prefixes", 409 | "6FWR - Block Scalar Keep", 410 | "2G84/01 - Literal modifers", 411 | "2G84/00 - Literal modifers", 412 | "DWX9 - Spec Example 8.8. Literal Content", 413 | "F8F9 - Spec Example 8.5. Chomping Trailing Lines", 414 | "MYW6 - Block Scalar Strip", 415 | "H2RW - Blank lines", 416 | "6JQW - Spec Example 2.13. In literals, newlines are preserved", 417 | "K858 - Spec Example 8.6. Empty Scalar Chomping", 418 | "5BVJ - Spec Example 5.7. Block Scalar Indicators", 419 | "T5N4 - Spec Example 8.7. Literal Scalar [1.3]", 420 | "M9B4 - Spec Example 8.7. Literal Scalar", 421 | "753E - Block Scalar Strip [1.3]", 422 | "HMK4 - Spec Example 2.16. Indentation determines scope", 423 | "Z9M4 - Spec Example 6.22. Global Tag Prefix", 424 | "9WXW - Spec Example 6.18. Primary Tag Handle", 425 | "565N - Construct Binary", 426 | "P76L - Spec Example 6.19. Secondary Tag Handle", 427 | "CC74 - Spec Example 6.20. Tag Handles", 428 | "CUP7 - Spec Example 5.6. Node Property Indicators", 429 | "6M2F - Aliases in Explicit Block Mapping", 430 | "HMQ5 - Spec Example 6.23. Node Properties", 431 | "JS2J - Spec Example 6.29. Node Anchors", 432 | "LE5A - Spec Example 7.24. Flow Nodes", 433 | "C4HZ - Spec Example 2.24. Global Tags", 434 | "X38W - Aliases in Flow Objects", 435 | "W5VH - Allowed characters in alias", 436 | "V55R - Aliases in Block Sequence", 437 | "6KGN - Anchor for empty node", 438 | "4QFQ - Spec Example 8.2. Block Indentation Indicator [1.3]", 439 | "R4YG - Spec Example 8.2. Block Indentation Indicator", 440 | "6BCT - Spec Example 6.3. Separation Spaces", 441 | "UT92 - Spec Example 9.4. Explicit Documents", 442 | "7Z25 - Bare document after document end marker", 443 | "EB22 - Missing document-end marker before directive", 444 | "3HFZ - Invalid content after document end marker", 445 | "QT73 - Comment and document-end marker", 446 | "HWV9 - Document-end marker", 447 | "RXY3 - Invalid document-end marker in single quoted string", 448 | "RTP8 - Spec Example 9.2. Document Markers", 449 | "W4TN - Spec Example 9.5. Directives Documents", 450 | "M7A3 - Spec Example 9.3. Bare Documents", 451 | "RZT7 - Spec Example 2.28. Log File", 452 | "5T43 - Colon at the beginning of adjacent flow scalar", 453 | "7BUB - Spec Example 2.10. Node for “Sammy Sosa” appears twice in this document", 454 | "5C5M - Spec Example 7.15. Flow Mappings", 455 | "ZCZ6 - Invalid mapping in plain single line value", 456 | "5MUD - Colon and adjacent value on next line", 457 | "54T7 - Flow Mapping", 458 | "6SLA - Allowed characters in quoted mapping key", 459 | "X8DW - Explicit key and value seperated by comment", 460 | "S3PD - Spec Example 8.18. Implicit Block Mapping Entries", 461 | "4ABK - Flow Mapping Separate Values", 462 | "8KB6 - Multiline plain flow mapping key without value", 463 | "7W2P - Block Mapping with Missing Values", 464 | "ZWK4 - Key with anchor after missing explicit mapping value", 465 | "2SXE - Anchors With Colon in Name", 466 | "4FJ6 - Nested implicit complex keys", 467 | "ZF4X - Spec Example 2.6. Mapping of Mappings", 468 | "ZH7C - Anchors in Mapping", 469 | "TE2A - Spec Example 8.16. Block Mappings", 470 | "SM9W/01 - Single character streams", 471 | "KK5P - Various combinations of explicit block mappings", 472 | "5U3A - Sequence on same Line as Mapping Key", 473 | "8QBE - Block Sequence in Block Mapping", 474 | "26DV - Whitespace around colon in mappings", 475 | "CT4Q - Spec Example 7.20. Single Pair Explicit Entry", 476 | "NKF9 - Empty keys in block and flow mapping", 477 | "R52L - Nested flow mapping sequence and mappings", 478 | "87E4 - Spec Example 7.8. Single Quoted Implicit Keys", 479 | "UGM3 - Spec Example 2.27. Invoice", 480 | "NJ66 - Multiline plain flow mapping key", 481 | "QF4Y - Spec Example 7.19. Single Pair Flow Mappings", 482 | "E76Z - Aliases in Implicit Block Mapping", 483 | "DFF7 - Spec Example 7.16. Flow Mapping Entries", 484 | "6JWB - Tags for Block Objects", 485 | "2JQS - Block Mapping with Missing Keys", 486 | "D88J - Flow Sequence in Block Mapping", 487 | "3GZX - Spec Example 7.1. Alias Nodes", 488 | "5NYZ - Spec Example 6.9. Separated Comment", 489 | "8CWC - Plain mapping key ending with colon", 490 | "5WE3 - Spec Example 8.17. Explicit Block Mapping Entries", 491 | "4EJS - Invalid tabs as indendation in a mapping", 492 | "JTV5 - Block Mapping with Multiline Scalars", 493 | "EHF6 - Tags for Flow Objects", 494 | "M7NX - Nested flow collections", 495 | "CN3R - Various location of anchors in flow sequence", 496 | "K3WX - Colon and adjacent value after comment on next line", 497 | "C2DT - Spec Example 7.18. Flow Mapping Adjacent Values", 498 | "36F6 - Multiline plain scalar with empty line", 499 | "Q88A - Spec Example 7.23. Flow Content", 500 | "L9U5 - Spec Example 7.11. Plain Implicit Keys", 501 | "F3CP - Nested flow collections on one line", 502 | "93JH - Block Mappings in Block Sequence", 503 | "V9D5 - Spec Example 8.19. Compact Block Mappings", 504 | "74H7 - Tags in Implicit Mapping", 505 | "RR7F - Mixed Block Mapping (implicit to explicit)", 506 | "J9HZ - Spec Example 2.9. Single Document with Two Comments", 507 | "229Q - Spec Example 2.4. Sequence of Mappings", 508 | "57H4 - Spec Example 8.22. Block Collection Nodes", 509 | "9SA2 - Multiline double quoted flow mapping key", 510 | "MXS3 - Flow Mapping in Block Sequence", 511 | "L94M - Tags in Explicit Mapping", 512 | "J7VC - Empty Lines Between Mapping Elements", 513 | "J7PZ - Spec Example 2.26. Ordered Mappings", 514 | "9KAX - Various combinations of tags and anchors", 515 | "7ZZ5 - Empty flow collections", 516 | "9U5K - Spec Example 2.12. Compact Nested Mapping", 517 | "6PBE - Zero-indented sequences in explicit mapping keys", 518 | "ZL4Z - Invalid nested mapping", 519 | "S4T7 - Document with footer", 520 | "4MUZ/01 - Flow mapping colon on line after key", 521 | "4MUZ/00 - Flow mapping colon on line after key", 522 | "4MUZ/02 - Flow mapping colon on line after key", 523 | "9KBC - Mapping starting at --- line", 524 | "9BXH - Multiline doublequoted flow mapping key without value", 525 | "9MMW - Single Pair Implicit Entries", 526 | "7BMT - Node and Mapping Key Anchors [1.3]", 527 | "LX3P - Implicit Flow Mapping Key on one line", 528 | "PBJ2 - Spec Example 2.3. Mapping Scalars to Sequences", 529 | "JQ4R - Spec Example 8.14. Block Sequence", 530 | "2EBW - Allowed characters in keys", 531 | "SBG9 - Flow Sequence in Flow Mapping", 532 | "UDR7 - Spec Example 5.4. Flow Collection Indicators", 533 | "FRK4 - Spec Example 7.3. Completely Empty Flow Nodes", 534 | "35KP - Tags for Root Objects", 535 | "58MP - Flow mapping edge cases", 536 | "S9E8 - Spec Example 5.3. Block Structure Indicators", 537 | "6BFJ - Mapping, key and flow sequence item anchors", 538 | "RZP5 - Various Trailing Comments [1.3]", 539 | "2XXW - Spec Example 2.25. Unordered Sets", 540 | "7FWL - Spec Example 6.24. Verbatim Tags", 541 | "M5DY - Spec Example 2.11. Mapping between Sequences", 542 | "GH63 - Mixed Block Mapping (explicit to implicit)", 543 | "HU3P - Invalid Mapping in plain scalar", 544 | "6HB6 - Spec Example 6.1. Indentation Spaces", 545 | "FP8R - Zero indented block scalar", 546 | "Z67P - Spec Example 8.21. Block Scalar Nodes [1.3]", 547 | "A2M4 - Spec Example 6.2. Indentation Indicators", 548 | "VJP3/01 - Flow collections over many lines", 549 | "6CA3 - Tab indented top flow", 550 | "BU8L - Node Anchor and Tag on Seperate Lines", 551 | "4HVU - Wrong indendation in Sequence", 552 | "U44R - Bad indentation in mapping (2)", 553 | "DMG6 - Wrong indendation in Map", 554 | "ZK9H - Nested top level flow mapping", 555 | "M6YH - Block sequence indentation", 556 | "M5C3 - Spec Example 8.21. Block Scalar Nodes", 557 | "9C9N - Wrong indented flow sequence", 558 | "N4JP - Bad indentation in mapping", 559 | "4WA9 - Literal scalars", 560 | "QB6E - Wrong indented multiline quoted scalar", 561 | "D83L - Block scalar indicator order", 562 | "RLU9 - Sequence Indent", 563 | "UV7Q - Legal tab after indentation", 564 | "K54U - Tab after document header", 565 | }; 566 | 567 | const skip_test_template = 568 | \\ return error.SkipZigTest; 569 | ; 570 | 571 | const no_output_template = 572 | \\ var yaml = try loadFromFile("{s}"); 573 | \\ defer yaml.deinit(alloc); 574 | \\ 575 | ; 576 | 577 | const expect_file_template = 578 | \\ var yaml = try loadFromFile("{s}"); 579 | \\ defer yaml.deinit(alloc); 580 | \\ 581 | \\ const expected = try loadFileString("{s}"); 582 | \\ defer alloc.free(expected); 583 | \\ 584 | \\ var buf = std.ArrayList(u8).init(alloc); 585 | \\ defer buf.deinit(); 586 | \\ try yaml.stringify(&buf.writer()); 587 | \\ const actual = try buf.toOwnedSlice(); 588 | \\ try testing.expect(std.meta.eql(expected, actual)); 589 | \\ 590 | ; 591 | 592 | const expect_err_template = 593 | \\ var yaml = loadFromFile("{s}") catch return; 594 | \\ defer yaml.deinit(alloc); 595 | \\ return error.UnexpectedSuccess; 596 | \\ 597 | ; 598 | 599 | fn emitTest(arena: Allocator, output: *std.array_list.Managed(u8), testcase: Testcase) !void { 600 | const head = try std.fmt.allocPrint(arena, "test \"{f}\" {{\n", .{ 601 | std.zig.fmtString(testcase.name), 602 | }); 603 | try output.appendSlice(head); 604 | 605 | switch (testcase.result) { 606 | .skip => { 607 | try output.appendSlice(skip_test_template); 608 | }, 609 | .none => { 610 | const body = try std.fmt.allocPrint(arena, no_output_template, .{ 611 | testcase.path, 612 | }); 613 | try output.appendSlice(body); 614 | }, 615 | .expected_output_path => { 616 | const body = try std.fmt.allocPrint(arena, expect_file_template, .{ 617 | testcase.path, 618 | testcase.result.expected_output_path, 619 | }); 620 | try output.appendSlice(body); 621 | }, 622 | .error_expected => { 623 | const body = try std.fmt.allocPrint(arena, expect_err_template, .{ 624 | testcase.path, 625 | }); 626 | try output.appendSlice(body); 627 | }, 628 | } 629 | 630 | try output.appendSlice("}\n\n"); 631 | } 632 | 633 | fn canAccess(dir: fs.Dir, file_path: []const u8) bool { 634 | if (dir.access(file_path, .{})) { 635 | return true; 636 | } else |_| { 637 | return false; 638 | } 639 | } 640 | -------------------------------------------------------------------------------- /src/Parser.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const assert = std.debug.assert; 3 | const log = std.log.scoped(.parser); 4 | const mem = std.mem; 5 | 6 | const Allocator = mem.Allocator; 7 | const ErrorBundle = std.zig.ErrorBundle; 8 | const LineCol = Tree.LineCol; 9 | const List = Tree.List; 10 | const Map = Tree.Map; 11 | const Node = Tree.Node; 12 | const Tokenizer = @import("Tokenizer.zig"); 13 | const Token = Tokenizer.Token; 14 | const TokenIterator = Tokenizer.TokenIterator; 15 | const TokenWithLineCol = Tree.TokenWithLineCol; 16 | const Tree = @import("Tree.zig"); 17 | const String = Tree.String; 18 | const Parser = @This(); 19 | const Yaml = @import("Yaml.zig"); 20 | 21 | source: []const u8, 22 | tokens: std.MultiArrayList(TokenWithLineCol) = .empty, 23 | token_it: TokenIterator = undefined, 24 | docs: std.ArrayListUnmanaged(Node.Index) = .empty, 25 | nodes: std.MultiArrayList(Node) = .empty, 26 | extra: std.ArrayListUnmanaged(u32) = .empty, 27 | string_bytes: std.ArrayListUnmanaged(u8) = .empty, 28 | errors: ErrorBundle.Wip, 29 | 30 | pub fn init(gpa: Allocator, source: []const u8) Allocator.Error!Parser { 31 | var self: Parser = .{ .source = source, .errors = undefined }; 32 | try self.errors.init(gpa); 33 | return self; 34 | } 35 | 36 | pub fn deinit(self: *Parser, gpa: Allocator) void { 37 | self.tokens.deinit(gpa); 38 | self.docs.deinit(gpa); 39 | self.nodes.deinit(gpa); 40 | self.extra.deinit(gpa); 41 | self.string_bytes.deinit(gpa); 42 | self.errors.deinit(); 43 | self.* = undefined; 44 | } 45 | 46 | pub fn parse(self: *Parser, gpa: Allocator) ParseError!void { 47 | var tokenizer = Tokenizer{ .buffer = self.source }; 48 | var line: u32 = 0; 49 | var prev_line_last_col: u32 = 0; 50 | 51 | while (true) { 52 | const tok = tokenizer.next(); 53 | const tok_index = try self.tokens.addOne(gpa); 54 | 55 | self.tokens.set(tok_index, .{ 56 | .token = tok, 57 | .line_col = .{ 58 | .line = line, 59 | .col = @intCast(tok.loc.start - prev_line_last_col), 60 | }, 61 | }); 62 | 63 | switch (tok.id) { 64 | .eof => break, 65 | .new_line => { 66 | line += 1; 67 | prev_line_last_col = @intCast(tok.loc.end); 68 | }, 69 | else => {}, 70 | } 71 | } 72 | 73 | self.token_it = .{ .buffer = self.tokens.items(.token) }; 74 | 75 | self.eatCommentsAndSpace(&.{}); 76 | 77 | while (true) { 78 | self.eatCommentsAndSpace(&.{}); 79 | const tok = self.token_it.next() orelse break; 80 | 81 | log.debug("(main) next {s}@{d}", .{ @tagName(tok.id), @intFromEnum(self.token_it.pos) - 1 }); 82 | 83 | switch (tok.id) { 84 | .eof => break, 85 | else => { 86 | self.token_it.seekBy(-1); 87 | const node_index = try self.doc(gpa); 88 | try self.docs.append(gpa, node_index); 89 | }, 90 | } 91 | } 92 | } 93 | 94 | pub fn toOwnedTree(self: *Parser, gpa: Allocator) Allocator.Error!Tree { 95 | return .{ 96 | .source = self.source, 97 | .tokens = self.tokens.toOwnedSlice(), 98 | .docs = try self.docs.toOwnedSlice(gpa), 99 | .nodes = self.nodes.toOwnedSlice(), 100 | .extra = try self.extra.toOwnedSlice(gpa), 101 | .string_bytes = try self.string_bytes.toOwnedSlice(gpa), 102 | }; 103 | } 104 | 105 | fn addString(self: *Parser, gpa: Allocator, string: []const u8) Allocator.Error!String { 106 | const index: u32 = @intCast(self.string_bytes.items.len); 107 | try self.string_bytes.ensureUnusedCapacity(gpa, string.len); 108 | self.string_bytes.appendSliceAssumeCapacity(string); 109 | return .{ .index = @enumFromInt(index), .len = @intCast(string.len) }; 110 | } 111 | 112 | fn addExtra(self: *Parser, gpa: Allocator, extra: anytype) Allocator.Error!u32 { 113 | const fields = std.meta.fields(@TypeOf(extra)); 114 | try self.extra.ensureUnusedCapacity(gpa, fields.len); 115 | return self.addExtraAssumeCapacity(extra); 116 | } 117 | 118 | fn addExtraAssumeCapacity(self: *Parser, extra: anytype) u32 { 119 | const result: u32 = @intCast(self.extra.items.len); 120 | self.extra.appendSliceAssumeCapacity(&payloadToExtraItems(extra)); 121 | return result; 122 | } 123 | 124 | fn payloadToExtraItems(data: anytype) [@typeInfo(@TypeOf(data)).@"struct".fields.len]u32 { 125 | const fields = @typeInfo(@TypeOf(data)).@"struct".fields; 126 | var result: [fields.len]u32 = undefined; 127 | inline for (&result, fields) |*val, field| { 128 | val.* = switch (field.type) { 129 | u32 => @field(data, field.name), 130 | i32 => @bitCast(@field(data, field.name)), 131 | Node.Index, Node.OptionalIndex, Token.Index => @intFromEnum(@field(data, field.name)), 132 | else => @compileError("bad field type: " ++ @typeName(field.type)), 133 | }; 134 | } 135 | return result; 136 | } 137 | 138 | fn value(self: *Parser, gpa: Allocator) ParseError!Node.OptionalIndex { 139 | self.eatCommentsAndSpace(&.{}); 140 | 141 | const pos = self.token_it.pos; 142 | const tok = self.token_it.next() orelse return error.UnexpectedEof; 143 | 144 | log.debug(" next {s}@{d}", .{ @tagName(tok.id), pos }); 145 | 146 | switch (tok.id) { 147 | .literal => if (self.eatToken(.map_value_ind, &.{ .new_line, .comment })) |_| { 148 | // map 149 | self.token_it.seekTo(pos); 150 | return self.map(gpa); 151 | } else { 152 | // leaf value 153 | self.token_it.seekTo(pos); 154 | return self.leafValue(gpa); 155 | }, 156 | .single_quoted, .double_quoted => { 157 | // leaf value 158 | self.token_it.seekBy(-1); 159 | return self.leafValue(gpa); 160 | }, 161 | .seq_item_ind => { 162 | // list 163 | self.token_it.seekBy(-1); 164 | return self.list(gpa); 165 | }, 166 | .flow_seq_start => { 167 | // list 168 | self.token_it.seekBy(-1); 169 | return self.listBracketed(gpa); 170 | }, 171 | else => return .none, 172 | } 173 | } 174 | 175 | fn doc(self: *Parser, gpa: Allocator) ParseError!Node.Index { 176 | const node_index = try self.nodes.addOne(gpa); 177 | const node_start = self.token_it.pos; 178 | 179 | log.debug("(doc) begin {s}@{d}", .{ @tagName(self.token(node_start).id), node_start }); 180 | 181 | // Parse header 182 | const header: union(enum) { 183 | directive: Token.Index, 184 | explicit, 185 | implicit, 186 | } = if (self.eatToken(.doc_start, &.{})) |doc_pos| explicit: { 187 | if (self.getCol(doc_pos) > 0) return error.MalformedYaml; 188 | if (self.eatToken(.tag, &.{ .new_line, .comment })) |_| { 189 | break :explicit .{ .directive = try self.expectToken(.literal, &.{ .new_line, .comment }) }; 190 | } 191 | break :explicit .explicit; 192 | } else .implicit; 193 | const directive = switch (header) { 194 | .directive => |index| index, 195 | else => null, 196 | }; 197 | const is_explicit = switch (header) { 198 | .directive, .explicit => true, 199 | .implicit => false, 200 | }; 201 | 202 | // Parse value 203 | const value_index = try self.value(gpa); 204 | if (value_index == .none) { 205 | self.token_it.seekBy(-1); 206 | } 207 | 208 | // Parse footer 209 | const node_end: Token.Index = footer: { 210 | if (self.eatToken(.doc_end, &.{})) |pos| { 211 | if (!is_explicit) { 212 | self.token_it.seekBy(-1); 213 | return self.fail(gpa, self.token_it.pos, "missing explicit document open marker '---'", .{}); 214 | } 215 | if (self.getCol(pos) > 0) return error.MalformedYaml; 216 | break :footer pos; 217 | } 218 | if (self.eatToken(.doc_start, &.{})) |pos| { 219 | if (!is_explicit) return error.UnexpectedToken; 220 | if (self.getCol(pos) > 0) return error.MalformedYaml; 221 | self.token_it.seekBy(-1); 222 | break :footer @enumFromInt(@intFromEnum(pos) - 1); 223 | } 224 | if (self.eatToken(.eof, &.{})) |pos| { 225 | break :footer @enumFromInt(@intFromEnum(pos) - 1); 226 | } 227 | 228 | return self.fail(gpa, self.token_it.pos, "expected end of document", .{}); 229 | }; 230 | 231 | log.debug("(doc) end {s}@{d}", .{ @tagName(self.token(node_end).id), node_end }); 232 | 233 | self.nodes.set(node_index, .{ 234 | .tag = if (directive == null) .doc else .doc_with_directive, 235 | .scope = .{ 236 | .start = node_start, 237 | .end = node_end, 238 | }, 239 | .data = if (directive == null) .{ 240 | .maybe_node = value_index, 241 | } else .{ 242 | .doc_with_directive = .{ 243 | .maybe_node = value_index, 244 | .directive = directive.?, 245 | }, 246 | }, 247 | }); 248 | 249 | return @enumFromInt(node_index); 250 | } 251 | 252 | fn map(self: *Parser, gpa: Allocator) ParseError!Node.OptionalIndex { 253 | const node_index = try self.nodes.addOne(gpa); 254 | const node_start = self.token_it.pos; 255 | 256 | var entries: std.ArrayListUnmanaged(Map.Entry) = .empty; 257 | defer entries.deinit(gpa); 258 | 259 | log.debug("(map) begin {s}@{d}", .{ @tagName(self.token(node_start).id), node_start }); 260 | 261 | const col = self.getCol(node_start); 262 | 263 | while (true) { 264 | self.eatCommentsAndSpace(&.{}); 265 | 266 | // Parse key 267 | const key_pos = self.token_it.pos; 268 | if (self.getCol(key_pos) < col) break; 269 | 270 | const key = self.token_it.next() orelse return error.UnexpectedEof; 271 | switch (key.id) { 272 | .literal => {}, 273 | .doc_start, .doc_end, .eof => { 274 | self.token_it.seekBy(-1); 275 | break; 276 | }, 277 | .flow_map_end => { 278 | break; 279 | }, 280 | else => return self.fail(gpa, self.token_it.pos, "unexpected token for 'key': {}", .{key}), 281 | } 282 | 283 | log.debug("(map) key {s}@{d}", .{ self.rawString(key_pos, key_pos), key_pos }); 284 | 285 | // Separator 286 | _ = self.expectToken(.map_value_ind, &.{ .new_line, .comment }) catch 287 | return self.fail(gpa, self.token_it.pos, "expected map separator ':'", .{}); 288 | 289 | // Parse value 290 | const value_index = try self.value(gpa); 291 | 292 | if (value_index.unwrap()) |v| { 293 | const value_start = self.nodes.items(.scope)[@intFromEnum(v)].start; 294 | if (self.getCol(value_start) < self.getCol(key_pos)) { 295 | return error.MalformedYaml; 296 | } 297 | if (self.nodes.items(.tag)[@intFromEnum(v)] == .value) { 298 | if (self.getCol(value_start) == self.getCol(key_pos)) { 299 | return self.fail(gpa, value_start, "'value' in map should have more indentation than the 'key'", .{}); 300 | } 301 | } 302 | } 303 | 304 | try entries.append(gpa, .{ 305 | .key = key_pos, 306 | .maybe_node = value_index, 307 | }); 308 | } 309 | 310 | const node_end: Token.Index = @enumFromInt(@intFromEnum(self.token_it.pos) - 1); 311 | 312 | log.debug("(map) end {s}@{d}", .{ @tagName(self.token(node_end).id), node_end }); 313 | 314 | const scope: Node.Scope = .{ 315 | .start = node_start, 316 | .end = node_end, 317 | }; 318 | 319 | if (entries.items.len == 1) { 320 | const entry = entries.items[0]; 321 | 322 | self.nodes.set(node_index, .{ 323 | .tag = .map_single, 324 | .scope = scope, 325 | .data = .{ .map = .{ 326 | .key = entry.key, 327 | .maybe_node = entry.maybe_node, 328 | } }, 329 | }); 330 | } else { 331 | try self.extra.ensureUnusedCapacity(gpa, entries.items.len * 2 + 1); 332 | const extra_index: u32 = @intCast(self.extra.items.len); 333 | 334 | _ = self.addExtraAssumeCapacity(Map{ .map_len = @intCast(entries.items.len) }); 335 | 336 | for (entries.items) |entry| { 337 | _ = self.addExtraAssumeCapacity(entry); 338 | } 339 | 340 | self.nodes.set(node_index, .{ 341 | .tag = .map_many, 342 | .scope = scope, 343 | .data = .{ .extra = @enumFromInt(extra_index) }, 344 | }); 345 | } 346 | 347 | return @as(Node.Index, @enumFromInt(node_index)).toOptional(); 348 | } 349 | 350 | fn list(self: *Parser, gpa: Allocator) ParseError!Node.OptionalIndex { 351 | const node_index: Node.Index = @enumFromInt(try self.nodes.addOne(gpa)); 352 | const node_start = self.token_it.pos; 353 | 354 | var values: std.ArrayListUnmanaged(List.Entry) = .empty; 355 | defer values.deinit(gpa); 356 | 357 | const first_col = self.getCol(node_start); 358 | 359 | log.debug("(list) begin {s}@{d}", .{ @tagName(self.token(node_start).id), node_start }); 360 | 361 | while (true) { 362 | self.eatCommentsAndSpace(&.{}); 363 | 364 | const pos = self.eatToken(.seq_item_ind, &.{}) orelse { 365 | log.debug("(list {d}) break", .{first_col}); 366 | break; 367 | }; 368 | const cur_col = self.getCol(pos); 369 | if (cur_col < first_col) { 370 | log.debug("(list {d}) << break", .{first_col}); 371 | // this hyphen belongs to an outer list 372 | self.token_it.seekBy(-1); 373 | // this will end this list 374 | break; 375 | } 376 | // an inner list will be parsed by self.value() so 377 | // checking for cur_col > first_col is not necessary here 378 | 379 | const value_index = try self.value(gpa); 380 | if (value_index == .none) return error.MalformedYaml; 381 | 382 | try values.append(gpa, .{ .node = value_index.unwrap().? }); 383 | } 384 | 385 | const node_end: Token.Index = @enumFromInt(@intFromEnum(self.token_it.pos) - 1); 386 | 387 | log.debug("(list) end {s}@{d}", .{ @tagName(self.token(node_end).id), node_end }); 388 | 389 | try self.encodeList(gpa, node_index, values.items, .{ 390 | .start = node_start, 391 | .end = node_end, 392 | }); 393 | 394 | return node_index.toOptional(); 395 | } 396 | 397 | fn listBracketed(self: *Parser, gpa: Allocator) ParseError!Node.OptionalIndex { 398 | const node_index: Node.Index = @enumFromInt(try self.nodes.addOne(gpa)); 399 | const node_start = self.token_it.pos; 400 | 401 | var values: std.ArrayListUnmanaged(List.Entry) = .empty; 402 | defer values.deinit(gpa); 403 | 404 | log.debug("(list) begin {s}@{d}", .{ @tagName(self.token(node_start).id), node_start }); 405 | 406 | _ = try self.expectToken(.flow_seq_start, &.{}); 407 | 408 | const node_end: Token.Index = while (true) { 409 | self.eatCommentsAndSpace(&.{.comment}); 410 | 411 | if (self.eatToken(.flow_seq_end, &.{.comment})) |pos| 412 | break pos; 413 | 414 | _ = self.eatToken(.comma, &.{.comment}); 415 | 416 | if (self.eatToken(.flow_seq_end, &.{.comment})) |pos| 417 | break pos; 418 | 419 | const value_raw_pos = self.token_it.pos; 420 | const value_index = try self.value(gpa); 421 | if (value_index == .none) { 422 | return self.fail(gpa, value_raw_pos, "expecting a value in flow sequence", .{}); 423 | } 424 | 425 | try values.append(gpa, .{ .node = value_index.unwrap().? }); 426 | }; 427 | 428 | if (self.eatToken(.comment, &.{.comment})) |pos| { 429 | return self.fail(gpa, pos, "comments must be separated from other tokens by white space characters", .{}); 430 | } 431 | 432 | log.debug("(list) end {s}@{d}", .{ @tagName(self.token(node_end).id), node_end }); 433 | 434 | try self.encodeList(gpa, node_index, values.items, .{ 435 | .start = node_start, 436 | .end = node_end, 437 | }); 438 | 439 | return node_index.toOptional(); 440 | } 441 | 442 | fn encodeList( 443 | self: *Parser, 444 | gpa: Allocator, 445 | node_index: Node.Index, 446 | values: []const List.Entry, 447 | node_scope: Node.Scope, 448 | ) Allocator.Error!void { 449 | const index = @intFromEnum(node_index); 450 | switch (values.len) { 451 | 0 => { 452 | self.nodes.set(index, .{ 453 | .tag = .list_empty, 454 | .scope = node_scope, 455 | .data = undefined, 456 | }); 457 | }, 458 | 1 => { 459 | self.nodes.set(index, .{ 460 | .tag = .list_one, 461 | .scope = node_scope, 462 | .data = .{ .node = values[0].node }, 463 | }); 464 | }, 465 | 2 => { 466 | self.nodes.set(index, .{ 467 | .tag = .list_two, 468 | .scope = node_scope, 469 | .data = .{ .list = .{ 470 | .el1 = values[0].node, 471 | .el2 = values[1].node, 472 | } }, 473 | }); 474 | }, 475 | else => { 476 | try self.extra.ensureUnusedCapacity(gpa, values.len + 1); 477 | const extra_index: u32 = @intCast(self.extra.items.len); 478 | 479 | _ = self.addExtraAssumeCapacity(List{ .list_len = @intCast(values.len) }); 480 | 481 | for (values) |entry| { 482 | _ = self.addExtraAssumeCapacity(entry); 483 | } 484 | 485 | self.nodes.set(index, .{ 486 | .tag = .list_many, 487 | .scope = node_scope, 488 | .data = .{ .extra = @enumFromInt(extra_index) }, 489 | }); 490 | }, 491 | } 492 | } 493 | 494 | fn leafValue(self: *Parser, gpa: Allocator) ParseError!Node.OptionalIndex { 495 | const node_index: Node.Index = @enumFromInt(try self.nodes.addOne(gpa)); 496 | const node_start = self.token_it.pos; 497 | 498 | // TODO handle multiline strings in new block scope 499 | while (self.token_it.next()) |tok| { 500 | switch (tok.id) { 501 | .single_quoted => { 502 | const node_end: Token.Index = @enumFromInt(@intFromEnum(self.token_it.pos) - 1); 503 | const raw = self.rawString(node_start, node_end); 504 | log.debug("(leaf) {s}", .{raw}); 505 | assert(raw.len > 0); 506 | const string = try self.parseSingleQuoted(gpa, raw); 507 | 508 | self.nodes.set(@intFromEnum(node_index), .{ 509 | .tag = .string_value, 510 | .scope = .{ 511 | .start = node_start, 512 | .end = node_end, 513 | }, 514 | .data = .{ .string = string }, 515 | }); 516 | 517 | return node_index.toOptional(); 518 | }, 519 | .double_quoted => { 520 | const node_end: Token.Index = @enumFromInt(@intFromEnum(self.token_it.pos) - 1); 521 | const raw = self.rawString(node_start, node_end); 522 | log.debug("(leaf) {s}", .{raw}); 523 | assert(raw.len > 0); 524 | const string = try self.parseDoubleQuoted(gpa, raw); 525 | 526 | self.nodes.set(@intFromEnum(node_index), .{ 527 | .tag = .string_value, 528 | .scope = .{ 529 | .start = node_start, 530 | .end = node_end, 531 | }, 532 | .data = .{ .string = string }, 533 | }); 534 | 535 | return node_index.toOptional(); 536 | }, 537 | .literal => {}, 538 | .space => { 539 | const trailing = @intFromEnum(self.token_it.pos) - 2; 540 | self.eatCommentsAndSpace(&.{}); 541 | if (self.token_it.peek()) |peek| { 542 | if (peek.id != .literal) { 543 | const node_end: Token.Index = @enumFromInt(trailing); 544 | log.debug("(leaf) {s}", .{self.rawString(node_start, node_end)}); 545 | self.nodes.set(@intFromEnum(node_index), .{ 546 | .tag = .value, 547 | .scope = .{ 548 | .start = node_start, 549 | .end = node_end, 550 | }, 551 | .data = undefined, 552 | }); 553 | return node_index.toOptional(); 554 | } 555 | } 556 | }, 557 | else => { 558 | self.token_it.seekBy(-1); 559 | const node_end: Token.Index = @enumFromInt(@intFromEnum(self.token_it.pos) - 1); 560 | log.debug("(leaf) {s}", .{self.rawString(node_start, node_end)}); 561 | self.nodes.set(@intFromEnum(node_index), .{ 562 | .tag = .value, 563 | .scope = .{ 564 | .start = node_start, 565 | .end = node_end, 566 | }, 567 | .data = undefined, 568 | }); 569 | return node_index.toOptional(); 570 | }, 571 | } 572 | } 573 | 574 | return error.MalformedYaml; 575 | } 576 | 577 | fn eatCommentsAndSpace(self: *Parser, comptime exclusions: []const Token.Id) void { 578 | log.debug("eatCommentsAndSpace", .{}); 579 | outer: while (self.token_it.next()) |tok| { 580 | log.debug(" (token '{s}')", .{@tagName(tok.id)}); 581 | switch (tok.id) { 582 | .comment, .space, .new_line => |space| { 583 | inline for (exclusions) |excl| { 584 | if (excl == space) { 585 | self.token_it.seekBy(-1); 586 | break :outer; 587 | } 588 | } else continue; 589 | }, 590 | else => { 591 | self.token_it.seekBy(-1); 592 | break; 593 | }, 594 | } 595 | } 596 | } 597 | 598 | fn eatToken(self: *Parser, id: Token.Id, comptime exclusions: []const Token.Id) ?Token.Index { 599 | log.debug("eatToken('{s}')", .{@tagName(id)}); 600 | self.eatCommentsAndSpace(exclusions); 601 | const pos = self.token_it.pos; 602 | const tok = self.token_it.next() orelse return null; 603 | if (tok.id == id) { 604 | log.debug(" (found at {d})", .{pos}); 605 | return pos; 606 | } else { 607 | log.debug(" (not found)", .{}); 608 | self.token_it.seekBy(-1); 609 | return null; 610 | } 611 | } 612 | 613 | fn expectToken(self: *Parser, id: Token.Id, comptime exclusions: []const Token.Id) ParseError!Token.Index { 614 | log.debug("expectToken('{s}')", .{@tagName(id)}); 615 | return self.eatToken(id, exclusions) orelse error.UnexpectedToken; 616 | } 617 | 618 | fn getLine(self: *Parser, index: Token.Index) usize { 619 | return self.tokens.items(.line_col)[@intFromEnum(index)].line; 620 | } 621 | 622 | fn getCol(self: *Parser, index: Token.Index) usize { 623 | return self.tokens.items(.line_col)[@intFromEnum(index)].col; 624 | } 625 | 626 | fn parseSingleQuoted(self: *Parser, gpa: Allocator, raw: []const u8) ParseError!String { 627 | assert(raw[0] == '\'' and raw[raw.len - 1] == '\''); 628 | const raw_no_quotes = raw[1 .. raw.len - 1]; 629 | 630 | try self.string_bytes.ensureUnusedCapacity(gpa, raw_no_quotes.len); 631 | var string: String = .{ 632 | .index = @enumFromInt(@as(u32, @intCast(self.string_bytes.items.len))), 633 | .len = 0, 634 | }; 635 | 636 | var state: enum { 637 | start, 638 | escape, 639 | } = .start; 640 | 641 | var index: usize = 0; 642 | 643 | while (index < raw_no_quotes.len) : (index += 1) { 644 | const c = raw_no_quotes[index]; 645 | switch (state) { 646 | .start => switch (c) { 647 | '\'' => { 648 | state = .escape; 649 | }, 650 | else => { 651 | self.string_bytes.appendAssumeCapacity(c); 652 | string.len += 1; 653 | }, 654 | }, 655 | .escape => switch (c) { 656 | '\'' => { 657 | state = .start; 658 | self.string_bytes.appendAssumeCapacity(c); 659 | string.len += 1; 660 | }, 661 | else => return error.InvalidEscapeSequence, 662 | }, 663 | } 664 | } 665 | 666 | return string; 667 | } 668 | 669 | fn parseDoubleQuoted(self: *Parser, gpa: Allocator, raw: []const u8) ParseError!String { 670 | assert(raw[0] == '"' and raw[raw.len - 1] == '"'); 671 | const raw_no_quotes = raw[1 .. raw.len - 1]; 672 | 673 | try self.string_bytes.ensureUnusedCapacity(gpa, raw_no_quotes.len); 674 | var string: String = .{ 675 | .index = @enumFromInt(@as(u32, @intCast(self.string_bytes.items.len))), 676 | .len = 0, 677 | }; 678 | 679 | var state: enum { 680 | start, 681 | escape, 682 | } = .start; 683 | 684 | var index: usize = 0; 685 | 686 | while (index < raw_no_quotes.len) : (index += 1) { 687 | const c = raw_no_quotes[index]; 688 | switch (state) { 689 | .start => switch (c) { 690 | '\\' => { 691 | state = .escape; 692 | }, 693 | else => { 694 | self.string_bytes.appendAssumeCapacity(c); 695 | string.len += 1; 696 | }, 697 | }, 698 | .escape => switch (c) { 699 | 'n' => { 700 | state = .start; 701 | self.string_bytes.appendAssumeCapacity('\n'); 702 | string.len += 1; 703 | }, 704 | 't' => { 705 | state = .start; 706 | self.string_bytes.appendAssumeCapacity('\t'); 707 | string.len += 1; 708 | }, 709 | '"' => { 710 | state = .start; 711 | self.string_bytes.appendAssumeCapacity('"'); 712 | string.len += 1; 713 | }, 714 | else => return error.InvalidEscapeSequence, 715 | }, 716 | } 717 | } 718 | 719 | return string; 720 | } 721 | 722 | fn rawString(self: Parser, start: Token.Index, end: Token.Index) []const u8 { 723 | const start_token = self.token(start); 724 | const end_token = self.token(end); 725 | return self.source[start_token.loc.start..end_token.loc.end]; 726 | } 727 | 728 | fn token(self: Parser, index: Token.Index) Token { 729 | return self.tokens.items(.token)[@intFromEnum(index)]; 730 | } 731 | 732 | fn fail(self: *Parser, gpa: Allocator, token_index: Token.Index, comptime format: []const u8, args: anytype) ParseError { 733 | const line_col = self.tokens.items(.line_col)[@intFromEnum(token_index)]; 734 | const msg = try std.fmt.allocPrint(gpa, format, args); 735 | defer gpa.free(msg); 736 | const line_info = getLineInfo(self.source, line_col); 737 | try self.errors.addRootErrorMessage(.{ 738 | .msg = try self.errors.addString(msg), 739 | .src_loc = try self.errors.addSourceLocation(.{ 740 | .src_path = try self.errors.addString("(memory)"), 741 | .line = line_col.line, 742 | .column = line_col.col, 743 | .span_start = line_info.span_start, 744 | .span_main = line_info.span_main, 745 | .span_end = line_info.span_end, 746 | .source_line = try self.errors.addString(line_info.line), 747 | }), 748 | .notes_len = 0, 749 | }); 750 | return error.ParseFailure; 751 | } 752 | 753 | fn getLineInfo(source: []const u8, line_col: LineCol) struct { 754 | line: []const u8, 755 | span_start: u32, 756 | span_main: u32, 757 | span_end: u32, 758 | } { 759 | const line = line: { 760 | var it = mem.splitScalar(u8, source, '\n'); 761 | var line_count: usize = 0; 762 | const line = while (it.next()) |line| { 763 | defer line_count += 1; 764 | if (line_count == line_col.line) break line; 765 | } else return .{ 766 | .line = &.{}, 767 | .span_start = 0, 768 | .span_main = 0, 769 | .span_end = 0, 770 | }; 771 | break :line line; 772 | }; 773 | 774 | const span_start: u32 = span_start: { 775 | const trimmed = mem.trimLeft(u8, line, " "); 776 | break :span_start @intCast(mem.indexOf(u8, line, trimmed).?); 777 | }; 778 | 779 | const span_end: u32 = @intCast(mem.trimRight(u8, line, " \r\n").len); 780 | 781 | return .{ 782 | .line = line, 783 | .span_start = span_start, 784 | .span_main = line_col.col, 785 | .span_end = span_end, 786 | }; 787 | } 788 | 789 | pub const ParseError = error{ 790 | InvalidEscapeSequence, 791 | MalformedYaml, 792 | NestedDocuments, 793 | UnexpectedEof, 794 | UnexpectedToken, 795 | ParseFailure, 796 | } || Allocator.Error; 797 | 798 | test { 799 | std.testing.refAllDecls(@This()); 800 | _ = @import("Parser/test.zig"); 801 | } 802 | -------------------------------------------------------------------------------- /src/Yaml/test.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const mem = std.mem; 3 | const stringify = @import("../stringify.zig").stringify; 4 | const testing = std.testing; 5 | 6 | const Arena = std.heap.ArenaAllocator; 7 | const Yaml = @import("../Yaml.zig"); 8 | 9 | test "simple list" { 10 | const source = 11 | \\- a 12 | \\- b 13 | \\- c 14 | ; 15 | 16 | var yaml: Yaml = .{ .source = source }; 17 | defer yaml.deinit(testing.allocator); 18 | try yaml.load(testing.allocator); 19 | 20 | try testing.expectEqual(yaml.docs.items.len, 1); 21 | 22 | const list = yaml.docs.items[0].list; 23 | try testing.expectEqual(list.len, 3); 24 | 25 | try testing.expectEqualStrings("a", list[0].scalar); 26 | try testing.expectEqualStrings("b", list[1].scalar); 27 | try testing.expectEqualStrings("c", list[2].scalar); 28 | } 29 | 30 | test "simple list parsed as booleans" { 31 | const source = 32 | \\- true 33 | \\- false 34 | \\- true 35 | ; 36 | 37 | var yaml: Yaml = .{ .source = source }; 38 | defer yaml.deinit(testing.allocator); 39 | try yaml.load(testing.allocator); 40 | 41 | var arena = Arena.init(testing.allocator); 42 | defer arena.deinit(); 43 | 44 | const parsed = try yaml.parse(arena.allocator(), []const bool); 45 | try testing.expectEqual(parsed.len, 3); 46 | 47 | try testing.expect(parsed[0]); 48 | try testing.expect(!parsed[1]); 49 | try testing.expect(parsed[2]); 50 | } 51 | 52 | test "simple list typed as array of strings" { 53 | const source = 54 | \\- a 55 | \\- b 56 | \\- c 57 | ; 58 | 59 | var yaml: Yaml = .{ .source = source }; 60 | defer yaml.deinit(testing.allocator); 61 | try yaml.load(testing.allocator); 62 | 63 | try testing.expectEqual(yaml.docs.items.len, 1); 64 | 65 | var arena = Arena.init(testing.allocator); 66 | defer arena.deinit(); 67 | 68 | const arr = try yaml.parse(arena.allocator(), [3][]const u8); 69 | try testing.expectEqual(3, arr.len); 70 | try testing.expectEqualStrings("a", arr[0]); 71 | try testing.expectEqualStrings("b", arr[1]); 72 | try testing.expectEqualStrings("c", arr[2]); 73 | } 74 | 75 | test "simple list typed as array of ints" { 76 | const source = 77 | \\- 0 78 | \\- 1 79 | \\- 2 80 | ; 81 | 82 | var yaml: Yaml = .{ .source = source }; 83 | defer yaml.deinit(testing.allocator); 84 | try yaml.load(testing.allocator); 85 | 86 | try testing.expectEqual(yaml.docs.items.len, 1); 87 | 88 | var arena = Arena.init(testing.allocator); 89 | defer arena.deinit(); 90 | 91 | const arr = try yaml.parse(arena.allocator(), [3]u8); 92 | try testing.expectEqualSlices(u8, &[_]u8{ 0, 1, 2 }, &arr); 93 | } 94 | 95 | test "list of mixed sign integer" { 96 | const source = 97 | \\- 0 98 | \\- -1 99 | \\- 2 100 | ; 101 | 102 | var yaml: Yaml = .{ .source = source }; 103 | defer yaml.deinit(testing.allocator); 104 | try yaml.load(testing.allocator); 105 | 106 | try testing.expectEqual(yaml.docs.items.len, 1); 107 | 108 | var arena = Arena.init(testing.allocator); 109 | defer arena.deinit(); 110 | 111 | const arr = try yaml.parse(arena.allocator(), [3]i8); 112 | try testing.expectEqualSlices(i8, &[_]i8{ 0, -1, 2 }, &arr); 113 | } 114 | 115 | test "several integer bases" { 116 | const source = 117 | \\- 10 118 | \\- -10 119 | \\- 0x10 120 | \\- -0X10 121 | \\- 0o10 122 | \\- -0O10 123 | ; 124 | 125 | var yaml: Yaml = .{ .source = source }; 126 | defer yaml.deinit(testing.allocator); 127 | try yaml.load(testing.allocator); 128 | 129 | try testing.expectEqual(yaml.docs.items.len, 1); 130 | 131 | var arena = Arena.init(testing.allocator); 132 | defer arena.deinit(); 133 | 134 | const arr = try yaml.parse(arena.allocator(), [6]i8); 135 | try testing.expectEqualSlices(i8, &[_]i8{ 10, -10, 16, -16, 8, -8 }, &arr); 136 | } 137 | 138 | test "simple flow sequence / bracket list" { 139 | const source = 140 | \\a_key: [a, b, c] 141 | ; 142 | 143 | var yaml: Yaml = .{ .source = source }; 144 | defer yaml.deinit(testing.allocator); 145 | try yaml.load(testing.allocator); 146 | 147 | try testing.expectEqual(yaml.docs.items.len, 1); 148 | 149 | const map = yaml.docs.items[0].map; 150 | 151 | const list = map.get("a_key").?.list; 152 | try testing.expectEqual(list.len, 3); 153 | 154 | try testing.expectEqualStrings("a", list[0].scalar); 155 | try testing.expectEqualStrings("b", list[1].scalar); 156 | try testing.expectEqualStrings("c", list[2].scalar); 157 | } 158 | 159 | test "simple flow sequence / bracket list with trailing comma" { 160 | const source = 161 | \\a_key: [a, b, c,] 162 | ; 163 | 164 | var yaml: Yaml = .{ .source = source }; 165 | defer yaml.deinit(testing.allocator); 166 | try yaml.load(testing.allocator); 167 | 168 | try testing.expectEqual(yaml.docs.items.len, 1); 169 | 170 | const map = yaml.docs.items[0].map; 171 | 172 | const list = map.get("a_key").?.list; 173 | try testing.expectEqual(list.len, 3); 174 | 175 | try testing.expectEqualStrings("a", list[0].scalar); 176 | try testing.expectEqualStrings("b", list[1].scalar); 177 | try testing.expectEqualStrings("c", list[2].scalar); 178 | } 179 | 180 | test "simple flow sequence / bracket list with invalid comment" { 181 | const source = 182 | \\a_key: [a, b, c]#invalid 183 | ; 184 | 185 | var yaml: Yaml = .{ .source = source }; 186 | defer yaml.deinit(testing.allocator); 187 | const err = yaml.load(testing.allocator); 188 | 189 | try std.testing.expectError(error.ParseFailure, err); 190 | } 191 | 192 | test "simple flow sequence / bracket list with double trailing commas" { 193 | const source = 194 | \\a_key: [a, b, c,,] 195 | ; 196 | 197 | var yaml: Yaml = .{ .source = source }; 198 | defer yaml.deinit(testing.allocator); 199 | const err = yaml.load(testing.allocator); 200 | 201 | try std.testing.expectError(error.ParseFailure, err); 202 | } 203 | 204 | test "more bools" { 205 | const source = 206 | \\- false 207 | \\- true 208 | \\- off 209 | \\- on 210 | \\- no 211 | \\- yes 212 | \\- n 213 | \\- y 214 | ; 215 | 216 | var yaml: Yaml = .{ .source = source }; 217 | defer yaml.deinit(testing.allocator); 218 | try yaml.load(testing.allocator); 219 | 220 | try testing.expectEqual(yaml.docs.items.len, 1); 221 | 222 | var arena = Arena.init(testing.allocator); 223 | defer arena.deinit(); 224 | 225 | const arr = try yaml.parse(arena.allocator(), [8]bool); 226 | try testing.expectEqualSlices(bool, &[_]bool{ 227 | false, 228 | true, 229 | false, 230 | true, 231 | false, 232 | true, 233 | false, 234 | true, 235 | }, &arr); 236 | } 237 | 238 | test "invalid enum" { 239 | const TestEnum = enum { 240 | alpha, 241 | bravo, 242 | charlie, 243 | }; 244 | 245 | const source = 246 | \\- delta 247 | \\- echo 248 | ; 249 | 250 | var yaml: Yaml = .{ .source = source }; 251 | defer yaml.deinit(testing.allocator); 252 | try yaml.load(testing.allocator); 253 | 254 | try testing.expectEqual(yaml.docs.items.len, 1); 255 | 256 | var arena = Arena.init(testing.allocator); 257 | defer arena.deinit(); 258 | 259 | const result = yaml.parse(arena.allocator(), [2]TestEnum); 260 | try testing.expectError(Yaml.Error.InvalidEnum, result); 261 | } 262 | 263 | test "simple map untyped" { 264 | const source = 265 | \\a: 0 266 | ; 267 | 268 | var yaml: Yaml = .{ .source = source }; 269 | defer yaml.deinit(testing.allocator); 270 | try yaml.load(testing.allocator); 271 | 272 | try testing.expectEqual(yaml.docs.items.len, 1); 273 | 274 | const map = yaml.docs.items[0].map; 275 | try testing.expect(map.contains("a")); 276 | try testing.expectEqualStrings("0", map.get("a").?.scalar); 277 | } 278 | 279 | test "simple map untyped with a list of maps" { 280 | const source = 281 | \\a: 0 282 | \\b: 283 | \\ - foo: 1 284 | \\ bar: 2 285 | \\ - foo: 3 286 | \\ bar: 4 287 | \\c: 1 288 | ; 289 | 290 | var yaml: Yaml = .{ .source = source }; 291 | defer yaml.deinit(testing.allocator); 292 | try yaml.load(testing.allocator); 293 | 294 | try testing.expectEqual(yaml.docs.items.len, 1); 295 | 296 | const map = yaml.docs.items[0].map; 297 | try testing.expect(map.contains("a")); 298 | try testing.expect(map.contains("b")); 299 | try testing.expect(map.contains("c")); 300 | try testing.expectEqualStrings("0", map.get("a").?.scalar); 301 | try testing.expectEqualStrings("1", map.get("c").?.scalar); 302 | try testing.expectEqualStrings("1", map.get("b").?.list[0].map.get("foo").?.scalar); 303 | try testing.expectEqualStrings("2", map.get("b").?.list[0].map.get("bar").?.scalar); 304 | try testing.expectEqualStrings("3", map.get("b").?.list[1].map.get("foo").?.scalar); 305 | try testing.expectEqualStrings("4", map.get("b").?.list[1].map.get("bar").?.scalar); 306 | } 307 | 308 | test "simple map untyped with a list of maps. no indent" { 309 | const source = 310 | \\b: 311 | \\- foo: 1 312 | \\c: 1 313 | ; 314 | 315 | var yaml: Yaml = .{ .source = source }; 316 | defer yaml.deinit(testing.allocator); 317 | try yaml.load(testing.allocator); 318 | 319 | try testing.expectEqual(yaml.docs.items.len, 1); 320 | 321 | const map = yaml.docs.items[0].map; 322 | try testing.expect(map.contains("b")); 323 | try testing.expect(map.contains("c")); 324 | try testing.expectEqualStrings("1", map.get("c").?.scalar); 325 | try testing.expectEqualStrings("1", map.get("b").?.list[0].map.get("foo").?.scalar); 326 | } 327 | 328 | test "simple map untyped with a list of maps. no indent 2" { 329 | const source = 330 | \\a: 0 331 | \\b: 332 | \\- foo: 1 333 | \\ bar: 2 334 | \\- foo: 3 335 | \\ bar: 4 336 | \\c: 1 337 | ; 338 | 339 | var yaml: Yaml = .{ .source = source }; 340 | defer yaml.deinit(testing.allocator); 341 | try yaml.load(testing.allocator); 342 | 343 | try testing.expectEqual(yaml.docs.items.len, 1); 344 | 345 | const map = yaml.docs.items[0].map; 346 | try testing.expect(map.contains("a")); 347 | try testing.expect(map.contains("b")); 348 | try testing.expect(map.contains("c")); 349 | try testing.expectEqualStrings("0", map.get("a").?.scalar); 350 | try testing.expectEqualStrings("1", map.get("c").?.scalar); 351 | try testing.expectEqualStrings("1", map.get("b").?.list[0].map.get("foo").?.scalar); 352 | try testing.expectEqualStrings("2", map.get("b").?.list[0].map.get("bar").?.scalar); 353 | try testing.expectEqualStrings("3", map.get("b").?.list[1].map.get("foo").?.scalar); 354 | try testing.expectEqualStrings("4", map.get("b").?.list[1].map.get("bar").?.scalar); 355 | } 356 | 357 | test "simple map typed" { 358 | const source = 359 | \\a: 0 360 | \\b: hello there 361 | \\c: 'wait, what?' 362 | ; 363 | 364 | var yaml: Yaml = .{ .source = source }; 365 | defer yaml.deinit(testing.allocator); 366 | try yaml.load(testing.allocator); 367 | 368 | var arena = Arena.init(testing.allocator); 369 | defer arena.deinit(); 370 | 371 | const simple = try yaml.parse(arena.allocator(), struct { a: usize, b: []const u8, c: []const u8 }); 372 | try testing.expectEqual(@as(usize, 0), simple.a); 373 | try testing.expectEqualStrings("hello there", simple.b); 374 | try testing.expectEqualStrings("wait, what?", simple.c); 375 | } 376 | 377 | test "typed nested structs" { 378 | const source = 379 | \\a: 380 | \\ b: hello there 381 | \\ c: 'wait, what?' 382 | ; 383 | 384 | var yaml: Yaml = .{ .source = source }; 385 | defer yaml.deinit(testing.allocator); 386 | try yaml.load(testing.allocator); 387 | 388 | var arena = Arena.init(testing.allocator); 389 | defer arena.deinit(); 390 | 391 | const simple = try yaml.parse(arena.allocator(), struct { 392 | a: struct { 393 | b: []const u8, 394 | c: []const u8, 395 | }, 396 | }); 397 | try testing.expectEqualStrings("hello there", simple.a.b); 398 | try testing.expectEqualStrings("wait, what?", simple.a.c); 399 | } 400 | 401 | test "typed union with nested struct" { 402 | const source = 403 | \\a: 404 | \\ b: hello there 405 | ; 406 | 407 | var yaml: Yaml = .{ .source = source }; 408 | defer yaml.deinit(testing.allocator); 409 | try yaml.load(testing.allocator); 410 | 411 | var arena = Arena.init(testing.allocator); 412 | defer arena.deinit(); 413 | 414 | const simple = try yaml.parse(arena.allocator(), union(enum) { 415 | tag_a: struct { 416 | a: struct { 417 | b: []const u8, 418 | }, 419 | }, 420 | tag_c: struct { 421 | c: struct { 422 | d: []const u8, 423 | }, 424 | }, 425 | }); 426 | try testing.expectEqualStrings("hello there", simple.tag_a.a.b); 427 | } 428 | 429 | test "typed union with nested struct 2" { 430 | const source = 431 | \\c: 432 | \\ d: hello there 433 | ; 434 | 435 | var yaml: Yaml = .{ .source = source }; 436 | defer yaml.deinit(testing.allocator); 437 | try yaml.load(testing.allocator); 438 | 439 | var arena = Arena.init(testing.allocator); 440 | defer arena.deinit(); 441 | 442 | const simple = try yaml.parse(arena.allocator(), union(enum) { 443 | tag_a: struct { 444 | a: struct { 445 | b: []const u8, 446 | }, 447 | }, 448 | tag_c: struct { 449 | c: struct { 450 | d: []const u8, 451 | }, 452 | }, 453 | }); 454 | try testing.expectEqualStrings("hello there", simple.tag_c.c.d); 455 | } 456 | 457 | test "single quoted string" { 458 | const source = 459 | \\- 'hello' 460 | \\- 'here''s an escaped quote' 461 | \\- 'newlines and tabs\nare not\tsupported' 462 | ; 463 | 464 | var yaml: Yaml = .{ .source = source }; 465 | defer yaml.deinit(testing.allocator); 466 | try yaml.load(testing.allocator); 467 | 468 | var arena = Arena.init(testing.allocator); 469 | defer arena.deinit(); 470 | 471 | const arr = try yaml.parse(arena.allocator(), [3][]const u8); 472 | try testing.expectEqual(arr.len, 3); 473 | try testing.expectEqualStrings("hello", arr[0]); 474 | try testing.expectEqualStrings("here's an escaped quote", arr[1]); 475 | try testing.expectEqualStrings("newlines and tabs\\nare not\\tsupported", arr[2]); 476 | } 477 | 478 | test "double quoted string" { 479 | const source = 480 | \\- "hello" 481 | \\- "\"here\" are some escaped quotes" 482 | \\- "newlines and tabs\nare\tsupported" 483 | \\- "let's have 484 | \\some fun!" 485 | ; 486 | 487 | var yaml: Yaml = .{ .source = source }; 488 | defer yaml.deinit(testing.allocator); 489 | try yaml.load(testing.allocator); 490 | 491 | var arena = Arena.init(testing.allocator); 492 | defer arena.deinit(); 493 | 494 | const arr = try yaml.parse(arena.allocator(), [4][]const u8); 495 | try testing.expectEqual(arr.len, 4); 496 | try testing.expectEqualStrings("hello", arr[0]); 497 | try testing.expectEqualStrings( 498 | \\"here" are some escaped quotes 499 | , arr[1]); 500 | try testing.expectEqualStrings("newlines and tabs\nare\tsupported", arr[2]); 501 | try testing.expectEqualStrings( 502 | \\let's have 503 | \\some fun! 504 | , arr[3]); 505 | } 506 | 507 | test "commas in string" { 508 | const source = 509 | \\a: 900,50,50 510 | ; 511 | 512 | var yaml: Yaml = .{ .source = source }; 513 | defer yaml.deinit(testing.allocator); 514 | try yaml.load(testing.allocator); 515 | 516 | var arena = Arena.init(testing.allocator); 517 | defer arena.deinit(); 518 | 519 | const simple = try yaml.parse(arena.allocator(), struct { 520 | a: []const u8, 521 | }); 522 | try testing.expectEqualStrings("900,50,50", simple.a); 523 | } 524 | 525 | test "multidoc typed as a slice of structs" { 526 | const source = 527 | \\--- 528 | \\a: 0 529 | \\--- 530 | \\a: 1 531 | \\... 532 | ; 533 | 534 | var yaml: Yaml = .{ .source = source }; 535 | defer yaml.deinit(testing.allocator); 536 | try yaml.load(testing.allocator); 537 | 538 | var arena = Arena.init(testing.allocator); 539 | defer arena.deinit(); 540 | 541 | { 542 | const result = try yaml.parse(arena.allocator(), [2]struct { a: usize }); 543 | try testing.expectEqual(result.len, 2); 544 | try testing.expectEqual(result[0].a, 0); 545 | try testing.expectEqual(result[1].a, 1); 546 | } 547 | 548 | { 549 | const result = try yaml.parse(arena.allocator(), []struct { a: usize }); 550 | try testing.expectEqual(result.len, 2); 551 | try testing.expectEqual(result[0].a, 0); 552 | try testing.expectEqual(result[1].a, 1); 553 | } 554 | } 555 | 556 | test "multidoc typed as a struct is an error" { 557 | const source = 558 | \\--- 559 | \\a: 0 560 | \\--- 561 | \\b: 1 562 | \\... 563 | ; 564 | 565 | var yaml: Yaml = .{ .source = source }; 566 | defer yaml.deinit(testing.allocator); 567 | try yaml.load(testing.allocator); 568 | 569 | var arena = Arena.init(testing.allocator); 570 | defer arena.deinit(); 571 | 572 | try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(arena.allocator(), struct { a: usize })); 573 | try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(arena.allocator(), struct { b: usize })); 574 | try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(arena.allocator(), struct { a: usize, b: usize })); 575 | } 576 | 577 | test "multidoc typed as a slice of structs with optionals" { 578 | const source = 579 | \\--- 580 | \\a: 0 581 | \\c: 1.0 582 | \\--- 583 | \\a: 1 584 | \\b: different field 585 | \\... 586 | ; 587 | 588 | var yaml: Yaml = .{ .source = source }; 589 | defer yaml.deinit(testing.allocator); 590 | try yaml.load(testing.allocator); 591 | 592 | var arena = Arena.init(testing.allocator); 593 | defer arena.deinit(); 594 | 595 | const result = try yaml.parse(arena.allocator(), []struct { a: usize, b: ?[]const u8, c: ?f16 }); 596 | try testing.expectEqual(result.len, 2); 597 | 598 | try testing.expectEqual(result[0].a, 0); 599 | try testing.expect(result[0].b == null); 600 | try testing.expect(result[0].c != null); 601 | try testing.expectEqual(result[0].c.?, 1.0); 602 | 603 | try testing.expectEqual(result[1].a, 1); 604 | try testing.expect(result[1].b != null); 605 | try testing.expectEqualStrings("different field", result[1].b.?); 606 | try testing.expect(result[1].c == null); 607 | } 608 | 609 | test "empty yaml can be represented as void" { 610 | const source = ""; 611 | 612 | var yaml: Yaml = .{ .source = source }; 613 | defer yaml.deinit(testing.allocator); 614 | try yaml.load(testing.allocator); 615 | 616 | var arena = Arena.init(testing.allocator); 617 | defer arena.deinit(); 618 | 619 | const result = try yaml.parse(arena.allocator(), void); 620 | try testing.expect(@TypeOf(result) == void); 621 | } 622 | 623 | test "nonempty yaml cannot be represented as void" { 624 | const source = 625 | \\a: b 626 | ; 627 | 628 | var yaml: Yaml = .{ .source = source }; 629 | defer yaml.deinit(testing.allocator); 630 | try yaml.load(testing.allocator); 631 | 632 | var arena = Arena.init(testing.allocator); 633 | defer arena.deinit(); 634 | 635 | try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(arena.allocator(), void)); 636 | } 637 | 638 | test "typed array size mismatch" { 639 | const source = 640 | \\- 0 641 | \\- 0 642 | ; 643 | 644 | var yaml: Yaml = .{ .source = source }; 645 | defer yaml.deinit(testing.allocator); 646 | try yaml.load(testing.allocator); 647 | 648 | var arena = Arena.init(testing.allocator); 649 | defer arena.deinit(); 650 | 651 | try testing.expectError(Yaml.Error.ArraySizeMismatch, yaml.parse(arena.allocator(), [1]usize)); 652 | try testing.expectError(Yaml.Error.ArraySizeMismatch, yaml.parse(arena.allocator(), [5]usize)); 653 | } 654 | 655 | test "comments" { 656 | const source = 657 | \\ 658 | \\key: # this is the key 659 | \\# first value 660 | \\ 661 | \\- val1 662 | \\ 663 | \\# second value 664 | \\- val2 665 | ; 666 | 667 | var yaml: Yaml = .{ .source = source }; 668 | defer yaml.deinit(testing.allocator); 669 | try yaml.load(testing.allocator); 670 | 671 | var arena = Arena.init(testing.allocator); 672 | defer arena.deinit(); 673 | 674 | const simple = try yaml.parse(arena.allocator(), struct { 675 | key: []const []const u8, 676 | }); 677 | try testing.expect(simple.key.len == 2); 678 | try testing.expectEqualStrings("val1", simple.key[0]); 679 | try testing.expectEqualStrings("val2", simple.key[1]); 680 | } 681 | 682 | test "promote ints to floats in a list mixed numeric types" { 683 | const source = 684 | \\a_list: [0, 1.0] 685 | ; 686 | 687 | var yaml: Yaml = .{ .source = source }; 688 | defer yaml.deinit(testing.allocator); 689 | try yaml.load(testing.allocator); 690 | 691 | var arena = Arena.init(testing.allocator); 692 | defer arena.deinit(); 693 | 694 | const simple = try yaml.parse(arena.allocator(), struct { 695 | a_list: []const f64, 696 | }); 697 | try testing.expectEqualSlices(f64, &[_]f64{ 0.0, 1.0 }, simple.a_list); 698 | } 699 | 700 | test "demoting floats to ints in a list is an error" { 701 | const source = 702 | \\a_list: [0, 1.0] 703 | ; 704 | 705 | var yaml: Yaml = .{ .source = source }; 706 | defer yaml.deinit(testing.allocator); 707 | try yaml.load(testing.allocator); 708 | 709 | var arena = Arena.init(testing.allocator); 710 | defer arena.deinit(); 711 | 712 | try testing.expectError(error.InvalidCharacter, yaml.parse(arena.allocator(), struct { 713 | a_list: []const u64, 714 | })); 715 | } 716 | 717 | test "duplicate map keys" { 718 | const source = 719 | \\a: b 720 | \\a: c 721 | ; 722 | var yaml: Yaml = .{ .source = source }; 723 | defer yaml.deinit(testing.allocator); 724 | try testing.expectError(error.DuplicateMapKey, yaml.load(testing.allocator)); 725 | } 726 | 727 | fn testStringify(expected: []const u8, input: anytype) !void { 728 | var writer: std.Io.Writer.Allocating = .init(testing.allocator); 729 | defer writer.deinit(); 730 | 731 | try stringify(testing.allocator, input, &writer.writer); 732 | try testing.expectEqualStrings(expected, writer.written()); 733 | } 734 | 735 | test "stringify an int" { 736 | try testStringify("128", @as(u32, 128)); 737 | } 738 | 739 | test "stringify a simple struct" { 740 | try testStringify( 741 | \\a: 1 742 | \\b: 2 743 | \\c: 2.5 744 | , struct { a: i64, b: f64, c: f64 }{ .a = 1, .b = 2.0, .c = 2.5 }); 745 | } 746 | 747 | test "stringify a struct with an optional" { 748 | try testStringify( 749 | \\a: 1 750 | \\b: 2 751 | \\c: 2.5 752 | , struct { a: i64, b: ?f64, c: f64 }{ .a = 1, .b = 2.0, .c = 2.5 }); 753 | 754 | try testStringify( 755 | \\a: 1 756 | \\c: 2.5 757 | , struct { a: i64, b: ?f64, c: f64 }{ .a = 1, .b = null, .c = 2.5 }); 758 | } 759 | 760 | test "stringify a struct with all optionals" { 761 | try testStringify("", struct { a: ?i64, b: ?f64 }{ .a = null, .b = null }); 762 | } 763 | 764 | test "stringify an optional" { 765 | try testStringify("", null); 766 | try testStringify("", @as(?u64, null)); 767 | } 768 | 769 | test "stringify a union" { 770 | const Dummy = union(enum) { 771 | x: u64, 772 | y: f64, 773 | }; 774 | try testStringify("a: 1", struct { a: Dummy }{ .a = .{ .x = 1 } }); 775 | try testStringify("a: 2.1", struct { a: Dummy }{ .a = .{ .y = 2.1 } }); 776 | } 777 | 778 | test "stringify a string" { 779 | try testStringify("a: name", struct { a: []const u8 }{ .a = "name" }); 780 | try testStringify("name", "name"); 781 | } 782 | 783 | test "stringify a list" { 784 | try testStringify("[ 1, 2, 3 ]", @as([]const u64, &.{ 1, 2, 3 })); 785 | try testStringify("[ 1, 2, 3 ]", .{ @as(i64, 1), 2, 3 }); 786 | try testStringify("[ 1, name, 3 ]", .{ 1, "name", 3 }); 787 | 788 | const arr: [3]i64 = .{ 1, 2, 3 }; 789 | try testStringify("[ 1, 2, 3 ]", arr); 790 | } 791 | 792 | test "pointer of a value" { 793 | const TestStruct = struct { 794 | a: usize, 795 | b: i64, 796 | c: u12, 797 | d: ?*const @This() = null, 798 | }; 799 | 800 | const source = 801 | \\a: 1 802 | \\b: 2 803 | \\c: 3 804 | \\d: 805 | \\ a: 4 806 | \\ b: 5 807 | \\ c: 6 808 | ; 809 | 810 | var arena = std.heap.ArenaAllocator.init(testing.allocator); 811 | defer arena.deinit(); 812 | 813 | var yaml = Yaml{ .source = source }; 814 | try yaml.load(arena.allocator()); 815 | 816 | const parsed = try yaml.parse(arena.allocator(), *TestStruct); 817 | try testing.expectEqual(1, parsed.a); 818 | try testing.expectEqual(2, parsed.b); 819 | try testing.expectEqual(3, parsed.c); 820 | try testing.expectEqual(4, parsed.d.?.a); 821 | try testing.expectEqual(5, parsed.d.?.b); 822 | try testing.expectEqual(6, parsed.d.?.c); 823 | try testing.expectEqual(@as(?*const TestStruct, null), parsed.d.?.d); 824 | } 825 | 826 | test "struct default value test" { 827 | const TestStruct = struct { 828 | a: i32, 829 | b: ?[]const u8 = "test", 830 | c: ?u8 = 5, 831 | d: u8 = 12, 832 | }; 833 | 834 | const TestCase = struct { 835 | yaml: []const u8, 836 | container: TestStruct, 837 | }; 838 | 839 | const tcs = [_]TestCase{ 840 | .{ 841 | .yaml = 842 | \\--- 843 | \\a: 1 844 | \\b: "asd" 845 | \\c: 3 846 | \\d: 1 847 | \\... 848 | , 849 | .container = .{ 850 | .a = 1, 851 | .b = "asd", 852 | .c = 3, 853 | .d = 1, 854 | }, 855 | }, 856 | .{ 857 | .yaml = 858 | \\--- 859 | \\a: 1 860 | \\c: 3 861 | \\d: 1 862 | \\... 863 | , 864 | .container = .{ 865 | .a = 1, 866 | .b = "test", 867 | .c = 3, 868 | .d = 1, 869 | }, 870 | }, 871 | .{ 872 | .yaml = 873 | \\--- 874 | \\a: 1 875 | \\b: "asd" 876 | \\d: 1 877 | \\... 878 | , 879 | .container = .{ 880 | .a = 1, 881 | .b = "asd", 882 | .c = 5, 883 | .d = 1, 884 | }, 885 | }, 886 | .{ 887 | .yaml = 888 | \\--- 889 | \\a: 1 890 | \\b: "asd" 891 | \\... 892 | , 893 | .container = .{ 894 | .a = 1, 895 | .b = "asd", 896 | .c = 5, 897 | .d = 12, 898 | }, 899 | }, 900 | }; 901 | 902 | for (&tcs) |tc| { 903 | var arena = std.heap.ArenaAllocator.init(testing.allocator); 904 | defer arena.deinit(); 905 | var yamlParser = Yaml{ .source = tc.yaml }; 906 | try yamlParser.load(arena.allocator()); 907 | const parsed = try yamlParser.parse(arena.allocator(), TestStruct); 908 | try testing.expectEqual(tc.container.a, parsed.a); 909 | try testing.expectEqualDeep(tc.container.b, parsed.b); 910 | try testing.expectEqual(tc.container.c, parsed.c); 911 | try testing.expectEqual(tc.container.d, parsed.d); 912 | } 913 | } 914 | 915 | test "enums" { 916 | const source = 917 | \\- a 918 | \\- b 919 | \\- c 920 | ; 921 | 922 | const Enum = enum { a, b, c }; 923 | 924 | var yaml: Yaml = .{ .source = source }; 925 | defer yaml.deinit(testing.allocator); 926 | try yaml.load(testing.allocator); 927 | 928 | var arena = Arena.init(testing.allocator); 929 | defer arena.deinit(); 930 | 931 | const parsed = try yaml.parse(arena.allocator(), []const Enum); 932 | try testing.expectEqualDeep(&[_]Enum{ 933 | .a, 934 | .b, 935 | .c, 936 | }, parsed); 937 | } 938 | 939 | test "stringify a bool" { 940 | try testStringify("false", false); 941 | try testStringify("true", true); 942 | } 943 | 944 | test "stringify an enum" { 945 | const TestEnum = enum { 946 | alpha, 947 | bravo, 948 | charlie, 949 | }; 950 | 951 | try testStringify("alpha", TestEnum.alpha); 952 | try testStringify("bravo", TestEnum.bravo); 953 | try testStringify("charlie", TestEnum.charlie); 954 | } 955 | 956 | test "parse struct as list of structs" { 957 | const source = 958 | \\a: 1 959 | ; 960 | 961 | const Struct = struct { a: u32 }; 962 | 963 | var yaml: Yaml = .{ .source = source }; 964 | defer yaml.deinit(testing.allocator); 965 | try yaml.load(testing.allocator); 966 | 967 | var arena = Arena.init(testing.allocator); 968 | defer arena.deinit(); 969 | 970 | const result = yaml.parse(arena.allocator(), []Struct); 971 | try testing.expectError(error.TypeMismatch, result); 972 | 973 | const parsed = try yaml.parse(arena.allocator(), Struct); 974 | try testing.expectEqualDeep(Struct{ .a = 1 }, parsed); 975 | } 976 | -------------------------------------------------------------------------------- /src/Parser/test.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const mem = std.mem; 3 | const testing = std.testing; 4 | 5 | const List = Tree.List; 6 | const Map = Tree.Map; 7 | const Node = Tree.Node; 8 | const Parser = @import("../Parser.zig"); 9 | const Tree = @import("../Tree.zig"); 10 | 11 | fn expectNodeScope(tree: Tree, node: Node.Index, from: usize, to: usize) !void { 12 | const scope = tree.nodeScope(node); 13 | try testing.expectEqual(from, @intFromEnum(scope.start)); 14 | try testing.expectEqual(to, @intFromEnum(scope.end)); 15 | } 16 | 17 | fn expectValueMapEntry(tree: Tree, entry_data: Map.Entry, exp_key: []const u8, exp_value: []const u8) !void { 18 | const key = tree.token(entry_data.key); 19 | try testing.expectEqual(key.id, .literal); 20 | try testing.expectEqualStrings(exp_key, tree.rawString(entry_data.key, entry_data.key)); 21 | 22 | const maybe_value = entry_data.maybe_node; 23 | try testing.expect(maybe_value != .none); 24 | try testing.expectEqual(.value, tree.nodeTag(maybe_value.unwrap().?)); 25 | 26 | const value = maybe_value.unwrap().?; 27 | const string = tree.nodeScope(value).rawString(tree); 28 | try testing.expectEqualStrings(exp_value, string); 29 | } 30 | 31 | fn expectStringValueMapEntry(tree: Tree, entry_data: Map.Entry, exp_key: []const u8, exp_value: []const u8) !void { 32 | const key = tree.token(entry_data.key); 33 | try testing.expectEqual(key.id, .literal); 34 | try testing.expectEqualStrings(exp_key, tree.rawString(entry_data.key, entry_data.key)); 35 | 36 | const maybe_value = entry_data.maybe_node; 37 | try testing.expect(maybe_value != .none); 38 | try testing.expectEqual(.string_value, tree.nodeTag(maybe_value.unwrap().?)); 39 | 40 | const value = maybe_value.unwrap().?; 41 | const string = tree.nodeData(value).string.slice(tree); 42 | try testing.expectEqualStrings(exp_value, string); 43 | } 44 | 45 | fn expectValueListEntry(tree: Tree, entry_data: List.Entry, exp_value: []const u8) !void { 46 | const value = entry_data.node; 47 | try testing.expectEqual(.value, tree.nodeTag(value)); 48 | 49 | const string = tree.nodeScope(value).rawString(tree); 50 | try testing.expectEqualStrings(exp_value, string); 51 | } 52 | 53 | fn expectNestedMapListEntry(tree: Tree, list_entry_data: List.Entry, exp_key: []const u8, exp_value: []const u8) !void { 54 | const value = list_entry_data.node; 55 | try testing.expectEqual(.map_single, tree.nodeTag(value)); 56 | 57 | const map_data = tree.nodeData(value).map; 58 | try expectValueMapEntry(tree, .{ 59 | .key = map_data.key, 60 | .maybe_node = map_data.maybe_node, 61 | }, exp_key, exp_value); 62 | } 63 | 64 | test "explicit doc" { 65 | const source = 66 | \\--- !tapi-tbd 67 | \\tbd-version: 4 68 | \\abc-version: 5 69 | \\... 70 | ; 71 | 72 | var parser = try Parser.init(testing.allocator, source); 73 | defer parser.deinit(testing.allocator); 74 | try parser.parse(testing.allocator); 75 | 76 | var tree = try parser.toOwnedTree(testing.allocator); 77 | defer tree.deinit(testing.allocator); 78 | 79 | try testing.expectEqual(1, tree.docs.len); 80 | 81 | const doc = tree.docs[0]; 82 | try testing.expectEqual(.doc_with_directive, tree.nodeTag(doc)); 83 | 84 | try expectNodeScope(tree, doc, 0, tree.tokens.len - 2); 85 | 86 | const directive = tree.directive(doc).?; 87 | try testing.expectEqualStrings("tapi-tbd", directive); 88 | 89 | const doc_value = tree.nodeData(doc).doc_with_directive.maybe_node; 90 | try testing.expect(doc_value != .none); 91 | try testing.expectEqual(.map_many, tree.nodeTag(doc_value.unwrap().?)); 92 | 93 | const map = doc_value.unwrap().?; 94 | 95 | try expectNodeScope(tree, map, 5, 14); 96 | 97 | const map_data = tree.extraData(Map, tree.nodeData(map).extra); 98 | try testing.expectEqual(2, map_data.data.map_len); 99 | 100 | var entry_data = tree.extraData(Map.Entry, map_data.end); 101 | try expectValueMapEntry(tree, entry_data.data, "tbd-version", "4"); 102 | 103 | entry_data = tree.extraData(Map.Entry, entry_data.end); 104 | try expectValueMapEntry(tree, entry_data.data, "abc-version", "5"); 105 | } 106 | 107 | test "leaf in quotes" { 108 | const source = 109 | \\key1: no quotes, comma 110 | \\key2: 'single quoted' 111 | \\key3: "double quoted" 112 | ; 113 | 114 | var parser = try Parser.init(testing.allocator, source); 115 | defer parser.deinit(testing.allocator); 116 | try parser.parse(testing.allocator); 117 | 118 | var tree = try parser.toOwnedTree(testing.allocator); 119 | defer tree.deinit(testing.allocator); 120 | 121 | try testing.expectEqual(1, tree.docs.len); 122 | 123 | const doc = tree.docs[0]; 124 | try testing.expectEqual(.doc, tree.nodeTag(doc)); 125 | 126 | try expectNodeScope(tree, doc, 0, tree.tokens.len - 2); 127 | 128 | const doc_value = tree.nodeData(doc).maybe_node; 129 | try testing.expect(doc_value != .none); 130 | try testing.expectEqual(.map_many, tree.nodeTag(doc_value.unwrap().?)); 131 | 132 | const map = doc_value.unwrap().?; 133 | 134 | try expectNodeScope(tree, map, 0, tree.tokens.len - 2); 135 | 136 | const map_data = tree.extraData(Map, tree.nodeData(map).extra); 137 | try testing.expectEqual(3, map_data.data.map_len); 138 | 139 | var entry_data = tree.extraData(Map.Entry, map_data.end); 140 | try expectValueMapEntry(tree, entry_data.data, "key1", "no quotes, comma"); 141 | 142 | entry_data = tree.extraData(Map.Entry, entry_data.end); 143 | try expectStringValueMapEntry(tree, entry_data.data, "key2", "single quoted"); 144 | 145 | entry_data = tree.extraData(Map.Entry, entry_data.end); 146 | try expectStringValueMapEntry(tree, entry_data.data, "key3", "double quoted"); 147 | } 148 | 149 | test "nested maps" { 150 | const source = 151 | \\key1: 152 | \\ key1_1 : value1_1 153 | \\ key1_2 : value1_2 154 | \\key2 : value2 155 | ; 156 | 157 | var parser = try Parser.init(testing.allocator, source); 158 | defer parser.deinit(testing.allocator); 159 | try parser.parse(testing.allocator); 160 | 161 | var tree = try parser.toOwnedTree(testing.allocator); 162 | defer tree.deinit(testing.allocator); 163 | 164 | try testing.expectEqual(1, tree.docs.len); 165 | 166 | const doc = tree.docs[0]; 167 | try testing.expectEqual(.doc, tree.nodeTag(doc)); 168 | 169 | try expectNodeScope(tree, doc, 0, tree.tokens.len - 2); 170 | 171 | const doc_value = tree.nodeData(doc).maybe_node; 172 | try testing.expect(doc_value != .none); 173 | try testing.expectEqual(.map_many, tree.nodeTag(doc_value.unwrap().?)); 174 | 175 | const map = doc_value.unwrap().?; 176 | 177 | try expectNodeScope(tree, map, 0, tree.tokens.len - 2); 178 | 179 | const map_data = tree.extraData(Map, tree.nodeData(map).extra); 180 | try testing.expectEqual(2, map_data.data.map_len); 181 | 182 | var entry_data = tree.extraData(Map.Entry, map_data.end); 183 | { 184 | const key = tree.token(entry_data.data.key); 185 | try testing.expectEqual(key.id, .literal); 186 | try testing.expectEqualStrings("key1", tree.rawString(entry_data.data.key, entry_data.data.key)); 187 | 188 | const maybe_nested_map = entry_data.data.maybe_node; 189 | try testing.expect(maybe_nested_map != .none); 190 | try testing.expectEqual(.map_many, tree.nodeTag(maybe_nested_map.unwrap().?)); 191 | 192 | const nested_map = maybe_nested_map.unwrap().?; 193 | 194 | try expectNodeScope(tree, nested_map, 4, 16); 195 | 196 | const nested_map_data = tree.extraData(Map, tree.nodeData(nested_map).extra); 197 | try testing.expectEqual(2, nested_map_data.data.map_len); 198 | 199 | var nested_entry_data = tree.extraData(Map.Entry, nested_map_data.end); 200 | try expectValueMapEntry(tree, nested_entry_data.data, "key1_1", "value1_1"); 201 | 202 | nested_entry_data = tree.extraData(Map.Entry, nested_entry_data.end); 203 | try expectValueMapEntry(tree, nested_entry_data.data, "key1_2", "value1_2"); 204 | } 205 | 206 | entry_data = tree.extraData(Map.Entry, entry_data.end); 207 | try expectValueMapEntry(tree, entry_data.data, "key2", "value2"); 208 | } 209 | 210 | test "map of list of values" { 211 | const source = 212 | \\ints: 213 | \\ - 0 214 | \\ - 1 215 | \\ - 2 216 | ; 217 | 218 | var parser = try Parser.init(testing.allocator, source); 219 | defer parser.deinit(testing.allocator); 220 | try parser.parse(testing.allocator); 221 | 222 | var tree = try parser.toOwnedTree(testing.allocator); 223 | defer tree.deinit(testing.allocator); 224 | 225 | try testing.expectEqual(1, tree.docs.len); 226 | 227 | const doc = tree.docs[0]; 228 | try expectNodeScope(tree, doc, 0, tree.tokens.len - 2); 229 | 230 | const doc_value = tree.nodeData(doc).maybe_node; 231 | try testing.expect(doc_value != .none); 232 | try testing.expectEqual(.map_single, tree.nodeTag(doc_value.unwrap().?)); 233 | 234 | const map = doc_value.unwrap().?; 235 | 236 | try expectNodeScope(tree, map, 0, tree.tokens.len - 2); 237 | 238 | const map_data = tree.nodeData(map).map; 239 | 240 | { 241 | const key = tree.token(map_data.key); 242 | try testing.expectEqual(key.id, .literal); 243 | try testing.expectEqualStrings("ints", tree.rawString(map_data.key, map_data.key)); 244 | 245 | const maybe_nested_list = map_data.maybe_node; 246 | try testing.expect(maybe_nested_list != .none); 247 | try testing.expectEqual(.list_many, tree.nodeTag(maybe_nested_list.unwrap().?)); 248 | 249 | const nested_list = maybe_nested_list.unwrap().?; 250 | 251 | try expectNodeScope(tree, nested_list, 4, tree.tokens.len - 2); 252 | 253 | const nested_list_data = tree.extraData(List, tree.nodeData(nested_list).extra); 254 | try testing.expectEqual(3, nested_list_data.data.list_len); 255 | 256 | var nested_entry_data = tree.extraData(List.Entry, nested_list_data.end); 257 | try expectValueListEntry(tree, nested_entry_data.data, "0"); 258 | 259 | nested_entry_data = tree.extraData(List.Entry, nested_entry_data.end); 260 | try expectValueListEntry(tree, nested_entry_data.data, "1"); 261 | 262 | nested_entry_data = tree.extraData(List.Entry, nested_entry_data.end); 263 | try expectValueListEntry(tree, nested_entry_data.data, "2"); 264 | } 265 | } 266 | 267 | test "map of list of maps" { 268 | const source = 269 | \\key1: 270 | \\- key2 : value2 271 | \\- key3 : value3 272 | \\- key4 : value4 273 | ; 274 | 275 | var parser = try Parser.init(testing.allocator, source); 276 | defer parser.deinit(testing.allocator); 277 | try parser.parse(testing.allocator); 278 | 279 | var tree = try parser.toOwnedTree(testing.allocator); 280 | defer tree.deinit(testing.allocator); 281 | 282 | try testing.expectEqual(1, tree.docs.len); 283 | 284 | const doc = tree.docs[0]; 285 | try expectNodeScope(tree, doc, 0, tree.tokens.len - 2); 286 | 287 | const doc_value = tree.nodeData(doc).maybe_node; 288 | try testing.expect(doc_value != .none); 289 | try testing.expectEqual(.map_single, tree.nodeTag(doc_value.unwrap().?)); 290 | 291 | const map = doc_value.unwrap().?; 292 | 293 | try expectNodeScope(tree, map, 0, tree.tokens.len - 2); 294 | 295 | const map_data = tree.nodeData(map).map; 296 | 297 | { 298 | const key = tree.token(map_data.key); 299 | try testing.expectEqual(key.id, .literal); 300 | try testing.expectEqualStrings("key1", tree.rawString(map_data.key, map_data.key)); 301 | 302 | const maybe_nested_list = map_data.maybe_node; 303 | try testing.expect(maybe_nested_list != .none); 304 | try testing.expectEqual(.list_many, tree.nodeTag(maybe_nested_list.unwrap().?)); 305 | 306 | const nested_list = maybe_nested_list.unwrap().?; 307 | 308 | try expectNodeScope(tree, nested_list, 3, tree.tokens.len - 2); 309 | 310 | const nested_list_data = tree.extraData(List, tree.nodeData(nested_list).extra); 311 | try testing.expectEqual(3, nested_list_data.data.list_len); 312 | 313 | var nested_entry_data = tree.extraData(List.Entry, nested_list_data.end); 314 | try expectNestedMapListEntry(tree, nested_entry_data.data, "key2", "value2"); 315 | 316 | nested_entry_data = tree.extraData(List.Entry, nested_entry_data.end); 317 | try expectNestedMapListEntry(tree, nested_entry_data.data, "key3", "value3"); 318 | 319 | nested_entry_data = tree.extraData(List.Entry, nested_entry_data.end); 320 | try expectNestedMapListEntry(tree, nested_entry_data.data, "key4", "value4"); 321 | } 322 | } 323 | 324 | test "map of list of maps with inner list" { 325 | const source = 326 | \\ outer: 327 | \\ - a: foo 328 | \\ fooers: 329 | \\ - name: inner-foo 330 | \\ - b: bar 331 | \\ fooers: 332 | \\ - name: inner-bar 333 | ; 334 | 335 | var parser = try Parser.init(testing.allocator, source); 336 | defer parser.deinit(testing.allocator); 337 | try parser.parse(testing.allocator); 338 | 339 | var tree = try parser.toOwnedTree(testing.allocator); 340 | defer tree.deinit(testing.allocator); 341 | 342 | try testing.expectEqual(1, tree.docs.len); 343 | 344 | const doc = tree.docs[0]; 345 | try expectNodeScope(tree, doc, 1, tree.tokens.len - 2); 346 | 347 | const doc_value = tree.nodeData(doc).maybe_node; 348 | try testing.expect(doc_value != .none); 349 | try testing.expectEqual(.map_single, tree.nodeTag(doc_value.unwrap().?)); 350 | 351 | const map = doc_value.unwrap().?; 352 | 353 | try expectNodeScope(tree, map, 1, tree.tokens.len - 2); 354 | 355 | const map_data = tree.nodeData(map).map; 356 | 357 | { 358 | const key = tree.token(map_data.key); 359 | try testing.expectEqual(key.id, .literal); 360 | try testing.expectEqualStrings("outer", tree.rawString(map_data.key, map_data.key)); 361 | 362 | const maybe_nested_list = map_data.maybe_node; 363 | try testing.expect(maybe_nested_list != .none); 364 | try testing.expectEqual(.list_two, tree.nodeTag(maybe_nested_list.unwrap().?)); 365 | 366 | const nested_list = maybe_nested_list.unwrap().?; 367 | 368 | try expectNodeScope(tree, nested_list, 5, tree.tokens.len - 2); 369 | 370 | const nested_list_data = tree.nodeData(nested_list).list; 371 | 372 | { 373 | const nested_map = nested_list_data.el1; 374 | try testing.expectEqual(.map_many, tree.nodeTag(nested_map)); 375 | 376 | const nested_map_data = tree.extraData(Map, tree.nodeData(nested_map).extra); 377 | try testing.expectEqual(2, nested_map_data.data.map_len); 378 | 379 | var nested_nested_entry_data = tree.extraData(Map.Entry, nested_map_data.end); 380 | try expectValueMapEntry(tree, nested_nested_entry_data.data, "a", "foo"); 381 | 382 | nested_nested_entry_data = tree.extraData(Map.Entry, nested_nested_entry_data.end); 383 | { 384 | const nested_nested_map_entry = nested_nested_entry_data.data; 385 | const nested_nested_key = tree.token(nested_nested_map_entry.key); 386 | try testing.expectEqual(nested_nested_key.id, .literal); 387 | try testing.expectEqualStrings("fooers", tree.rawString(nested_nested_map_entry.key, nested_nested_map_entry.key)); 388 | 389 | const nested_nested_value = nested_nested_map_entry.maybe_node; 390 | try testing.expect(nested_nested_value != .none); 391 | try testing.expectEqual(.list_one, tree.nodeTag(nested_nested_value.unwrap().?)); 392 | 393 | const nested_nested_list = nested_nested_value.unwrap().?; 394 | const nested_nested_list_data = tree.nodeData(nested_nested_list).node; 395 | try expectNestedMapListEntry(tree, .{ .node = nested_nested_list_data }, "name", "inner-foo"); 396 | } 397 | } 398 | 399 | { 400 | const nested_map = nested_list_data.el2; 401 | try testing.expectEqual(.map_many, tree.nodeTag(nested_map)); 402 | 403 | const nested_map_data = tree.extraData(Map, tree.nodeData(nested_map).extra); 404 | try testing.expectEqual(2, nested_map_data.data.map_len); 405 | 406 | var nested_nested_entry_data = tree.extraData(Map.Entry, nested_map_data.end); 407 | try expectValueMapEntry(tree, nested_nested_entry_data.data, "b", "bar"); 408 | 409 | nested_nested_entry_data = tree.extraData(Map.Entry, nested_nested_entry_data.end); 410 | { 411 | const nested_nested_map_entry = nested_nested_entry_data.data; 412 | const nested_nested_key = tree.token(nested_nested_map_entry.key); 413 | try testing.expectEqual(nested_nested_key.id, .literal); 414 | try testing.expectEqualStrings("fooers", tree.rawString(nested_nested_map_entry.key, nested_nested_map_entry.key)); 415 | 416 | const nested_nested_value = nested_nested_map_entry.maybe_node; 417 | try testing.expect(nested_nested_value != .none); 418 | try testing.expectEqual(.list_one, tree.nodeTag(nested_nested_value.unwrap().?)); 419 | 420 | const nested_nested_list = nested_nested_value.unwrap().?; 421 | const nested_nested_list_data = tree.nodeData(nested_nested_list).node; 422 | try expectNestedMapListEntry(tree, .{ .node = nested_nested_list_data }, "name", "inner-bar"); 423 | } 424 | } 425 | } 426 | } 427 | 428 | test "list of lists" { 429 | const source = 430 | \\- [name , hr, avg ] 431 | \\- [Mark McGwire , 65, 0.278] 432 | \\- [Sammy Sosa , 63, 0.288] 433 | ; 434 | 435 | var parser = try Parser.init(testing.allocator, source); 436 | defer parser.deinit(testing.allocator); 437 | try parser.parse(testing.allocator); 438 | 439 | var tree = try parser.toOwnedTree(testing.allocator); 440 | defer tree.deinit(testing.allocator); 441 | 442 | try testing.expectEqual(1, tree.docs.len); 443 | 444 | const doc = tree.docs[0]; 445 | try expectNodeScope(tree, doc, 0, tree.tokens.len - 2); 446 | 447 | const doc_value = tree.nodeData(doc).maybe_node; 448 | try testing.expect(doc_value != .none); 449 | try testing.expectEqual(.list_many, tree.nodeTag(doc_value.unwrap().?)); 450 | 451 | const list = doc_value.unwrap().?; 452 | 453 | try expectNodeScope(tree, list, 0, tree.tokens.len - 2); 454 | 455 | const list_data = tree.extraData(List, tree.nodeData(list).extra); 456 | try testing.expectEqual(3, list_data.data.list_len); 457 | 458 | var entry_data = tree.extraData(List.Entry, list_data.end); 459 | { 460 | const nested_list = entry_data.data.node; 461 | 462 | try expectNodeScope(tree, nested_list, 1, 11); 463 | 464 | const nested_list_data = tree.extraData(List, tree.nodeData(nested_list).extra); 465 | try testing.expectEqual(3, nested_list_data.data.list_len); 466 | 467 | var nested_entry_data = tree.extraData(List.Entry, nested_list_data.end); 468 | try expectValueListEntry(tree, nested_entry_data.data, "name"); 469 | 470 | nested_entry_data = tree.extraData(List.Entry, nested_entry_data.end); 471 | try expectValueListEntry(tree, nested_entry_data.data, "hr"); 472 | 473 | nested_entry_data = tree.extraData(List.Entry, nested_entry_data.end); 474 | try expectValueListEntry(tree, nested_entry_data.data, "avg"); 475 | } 476 | 477 | entry_data = tree.extraData(List.Entry, entry_data.end); 478 | { 479 | const nested_list = entry_data.data.node; 480 | 481 | try expectNodeScope(tree, nested_list, 14, 25); 482 | 483 | const nested_list_data = tree.extraData(List, tree.nodeData(nested_list).extra); 484 | try testing.expectEqual(3, nested_list_data.data.list_len); 485 | 486 | var nested_entry_data = tree.extraData(List.Entry, nested_list_data.end); 487 | try expectValueListEntry(tree, nested_entry_data.data, "Mark McGwire"); 488 | 489 | nested_entry_data = tree.extraData(List.Entry, nested_entry_data.end); 490 | try expectValueListEntry(tree, nested_entry_data.data, "65"); 491 | 492 | nested_entry_data = tree.extraData(List.Entry, nested_entry_data.end); 493 | try expectValueListEntry(tree, nested_entry_data.data, "0.278"); 494 | } 495 | 496 | entry_data = tree.extraData(List.Entry, entry_data.end); 497 | { 498 | const nested_list = entry_data.data.node; 499 | 500 | try expectNodeScope(tree, nested_list, 28, 39); 501 | 502 | const nested_list_data = tree.extraData(List, tree.nodeData(nested_list).extra); 503 | try testing.expectEqual(3, nested_list_data.data.list_len); 504 | 505 | var nested_entry_data = tree.extraData(List.Entry, nested_list_data.end); 506 | try expectValueListEntry(tree, nested_entry_data.data, "Sammy Sosa"); 507 | 508 | nested_entry_data = tree.extraData(List.Entry, nested_entry_data.end); 509 | try expectValueListEntry(tree, nested_entry_data.data, "63"); 510 | 511 | nested_entry_data = tree.extraData(List.Entry, nested_entry_data.end); 512 | try expectValueListEntry(tree, nested_entry_data.data, "0.288"); 513 | } 514 | } 515 | 516 | test "inline list" { 517 | const source = 518 | \\[name , hr, avg ] 519 | ; 520 | 521 | var parser = try Parser.init(testing.allocator, source); 522 | defer parser.deinit(testing.allocator); 523 | try parser.parse(testing.allocator); 524 | 525 | var tree = try parser.toOwnedTree(testing.allocator); 526 | defer tree.deinit(testing.allocator); 527 | 528 | try testing.expectEqual(1, tree.docs.len); 529 | 530 | const doc = tree.docs[0]; 531 | try expectNodeScope(tree, doc, 0, tree.tokens.len - 2); 532 | 533 | const doc_value = tree.nodeData(doc).maybe_node; 534 | try testing.expect(doc_value != .none); 535 | try testing.expectEqual(.list_many, tree.nodeTag(doc_value.unwrap().?)); 536 | 537 | const list = doc_value.unwrap().?; 538 | 539 | try expectNodeScope(tree, list, 0, tree.tokens.len - 2); 540 | 541 | const list_data = tree.extraData(List, tree.nodeData(list).extra); 542 | try testing.expectEqual(3, list_data.data.list_len); 543 | 544 | var entry_data = tree.extraData(List.Entry, list_data.end); 545 | try expectValueListEntry(tree, entry_data.data, "name"); 546 | 547 | entry_data = tree.extraData(List.Entry, entry_data.end); 548 | try expectValueListEntry(tree, entry_data.data, "hr"); 549 | 550 | entry_data = tree.extraData(List.Entry, entry_data.end); 551 | try expectValueListEntry(tree, entry_data.data, "avg"); 552 | } 553 | 554 | test "inline list as mapping value" { 555 | const source = 556 | \\key : [ 557 | \\ name , 558 | \\ hr, avg ] 559 | ; 560 | 561 | var parser = try Parser.init(testing.allocator, source); 562 | defer parser.deinit(testing.allocator); 563 | try parser.parse(testing.allocator); 564 | 565 | var tree = try parser.toOwnedTree(testing.allocator); 566 | defer tree.deinit(testing.allocator); 567 | 568 | try testing.expectEqual(1, tree.docs.len); 569 | 570 | const doc = tree.docs[0]; 571 | try expectNodeScope(tree, doc, 0, tree.tokens.len - 2); 572 | 573 | const doc_value = tree.nodeData(doc).maybe_node; 574 | try testing.expect(doc_value != .none); 575 | try testing.expectEqual(.map_single, tree.nodeTag(doc_value.unwrap().?)); 576 | 577 | const map = doc_value.unwrap().?; 578 | 579 | try expectNodeScope(tree, map, 0, tree.tokens.len - 2); 580 | 581 | const map_data = tree.nodeData(map).map; 582 | 583 | const key = tree.token(map_data.key); 584 | try testing.expectEqual(key.id, .literal); 585 | try testing.expectEqualStrings("key", tree.rawString(map_data.key, map_data.key)); 586 | 587 | const maybe_nested_list = map_data.maybe_node; 588 | try testing.expect(maybe_nested_list != .none); 589 | try testing.expectEqual(.list_many, tree.nodeTag(maybe_nested_list.unwrap().?)); 590 | 591 | const nested_list = maybe_nested_list.unwrap().?; 592 | 593 | try expectNodeScope(tree, nested_list, 4, tree.tokens.len - 2); 594 | 595 | const nested_list_data = tree.extraData(List, tree.nodeData(nested_list).extra); 596 | try testing.expectEqual(3, nested_list_data.data.list_len); 597 | 598 | var nested_entry_data = tree.extraData(List.Entry, nested_list_data.end); 599 | try expectValueListEntry(tree, nested_entry_data.data, "name"); 600 | 601 | nested_entry_data = tree.extraData(List.Entry, nested_entry_data.end); 602 | try expectValueListEntry(tree, nested_entry_data.data, "hr"); 603 | 604 | nested_entry_data = tree.extraData(List.Entry, nested_entry_data.end); 605 | try expectValueListEntry(tree, nested_entry_data.data, "avg"); 606 | } 607 | 608 | fn parseSuccess(comptime source: []const u8) !void { 609 | var parser = try Parser.init(testing.allocator, source); 610 | defer parser.deinit(testing.allocator); 611 | try parser.parse(testing.allocator); 612 | } 613 | 614 | fn parseError(comptime source: []const u8, err: Parser.ParseError) !void { 615 | var parser = try Parser.init(testing.allocator, source); 616 | defer parser.deinit(testing.allocator); 617 | try testing.expectError(err, parser.parse(testing.allocator)); 618 | } 619 | 620 | fn parseError2(source: []const u8, comptime format: []const u8, args: anytype) !void { 621 | var parser = try Parser.init(testing.allocator, source); 622 | defer parser.deinit(testing.allocator); 623 | 624 | const res = parser.parse(testing.allocator); 625 | try testing.expectError(error.ParseFailure, res); 626 | 627 | var bundle = try parser.errors.toOwnedBundle(""); 628 | defer bundle.deinit(testing.allocator); 629 | try testing.expect(bundle.errorMessageCount() > 0); 630 | 631 | var given: std.Io.Writer.Allocating = .init(testing.allocator); 632 | defer given.deinit(); 633 | try bundle.renderToWriter(.{ .ttyconf = .no_color }, &given.writer); 634 | 635 | const expected = try std.fmt.allocPrint(testing.allocator, format, args); 636 | defer testing.allocator.free(expected); 637 | try testing.expectEqualStrings(expected, given.written()); 638 | } 639 | 640 | test "empty doc with spaces and comments" { 641 | try parseSuccess( 642 | \\ 643 | \\ 644 | \\ # this is a comment in a weird place 645 | \\# and this one is too 646 | ); 647 | } 648 | 649 | test "comment between --- and ! in document start" { 650 | try parseError2( 651 | \\--- # what is it? 652 | \\! 653 | , 654 | \\(memory):2:1: error: expected end of document 655 | \\! 656 | \\^ 657 | \\ 658 | , .{}); 659 | } 660 | 661 | test "correct doc start with tag" { 662 | try parseSuccess( 663 | \\--- !some-tag 664 | \\ 665 | ); 666 | } 667 | 668 | test "doc close without explicit doc open" { 669 | try parseError2( 670 | \\ 671 | \\ 672 | \\# something cool 673 | \\... 674 | , 675 | \\(memory):4:1: error: missing explicit document open marker '---' 676 | \\... 677 | \\^~~ 678 | \\ 679 | , .{}); 680 | } 681 | 682 | test "doc open and close are ok" { 683 | try parseSuccess( 684 | \\--- 685 | \\# first doc 686 | \\ 687 | \\ 688 | \\--- 689 | \\# second doc 690 | \\ 691 | \\ 692 | \\... 693 | ); 694 | } 695 | 696 | test "doc with a single string is ok" { 697 | try parseSuccess( 698 | \\a string of some sort 699 | \\ 700 | ); 701 | } 702 | 703 | test "explicit doc with a single string is ok" { 704 | try parseSuccess( 705 | \\--- !anchor 706 | \\# nothing to see here except one string 707 | \\ # not a lot to go on with 708 | \\a single string 709 | \\... 710 | ); 711 | } 712 | 713 | test "doc with two string is bad" { 714 | try parseError2( 715 | \\first 716 | \\second 717 | \\# this should fail already 718 | , 719 | \\(memory):2:1: error: expected end of document 720 | \\second 721 | \\^~~~~~ 722 | \\ 723 | , .{}); 724 | } 725 | 726 | test "single quote string can have new lines" { 727 | try parseSuccess( 728 | \\'what is this 729 | \\ thing?' 730 | ); 731 | } 732 | 733 | test "single quote string on one line is fine" { 734 | try parseSuccess( 735 | \\'here''s an apostrophe' 736 | ); 737 | } 738 | 739 | test "double quote string can have new lines" { 740 | try parseSuccess( 741 | \\"what is this 742 | \\ thing?" 743 | ); 744 | } 745 | 746 | test "double quote string on one line is fine" { 747 | try parseSuccess( 748 | \\"a newline\nand a\ttab" 749 | ); 750 | } 751 | 752 | test "map with key and value literals" { 753 | try parseSuccess( 754 | \\key1: val1 755 | \\key2 : val2 756 | ); 757 | } 758 | 759 | test "map of maps" { 760 | try parseSuccess( 761 | \\ 762 | \\# the first key 763 | \\key1: 764 | \\ # the first subkey 765 | \\ key1_1: 0 766 | \\ key1_2: 1 767 | \\# the second key 768 | \\key2: 769 | \\ key2_1: -1 770 | \\ key2_2: -2 771 | \\# the end of map 772 | ); 773 | } 774 | 775 | test "map value indicator needs to be on the same line" { 776 | try parseError2( 777 | \\a 778 | \\ : b 779 | , 780 | \\(memory):2:3: error: expected end of document 781 | \\ : b 782 | \\ ^~~ 783 | \\ 784 | , .{}); 785 | } 786 | 787 | test "value needs to be indented" { 788 | try parseError2( 789 | \\a: 790 | \\b 791 | , 792 | \\(memory):2:1: error: 'value' in map should have more indentation than the 'key' 793 | \\b 794 | \\^ 795 | \\ 796 | , .{}); 797 | } 798 | 799 | test "comment between a key and a value is fine" { 800 | try parseSuccess( 801 | \\a: 802 | \\ # this is a value 803 | \\ b 804 | ); 805 | } 806 | 807 | test "simple list" { 808 | try parseSuccess( 809 | \\# first el 810 | \\- a 811 | \\# second el 812 | \\- b 813 | \\# third el 814 | \\- c 815 | ); 816 | } 817 | 818 | test "list indentation matters" { 819 | try parseError2( 820 | \\ - a 821 | \\- b 822 | , 823 | \\(memory):2:1: error: expected end of document 824 | \\- b 825 | \\^~~ 826 | \\ 827 | , .{}); 828 | 829 | try parseSuccess( 830 | \\- a 831 | \\ - b 832 | ); 833 | } 834 | 835 | test "unindented list is fine too" { 836 | try parseSuccess( 837 | \\a: 838 | \\- 0 839 | \\- 1 840 | ); 841 | } 842 | 843 | test "empty values in a map" { 844 | try parseSuccess( 845 | \\a: 846 | \\b: 847 | \\- 0 848 | ); 849 | } 850 | 851 | test "weirdly nested map of maps of lists" { 852 | try parseSuccess( 853 | \\a: 854 | \\ b: 855 | \\ - 0 856 | \\ - 1 857 | ); 858 | } 859 | 860 | test "square brackets denote a list" { 861 | try parseSuccess( 862 | \\[ a, 863 | \\ b, c ] 864 | ); 865 | } 866 | 867 | test "empty list" { 868 | try parseSuccess( 869 | \\[ ] 870 | ); 871 | } 872 | 873 | test "empty map" { 874 | try parseSuccess( 875 | \\a: 876 | \\ b: {} 877 | \\ c: { } 878 | ); 879 | } 880 | 881 | test "comment within a bracketed list is an error" { 882 | try parseError( 883 | \\[ # something 884 | \\] 885 | , error.ParseFailure); 886 | } 887 | 888 | test "mixed ints with floats in a list" { 889 | try parseSuccess( 890 | \\[0, 1.0] 891 | ); 892 | } 893 | 894 | test "expect end of document" { 895 | try parseError2( 896 | \\ key1: value1 897 | \\key2: value2 898 | , 899 | \\(memory):2:1: error: expected end of document 900 | \\key2: value2 901 | \\^~~~~~~~~~~~ 902 | \\ 903 | , .{}); 904 | } 905 | 906 | test "expect map separator" { 907 | try parseError2( 908 | \\key1: value1 909 | \\key2 910 | , 911 | \\(memory):2:5: error: expected map separator ':' 912 | \\key2 913 | \\~~~~^ 914 | \\ 915 | , .{}); 916 | } 917 | --------------------------------------------------------------------------------