├── .gitignore ├── README.md ├── LICENSE └── src ├── docgen.zig └── doctest.zig /.gitignore: -------------------------------------------------------------------------------- 1 | .zig-cache/ 2 | /zig-out/ 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # docgen 2 | 3 | A tool for generating documents such as 4 | [Zig's language reference](https://ziglang.org/documentation/0.12.0/) or 5 | [release notes](https://ziglang.org/download/0.12.0/release-notes.html). 6 | 7 | This repository is provided as a Zig package for convenience. The canonical 8 | source lives in the zig compiler repository: 9 | * [doctest.zig](https://github.com/ziglang/zig/blob/master/tools/doctest.zig) 10 | * [docgen.zig](https://github.com/ziglang/zig/blob/master/tools/docgen.zig) 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (Expat) 2 | 3 | Copyright (c) Zig contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/docgen.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const builtin = @import("builtin"); 3 | const io = std.io; 4 | const fs = std.fs; 5 | const process = std.process; 6 | const ChildProcess = std.process.Child; 7 | const Progress = std.Progress; 8 | const print = std.debug.print; 9 | const mem = std.mem; 10 | const testing = std.testing; 11 | const Allocator = std.mem.Allocator; 12 | const getExternalExecutor = std.zig.system.getExternalExecutor; 13 | const fatal = std.process.fatal; 14 | 15 | const max_doc_file_size = 10 * 1024 * 1024; 16 | 17 | const obj_ext = builtin.object_format.fileExt(builtin.cpu.arch); 18 | 19 | const usage = 20 | \\Usage: docgen [options] input output 21 | \\ 22 | \\ Generates an HTML document from a docgen template. 23 | \\ 24 | \\Options: 25 | \\ --code-dir dir Path to directory containing code example outputs 26 | \\ -h, --help Print this help and exit 27 | \\ 28 | ; 29 | 30 | pub fn main() !void { 31 | var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator); 32 | defer arena_instance.deinit(); 33 | 34 | const arena = arena_instance.allocator(); 35 | 36 | var args_it = try process.argsWithAllocator(arena); 37 | if (!args_it.skip()) @panic("expected self arg"); 38 | 39 | var opt_code_dir: ?[]const u8 = null; 40 | var opt_input: ?[]const u8 = null; 41 | var opt_output: ?[]const u8 = null; 42 | 43 | while (args_it.next()) |arg| { 44 | if (mem.startsWith(u8, arg, "-")) { 45 | if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { 46 | try std.fs.File.stdout().writeAll(usage); 47 | process.exit(0); 48 | } else if (mem.eql(u8, arg, "--code-dir")) { 49 | if (args_it.next()) |param| { 50 | opt_code_dir = param; 51 | } else { 52 | fatal("expected parameter after --code-dir", .{}); 53 | } 54 | } else { 55 | fatal("unrecognized option: '{s}'", .{arg}); 56 | } 57 | } else if (opt_input == null) { 58 | opt_input = arg; 59 | } else if (opt_output == null) { 60 | opt_output = arg; 61 | } else { 62 | fatal("unexpected positional argument: '{s}'", .{arg}); 63 | } 64 | } 65 | const input_path = opt_input orelse fatal("missing input file", .{}); 66 | const output_path = opt_output orelse fatal("missing output file", .{}); 67 | const code_dir_path = opt_code_dir orelse fatal("missing --code-dir argument", .{}); 68 | 69 | var in_file = try fs.cwd().openFile(input_path, .{}); 70 | defer in_file.close(); 71 | var in_file_reader = in_file.reader(&.{}); 72 | 73 | var out_file = try fs.cwd().createFile(output_path, .{}); 74 | defer out_file.close(); 75 | var out_file_buffer: [4096]u8 = undefined; 76 | var out_file_writer = out_file.writer(&out_file_buffer); 77 | 78 | var code_dir = try fs.cwd().openDir(code_dir_path, .{}); 79 | defer code_dir.close(); 80 | 81 | const input_file_bytes = try in_file_reader.interface.allocRemaining(arena, .limited(max_doc_file_size)); 82 | 83 | var tokenizer = Tokenizer.init(input_path, input_file_bytes); 84 | var toc = try genToc(arena, &tokenizer); 85 | 86 | try genHtml(arena, &tokenizer, &toc, code_dir, &out_file_writer.interface); 87 | try out_file_writer.end(); 88 | } 89 | 90 | const Token = struct { 91 | id: Id, 92 | start: usize, 93 | end: usize, 94 | 95 | const Id = enum { 96 | invalid, 97 | content, 98 | bracket_open, 99 | tag_content, 100 | separator, 101 | bracket_close, 102 | eof, 103 | }; 104 | }; 105 | 106 | const Tokenizer = struct { 107 | buffer: []const u8, 108 | index: usize, 109 | state: State, 110 | source_file_name: []const u8, 111 | 112 | const State = enum { 113 | start, 114 | l_bracket, 115 | hash, 116 | tag_name, 117 | eof, 118 | }; 119 | 120 | fn init(source_file_name: []const u8, buffer: []const u8) Tokenizer { 121 | return Tokenizer{ 122 | .buffer = buffer, 123 | .index = 0, 124 | .state = .start, 125 | .source_file_name = source_file_name, 126 | }; 127 | } 128 | 129 | fn next(self: *Tokenizer) Token { 130 | var result = Token{ 131 | .id = .eof, 132 | .start = self.index, 133 | .end = undefined, 134 | }; 135 | while (self.index < self.buffer.len) : (self.index += 1) { 136 | const c = self.buffer[self.index]; 137 | switch (self.state) { 138 | .start => switch (c) { 139 | '{' => { 140 | self.state = .l_bracket; 141 | }, 142 | else => { 143 | result.id = .content; 144 | }, 145 | }, 146 | .l_bracket => switch (c) { 147 | '#' => { 148 | if (result.id != .eof) { 149 | self.index -= 1; 150 | self.state = .start; 151 | break; 152 | } else { 153 | result.id = .bracket_open; 154 | self.index += 1; 155 | self.state = .tag_name; 156 | break; 157 | } 158 | }, 159 | else => { 160 | result.id = .content; 161 | self.state = .start; 162 | }, 163 | }, 164 | .tag_name => switch (c) { 165 | '|' => { 166 | if (result.id != .eof) { 167 | break; 168 | } else { 169 | result.id = .separator; 170 | self.index += 1; 171 | break; 172 | } 173 | }, 174 | '#' => { 175 | self.state = .hash; 176 | }, 177 | else => { 178 | result.id = .tag_content; 179 | }, 180 | }, 181 | .hash => switch (c) { 182 | '}' => { 183 | if (result.id != .eof) { 184 | self.index -= 1; 185 | self.state = .tag_name; 186 | break; 187 | } else { 188 | result.id = .bracket_close; 189 | self.index += 1; 190 | self.state = .start; 191 | break; 192 | } 193 | }, 194 | else => { 195 | result.id = .tag_content; 196 | self.state = .tag_name; 197 | }, 198 | }, 199 | .eof => unreachable, 200 | } 201 | } else { 202 | switch (self.state) { 203 | .start, .l_bracket, .eof => {}, 204 | else => { 205 | result.id = .invalid; 206 | }, 207 | } 208 | self.state = .eof; 209 | } 210 | result.end = self.index; 211 | return result; 212 | } 213 | 214 | const Location = struct { 215 | line: usize, 216 | column: usize, 217 | line_start: usize, 218 | line_end: usize, 219 | }; 220 | 221 | fn getTokenLocation(self: *Tokenizer, token: Token) Location { 222 | var loc = Location{ 223 | .line = 0, 224 | .column = 0, 225 | .line_start = 0, 226 | .line_end = 0, 227 | }; 228 | for (self.buffer, 0..) |c, i| { 229 | if (i == token.start) { 230 | loc.line_end = i; 231 | while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {} 232 | return loc; 233 | } 234 | if (c == '\n') { 235 | loc.line += 1; 236 | loc.column = 0; 237 | loc.line_start = i + 1; 238 | } else { 239 | loc.column += 1; 240 | } 241 | } 242 | return loc; 243 | } 244 | }; 245 | 246 | fn parseError(tokenizer: *Tokenizer, token: Token, comptime fmt: []const u8, args: anytype) anyerror { 247 | const loc = tokenizer.getTokenLocation(token); 248 | const args_prefix = .{ tokenizer.source_file_name, loc.line + 1, loc.column + 1 }; 249 | print("{s}:{d}:{d}: error: " ++ fmt ++ "\n", args_prefix ++ args); 250 | if (loc.line_start <= loc.line_end) { 251 | print("{s}\n", .{tokenizer.buffer[loc.line_start..loc.line_end]}); 252 | { 253 | var i: usize = 0; 254 | while (i < loc.column) : (i += 1) { 255 | print(" ", .{}); 256 | } 257 | } 258 | { 259 | const caret_count = @min(token.end, loc.line_end) - token.start; 260 | var i: usize = 0; 261 | while (i < caret_count) : (i += 1) { 262 | print("~", .{}); 263 | } 264 | } 265 | print("\n", .{}); 266 | } 267 | return error.ParseError; 268 | } 269 | 270 | fn assertToken(tokenizer: *Tokenizer, token: Token, id: Token.Id) !void { 271 | if (token.id != id) { 272 | return parseError(tokenizer, token, "expected {s}, found {s}", .{ @tagName(id), @tagName(token.id) }); 273 | } 274 | } 275 | 276 | fn eatToken(tokenizer: *Tokenizer, id: Token.Id) !Token { 277 | const token = tokenizer.next(); 278 | try assertToken(tokenizer, token, id); 279 | return token; 280 | } 281 | 282 | const HeaderOpen = struct { 283 | name: []const u8, 284 | url: []const u8, 285 | n: usize, 286 | }; 287 | 288 | const SeeAlsoItem = struct { 289 | name: []const u8, 290 | token: Token, 291 | }; 292 | 293 | const Code = struct { 294 | name: []const u8, 295 | token: Token, 296 | }; 297 | 298 | const Link = struct { 299 | url: []const u8, 300 | name: []const u8, 301 | token: Token, 302 | }; 303 | 304 | const SyntaxBlock = struct { 305 | source_type: SourceType, 306 | name: []const u8, 307 | source_token: Token, 308 | 309 | const SourceType = enum { 310 | zig, 311 | c, 312 | peg, 313 | javascript, 314 | }; 315 | }; 316 | 317 | const Node = union(enum) { 318 | Content: []const u8, 319 | Nav, 320 | Builtin: Token, 321 | HeaderOpen: HeaderOpen, 322 | SeeAlso: []const SeeAlsoItem, 323 | Code: Code, 324 | Embed: Code, 325 | Link: Link, 326 | InlineSyntax: Token, 327 | Shell: Token, 328 | SyntaxBlock: SyntaxBlock, 329 | }; 330 | 331 | const Toc = struct { 332 | nodes: []Node, 333 | toc: []u8, 334 | urls: std.StringHashMap(Token), 335 | }; 336 | 337 | const Action = enum { 338 | open, 339 | close, 340 | }; 341 | 342 | fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { 343 | var urls = std.StringHashMap(Token).init(allocator); 344 | errdefer urls.deinit(); 345 | 346 | var header_stack_size: usize = 0; 347 | var last_action: Action = .open; 348 | var last_columns: ?u8 = null; 349 | 350 | var toc_buf = std.array_list.Managed(u8).init(allocator); 351 | defer toc_buf.deinit(); 352 | 353 | var toc = toc_buf.writer(); 354 | 355 | var nodes = std.array_list.Managed(Node).init(allocator); 356 | defer nodes.deinit(); 357 | 358 | try toc.writeByte('\n'); 359 | 360 | while (true) { 361 | const token = tokenizer.next(); 362 | switch (token.id) { 363 | .eof => { 364 | if (header_stack_size != 0) { 365 | return parseError(tokenizer, token, "unbalanced headers", .{}); 366 | } 367 | try toc.writeAll(" \n"); 368 | break; 369 | }, 370 | .content => { 371 | try nodes.append(Node{ .Content = tokenizer.buffer[token.start..token.end] }); 372 | }, 373 | .bracket_open => { 374 | const tag_token = try eatToken(tokenizer, .tag_content); 375 | const tag_name = tokenizer.buffer[tag_token.start..tag_token.end]; 376 | 377 | if (mem.eql(u8, tag_name, "nav")) { 378 | _ = try eatToken(tokenizer, .bracket_close); 379 | 380 | try nodes.append(Node.Nav); 381 | } else if (mem.eql(u8, tag_name, "builtin")) { 382 | _ = try eatToken(tokenizer, .bracket_close); 383 | try nodes.append(Node{ .Builtin = tag_token }); 384 | } else if (mem.eql(u8, tag_name, "header_open")) { 385 | _ = try eatToken(tokenizer, .separator); 386 | const content_token = try eatToken(tokenizer, .tag_content); 387 | const content = tokenizer.buffer[content_token.start..content_token.end]; 388 | var columns: ?u8 = null; 389 | while (true) { 390 | const bracket_tok = tokenizer.next(); 391 | switch (bracket_tok.id) { 392 | .bracket_close => break, 393 | .separator => continue, 394 | .tag_content => { 395 | const param = tokenizer.buffer[bracket_tok.start..bracket_tok.end]; 396 | if (mem.eql(u8, param, "2col")) { 397 | columns = 2; 398 | } else { 399 | return parseError( 400 | tokenizer, 401 | bracket_tok, 402 | "unrecognized header_open param: {s}", 403 | .{param}, 404 | ); 405 | } 406 | }, 407 | else => return parseError(tokenizer, bracket_tok, "invalid header_open token", .{}), 408 | } 409 | } 410 | 411 | header_stack_size += 1; 412 | 413 | const urlized = try urlize(allocator, content); 414 | try nodes.append(Node{ 415 | .HeaderOpen = HeaderOpen{ 416 | .name = content, 417 | .url = urlized, 418 | .n = header_stack_size + 1, // highest-level section headers start at h2 419 | }, 420 | }); 421 | if (try urls.fetchPut(urlized, tag_token)) |kv| { 422 | parseError(tokenizer, tag_token, "duplicate header url: #{s}", .{urlized}) catch {}; 423 | parseError(tokenizer, kv.value, "other tag here", .{}) catch {}; 424 | return error.ParseError; 425 | } 426 | if (last_action == .open) { 427 | try toc.writeByte('\n'); 428 | try toc.writeByteNTimes(' ', header_stack_size * 4); 429 | if (last_columns) |n| { 430 | try toc.print("