From 9f7db3a9161f0b90fa9a9feeebdea0fa050ec46b Mon Sep 17 00:00:00 2001 From: Spencer Brower <6729162+sbrow@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:27:38 -0400 Subject: [PATCH] perf: Rewriting `convert_deflists` for performance. --- flake.nix | 4 + markdown/deflists.odin | 348 ++++++++++++------ markdown/deflists_fixtures_test.odin | 92 +++++ markdown/testdata/deflists/01-basic.md | 2 + .../testdata/deflists/02-multiple-defs.md | 7 + .../testdata/deflists/03-multiline-def.md | 5 + .../testdata/deflists/04-markdown-in-def.md | 7 + markdown/testdata/deflists/05-nested.md | 6 + markdown/testdata/deflists/06-compact.md | 1 + markdown/testdata/deflists/07-extra-space.md | 2 + markdown/testdata/deflists/08-glossary.md | 11 + markdown/testdata/deflists/09-api-params.md | 14 + .../testdata/deflists/expected/01-basic.html | 1 + .../deflists/expected/02-multiple-defs.html | 1 + .../deflists/expected/03-multiline-def.html | 3 + .../deflists/expected/04-markdown-in-def.html | 5 + .../testdata/deflists/expected/05-nested.html | 1 + .../deflists/expected/06-compact.html | 1 + .../deflists/expected/07-extra-space.html | 1 + .../deflists/expected/08-glossary.html | 1 + .../deflists/expected/09-api-params.html | 1 + .../testdata/deflists/reference/COMPARISON.md | 60 +++ .../testdata/deflists/reference/README.md | 44 +++ .../deflists/reference/hugo/01-basic.html | 4 + .../reference/hugo/02-multiple-defs.html | 8 + .../reference/hugo/03-multiline-def.html | 7 + .../reference/hugo/04-markdown-in-def.html | 16 + .../deflists/reference/hugo/05-nested.html | 10 + .../deflists/reference/hugo/06-compact.html | 1 + .../reference/hugo/07-extra-space.html | 2 + .../deflists/reference/hugo/08-glossary.html | 10 + .../reference/hugo/09-api-params.html | 14 + .../deflists/reference/pandoc/01-basic.html | 6 + .../reference/pandoc/02-multiple-defs.html | 16 + .../reference/pandoc/03-multiline-def.html | 9 + .../reference/pandoc/04-markdown-in-def.html | 9 + .../deflists/reference/pandoc/05-nested.html | 13 + .../deflists/reference/pandoc/06-compact.html | 1 + .../reference/pandoc/07-extra-space.html | 6 + .../reference/pandoc/08-glossary.html | 22 ++ .../reference/pandoc/09-api-params.html | 33 ++ 41 files changed, 688 insertions(+), 117 deletions(-) create mode 100644 markdown/deflists_fixtures_test.odin create mode 100644 markdown/testdata/deflists/01-basic.md create mode 100644 markdown/testdata/deflists/02-multiple-defs.md create mode 100644 markdown/testdata/deflists/03-multiline-def.md create mode 100644 markdown/testdata/deflists/04-markdown-in-def.md create mode 100644 markdown/testdata/deflists/05-nested.md create mode 100644 markdown/testdata/deflists/06-compact.md create mode 100644 markdown/testdata/deflists/07-extra-space.md create mode 100644 markdown/testdata/deflists/08-glossary.md create mode 100644 markdown/testdata/deflists/09-api-params.md create mode 100644 markdown/testdata/deflists/expected/01-basic.html create mode 100644 markdown/testdata/deflists/expected/02-multiple-defs.html create mode 100644 markdown/testdata/deflists/expected/03-multiline-def.html create mode 100644 markdown/testdata/deflists/expected/04-markdown-in-def.html create mode 100644 markdown/testdata/deflists/expected/05-nested.html create mode 100644 markdown/testdata/deflists/expected/06-compact.html create mode 100644 markdown/testdata/deflists/expected/07-extra-space.html create mode 100644 markdown/testdata/deflists/expected/08-glossary.html create mode 100644 markdown/testdata/deflists/expected/09-api-params.html create mode 100644 markdown/testdata/deflists/reference/COMPARISON.md create mode 100644 markdown/testdata/deflists/reference/README.md create mode 100644 markdown/testdata/deflists/reference/hugo/01-basic.html create mode 100644 markdown/testdata/deflists/reference/hugo/02-multiple-defs.html create mode 100644 markdown/testdata/deflists/reference/hugo/03-multiline-def.html create mode 100644 markdown/testdata/deflists/reference/hugo/04-markdown-in-def.html create mode 100644 markdown/testdata/deflists/reference/hugo/05-nested.html create mode 100644 markdown/testdata/deflists/reference/hugo/06-compact.html create mode 100644 markdown/testdata/deflists/reference/hugo/07-extra-space.html create mode 100644 markdown/testdata/deflists/reference/hugo/08-glossary.html create mode 100644 markdown/testdata/deflists/reference/hugo/09-api-params.html create mode 100644 markdown/testdata/deflists/reference/pandoc/01-basic.html create mode 100644 markdown/testdata/deflists/reference/pandoc/02-multiple-defs.html create mode 100644 markdown/testdata/deflists/reference/pandoc/03-multiline-def.html create mode 100644 markdown/testdata/deflists/reference/pandoc/04-markdown-in-def.html create mode 100644 markdown/testdata/deflists/reference/pandoc/05-nested.html create mode 100644 markdown/testdata/deflists/reference/pandoc/06-compact.html create mode 100644 markdown/testdata/deflists/reference/pandoc/07-extra-space.html create mode 100644 markdown/testdata/deflists/reference/pandoc/08-glossary.html create mode 100644 markdown/testdata/deflists/reference/pandoc/09-api-params.html diff --git a/flake.nix b/flake.nix index 48fe67f..b3695ac 100644 --- a/flake.nix +++ b/flake.nix @@ -169,6 +169,10 @@ bloaty # du for binaries? + # temp + hugo + pandoc + # IDE unstable.helix claude-code diff --git a/markdown/deflists.odin b/markdown/deflists.odin index 0525b77..ac542fe 100644 --- a/markdown/deflists.odin +++ b/markdown/deflists.odin @@ -4,140 +4,275 @@ import cm "vendor:commonmark" import "core:strings" -// DefList_Entry represents a single term-definition pair in a definition list. -DefList_Entry :: struct { - term: string, - definition: string, -} - // convert_deflists scans markdown text for definition list patterns and // converts them to
HTML blocks before cmark processing. // -// A definition line starts with optional whitespace followed by a colon and -// a space. The term is the nearest preceding non-blank line (immediately or -// within one blank line). Consecutive term+definition pairs are grouped into -// a single
block. +// A definition line starts with fewer than four columns of leading whitespace +// followed by a colon and a space (or end-of-line). The term is the nearest +// preceding non-blank line (immediately or within one blank line). Consecutive +// term+definition pairs are grouped into a single
block, and consecutive +// definition lines under one term become multiple
elements. // -// Terms and definitions are rendered through cmark individually so that -// inline markdown (code, links, emphasis) is processed. +// The scanner walks the body one line at a time using byte offsets: it never +// splits the input, allocates nothing per line, and copies non-deflist text +// through verbatim. Matching is disabled inside fenced code blocks. Terms and +// definitions are rendered through cmark individually so that inline markdown +// (code, links, emphasis) is processed. +// Line_Kind classifies a line for convert_deflists's group-finding scan. Other +// covers fenced-code delimiters and their contents, where definition syntax is +// inert. +Line_Kind :: enum { + None, + Blank, + Term, + Def, + Other, +} + convert_deflists :: proc(body: string, allocator := context.allocator) -> string { - lines := strings.split(body, "\n", allocator = context.temp_allocator) + // Fast path: a body with no definition line at all cannot contain a + // definition list, so return it untouched — skipping the builder allocation + // and the classifying scan below. This is the common case (most pages have + // no definition lists), and the guard scan is leaner than the full loop. + if !has_def_line(body) { + return body + } sb := strings.builder_make(context.temp_allocator) - first := true - need_blank := false + // emitted_up_to is the byte offset up to which `body` has been copied into + // `sb`. It stays at 0 until the first deflist is emitted; if it is still 0 + // at the end, nothing matched and we can return the body unchanged. + emitted_up_to := 0 - i := 0 - for i < len(lines) { - entries, matched, next := try_match_deflist(lines, i) - if matched { - html := render_deflist(entries) - if !first { - strings.write_string(&sb, "\n\n") + in_fence := false + fence_marker: u8 = 0 + + // The scan is definition-driven: it walks line by line, remembering the two + // previous lines' kinds and offsets, and only does group-finding work when it + // reaches a (rare) definition line — whose term is the immediately preceding + // line, or the one before a single blank line. This avoids a per-line forward + // lookahead: each line is classified once, with no rescanning. + prev1_kind, prev2_kind := Line_Kind.None, Line_Kind.None + prev1_pos, prev2_pos := 0, 0 + + pos := 0 + for { + line, next, ok := line_at(body, pos) + if !ok { + break + } + + // Classify the line. Definition-list syntax is inert inside fenced code + // blocks, so fence delimiters and their contents are classified Other. + kind: Line_Kind + if in_fence { + if is_fence_line(line, fence_marker) { + in_fence = false } - strings.write_string(&sb, html) - first = false - need_blank = true - i = next - continue + kind = .Other + } else if marker, opened := fence_open(line); opened { + in_fence = true + fence_marker = marker + kind = .Other + } else if is_def_line(line) { + kind = .Def + } else if is_blank_line(line) { + kind = .Blank + } else { + kind = .Term } - if need_blank { - strings.write_string(&sb, "\n\n") - need_blank = false - } else if !first { - strings.write_string(&sb, "\n") + // A definition-list group begins at the term preceding this def line + // (immediately, or across a single blank line). Flush pending passthrough + // text verbatim, then render the whole group directly into the builder. + if kind == .Def { + term_pos := -1 + if prev1_kind == .Term { + term_pos = prev1_pos + } else if prev1_kind == .Blank && prev2_kind == .Term { + term_pos = prev2_pos + } + if term_pos >= 0 { + strings.write_string(&sb, body[emitted_up_to:term_pos]) + group_end := write_deflist_group(&sb, body, term_pos) + emitted_up_to = group_end + pos = group_end + prev1_kind, prev2_kind = .None, .None + in_fence = false + continue + } } - strings.write_string(&sb, lines[i]) - first = false - i += 1 + + prev2_kind, prev2_pos = prev1_kind, prev1_pos + prev1_kind, prev1_pos = kind, pos + pos = next } + if emitted_up_to == 0 { + // No definition list anywhere; the body passes through unchanged. + return body + } + strings.write_string(&sb, body[emitted_up_to:]) return strings.clone(strings.to_string(sb), allocator) } -// try_match_deflist attempts to match a definition list group starting at -// lines[start]. A group is one or more term+definition pairs. Returns the -// matched entries, whether a match was found, and the index past the group. -try_match_deflist :: proc( - lines: []string, - start: int, -) -> ( - entries: [dynamic]DefList_Entry, - ok: bool, - end: int, -) { - entries = make([dynamic]DefList_Entry, 0, allocator = context.temp_allocator) - end = start +// line_at returns the line beginning at byte offset `pos` (excluding its +// trailing newline) along with the offset of the following line. `pos` must be +// 0 or one past a '\n'. ok is false once `pos` reaches the end of `body`. +line_at :: proc(body: string, pos: int) -> (line: string, next: int, ok: bool) { + if pos >= len(body) { + return "", pos, false + } + nl := strings.index_byte(body[pos:], '\n') + if nl < 0 { + return body[pos:], len(body), true + } + return body[pos:pos + nl], pos + nl + 1, true +} - i := start - for i < len(lines) { - // A term must be non-blank and not itself a def line - if is_blank_line(lines[i]) || is_def_line(lines[i]) { +// has_def_line reports whether any line in `body` is a definition line (see +// is_def_line). It is a cheap guard for convert_deflists: a body with no +// definition line cannot contain a definition list. There are no false +// negatives (every definition list requires a definition line); a false +// positive — e.g. a colon line that only appears inside a code fence — simply +// falls through to the full scan, which then finds no match and returns `body`. +has_def_line :: proc(body: string) -> bool { + pos := 0 + for { + line, next := line_at(body, pos) or_break + if is_def_line(line) { + return true + } + pos = next + } + return false +} + +// write_deflist_group renders the definition-list group beginning at `start` +// directly into `sb`, wrapped in
...
, and returns the byte offset one +// past the group (the start of the first line not consumed). The caller must +// have confirmed via term_has_def that a group starts at `start`, so the group +// always contains at least one term+definition pair. A trailing blank line that +// separates the group from following content is left unconsumed. +write_deflist_group :: proc(sb: ^strings.Builder, body: string, start: int) -> (end: int) { + strings.write_string(sb, "
") + + pos := start + for { + term_line, term_next, term_ok := line_at(body, pos) + if !term_ok || is_blank_line(term_line) || is_def_line(term_line) { break } - // Look for a def line: immediately after or with one blank line - def_idx := i + 1 - if def_idx < len(lines) && is_blank_line(lines[def_idx]) { - def_idx += 1 + // The definition follows immediately or across a single blank line. + def_pos := term_next + def_line, def_next, def_ok := line_at(body, def_pos) + if def_ok && is_blank_line(def_line) { + def_pos = def_next + def_line, def_next, def_ok = line_at(body, def_pos) } - - if def_idx >= len(lines) || !is_def_line(lines[def_idx]) { + if !def_ok || !is_def_line(def_line) { break } - // Found a term + def pair - append( - &entries, - DefList_Entry { - term = strings.trim_space(lines[i]), - definition = def_content(lines[def_idx]), - }, - ) - i = def_idx + 1 + // Emit the term with its first definition, then any consecutive + // definition lines as additional
s under the same term. + strings.write_string(sb, "
") + strings.write_string(sb, render_inline_md(strings.trim_space(term_line))) + strings.write_string(sb, "
") + strings.write_string(sb, render_inline_md(def_content(def_line))) + strings.write_string(sb, "
") + pos = def_next + for { + cont, cont_next, cont_ok := line_at(body, pos) + if !cont_ok || !is_def_line(cont) { + break + } + strings.write_string(sb, "
") + strings.write_string(sb, render_inline_md(def_content(cont))) + strings.write_string(sb, "
") + pos = cont_next + } - // After a pair, check if another pair follows (optionally - // separated by one blank line). If so, continue the group. - // If not, break without consuming the blank line. - if i < len(lines) && is_blank_line(lines[i]) { - after_blank := i + 1 - if after_blank < len(lines) && - !is_blank_line(lines[after_blank]) && - !is_def_line(lines[after_blank]) { - // Check whether a def follows this potential term - check_def := after_blank + 1 - if check_def < len(lines) && is_blank_line(lines[check_def]) { - check_def += 1 - } - if check_def < len(lines) && is_def_line(lines[check_def]) { - i = after_blank - continue - } + // Continue the group if another term+definition pair follows, + // optionally separated by one blank line. Otherwise stop here without + // consuming a trailing blank line. + blank_line, after_blank, blank_ok := line_at(body, pos) + if blank_ok && is_blank_line(blank_line) { + if term_has_def(body, after_blank) { + pos = after_blank + continue } break } + // No blank line: loop and let the top re-test this line as a term. } - if len(entries) > 0 { - ok = true - end = i - } - return + strings.write_string(sb, "
") + return pos } -// is_def_line returns true if the line is a definition line: -// optional leading whitespace, a colon, then whitespace or end-of-line. -is_def_line :: proc(line: string) -> bool { - trimmed := strings.trim_left(line, " \t") - if len(trimmed) < 1 || trimmed[0] != ':' { +// term_has_def reports whether the line at `pos` is a term that is immediately +// (or across one blank line) followed by a definition line. +term_has_def :: proc(body: string, pos: int) -> bool { + line, next, ok := line_at(body, pos) + if !ok || is_blank_line(line) || is_def_line(line) { return false } - if len(trimmed) == 1 { + def_pos := next + def_line, def_next, def_ok := line_at(body, def_pos) + if def_ok && is_blank_line(def_line) { + def_pos = def_next + def_line, def_next, def_ok = line_at(body, def_pos) + } + return def_ok && is_def_line(def_line) +} + +// fence_open reports whether `line` opens a fenced code block and, if so, which +// marker character it uses. Leading spaces are ignored. +fence_open :: proc(line: string) -> (marker: u8, ok: bool) { + trimmed := strings.trim_left(line, " ") + if strings.has_prefix(trimmed, "```") { + return '`', true + } + if strings.has_prefix(trimmed, "~~~") { + return '~', true + } + return 0, false +} + +// is_fence_line reports whether `line` is a fence using the given marker +// character (used to detect the closing fence of an open block). +is_fence_line :: proc(line: string, marker: u8) -> bool { + trimmed := strings.trim_left(line, " ") + if marker == '~' { + return strings.has_prefix(trimmed, "~~~") + } + return strings.has_prefix(trimmed, "```") +} + +// is_def_line returns true if the line is a definition line: fewer than four +// columns of leading whitespace, a colon, then whitespace or end-of-line. Four +// or more columns of indentation is a code block, not a definition. +is_def_line :: proc(line: string) -> bool { + indent := 0 + n := 0 + for n < len(line) && (line[n] == ' ' || line[n] == '\t') { + indent += line[n] == '\t' ? 4 : 1 + n += 1 + } + if indent >= 4 { + return false + } + rest := line[n:] + if len(rest) < 1 || rest[0] != ':' { + return false + } + if len(rest) == 1 { return true } - return trimmed[1] == ' ' || trimmed[1] == '\t' + return rest[1] == ' ' || rest[1] == '\t' } // def_content extracts the definition text from a definition line, @@ -154,27 +289,6 @@ is_blank_line :: proc(line: string) -> bool { return strings.trim_space(line) == "" } -// render_deflist builds the
HTML block from a list of entries. -// Each term and definition is rendered through cmark to process inline -// markdown. Result lives in context.temp_allocator. -render_deflist :: proc(entries: [dynamic]DefList_Entry) -> string { - sb := strings.builder_make(context.temp_allocator) - - strings.write_string(&sb, "
") - for entry in entries { - term_html := render_inline_md(entry.term) - def_html := render_inline_md(entry.definition) - strings.write_string(&sb, "
") - strings.write_string(&sb, term_html) - strings.write_string(&sb, "
") - strings.write_string(&sb, def_html) - strings.write_string(&sb, "
") - } - strings.write_string(&sb, "
") - - return strings.to_string(sb) -} - // render_inline_md renders a snippet of markdown through cmark and strips // the surrounding

tags. Result lives in context.temp_allocator. render_inline_md :: proc(text: string) -> string { diff --git a/markdown/deflists_fixtures_test.odin b/markdown/deflists_fixtures_test.odin new file mode 100644 index 0000000..fc62f88 --- /dev/null +++ b/markdown/deflists_fixtures_test.odin @@ -0,0 +1,92 @@ +#+test +package markdown + +import "core:testing" + +// Fixture-driven tests for convert_deflists. Every fixture is exact-matched +// against its oracle in testdata/deflists/expected/. See +// testdata/deflists/reference/README.md for how each expected file was decided +// (cross-checked against Pandoc and Hugo/Goldmark). +// +// The fixture inputs are the definition-list examples from +// https://www.markdownlang.com/extended/definition-lists.html +// +// Provenance of each expected file: +// 01,02,08,09 Pandoc and Hugo agree; oracle = that agreed HTML. +// 06 Both agree it is NOT a deflist; convert_deflists passes the +// markdown through unchanged, so expected = the raw input. +// 07 CommonMark's <4-space marker rule: we accept the 2-space colon +// (Pandoc does; Hugo does not). +// 05 Block nesting is unsupported; our flatten happens to match +// Pandoc's output exactly. +// 03,04 Block content in definitions is unsupported; expected captures +// our current degraded behavior (a

for the term/first line, +// the rest falling through as markdown) as a regression guard. +Deflist_Fixture :: struct { + name: string, + input: string, + expected: string, +} + +@(test) +test_deflist_fixtures :: proc(t: ^testing.T) { + cases := []Deflist_Fixture { + { + "01-basic", + #load(#directory + "testdata/deflists/01-basic.md", string), + #load(#directory + "testdata/deflists/expected/01-basic.html", string), + }, + { + "02-multiple-defs", + #load(#directory + "testdata/deflists/02-multiple-defs.md", string), + #load(#directory + "testdata/deflists/expected/02-multiple-defs.html", string), + }, + { + "03-multiline-def", + #load(#directory + "testdata/deflists/03-multiline-def.md", string), + #load(#directory + "testdata/deflists/expected/03-multiline-def.html", string), + }, + { + "04-markdown-in-def", + #load(#directory + "testdata/deflists/04-markdown-in-def.md", string), + #load(#directory + "testdata/deflists/expected/04-markdown-in-def.html", string), + }, + { + "05-nested", + #load(#directory + "testdata/deflists/05-nested.md", string), + #load(#directory + "testdata/deflists/expected/05-nested.html", string), + }, + { + "06-compact", + #load(#directory + "testdata/deflists/06-compact.md", string), + #load(#directory + "testdata/deflists/expected/06-compact.html", string), + }, + { + "07-extra-space", + #load(#directory + "testdata/deflists/07-extra-space.md", string), + #load(#directory + "testdata/deflists/expected/07-extra-space.html", string), + }, + { + "08-glossary", + #load(#directory + "testdata/deflists/08-glossary.md", string), + #load(#directory + "testdata/deflists/expected/08-glossary.html", string), + }, + { + "09-api-params", + #load(#directory + "testdata/deflists/09-api-params.md", string), + #load(#directory + "testdata/deflists/expected/09-api-params.html", string), + }, + } + + for c in cases { + result := convert_deflists(c.input, context.temp_allocator) + testing.expectf( + t, + result == c.expected, + "[%s]\n expected: %q\n got: %q", + c.name, + c.expected, + result, + ) + } +} diff --git a/markdown/testdata/deflists/01-basic.md b/markdown/testdata/deflists/01-basic.md new file mode 100644 index 0000000..d69adf9 --- /dev/null +++ b/markdown/testdata/deflists/01-basic.md @@ -0,0 +1,2 @@ +Term +: Definition content diff --git a/markdown/testdata/deflists/02-multiple-defs.md b/markdown/testdata/deflists/02-multiple-defs.md new file mode 100644 index 0000000..626efa5 --- /dev/null +++ b/markdown/testdata/deflists/02-multiple-defs.md @@ -0,0 +1,7 @@ +Markdown +: A lightweight markup language +: Created by John Gruber in 2004 + +HTML +: A standard markup language used to create web pages +: Used to create web pages diff --git a/markdown/testdata/deflists/03-multiline-def.md b/markdown/testdata/deflists/03-multiline-def.md new file mode 100644 index 0000000..5c7f287 --- /dev/null +++ b/markdown/testdata/deflists/03-multiline-def.md @@ -0,0 +1,5 @@ +Term +: This is the first line of definition content + This is the second line, needs at least one space indentation + + This is a new paragraph, needs at least one space indentation and a preceding blank line diff --git a/markdown/testdata/deflists/04-markdown-in-def.md b/markdown/testdata/deflists/04-markdown-in-def.md new file mode 100644 index 0000000..ace64a5 --- /dev/null +++ b/markdown/testdata/deflists/04-markdown-in-def.md @@ -0,0 +1,7 @@ +Markdown Syntax +: **Markdown** supports multiple text formatting: + - *Italic* and **Bold** + - [Link](https://www.markdownlang.com) + - `Inline code` + - > Block quote + - Lists and other elements diff --git a/markdown/testdata/deflists/05-nested.md b/markdown/testdata/deflists/05-nested.md new file mode 100644 index 0000000..b1b294e --- /dev/null +++ b/markdown/testdata/deflists/05-nested.md @@ -0,0 +1,6 @@ +Outer Term +: Outer Definition + + Inner Term + : Inner Definition + : Another Inner Definition diff --git a/markdown/testdata/deflists/06-compact.md b/markdown/testdata/deflists/06-compact.md new file mode 100644 index 0000000..f9da622 --- /dev/null +++ b/markdown/testdata/deflists/06-compact.md @@ -0,0 +1 @@ +Term: Definition diff --git a/markdown/testdata/deflists/07-extra-space.md b/markdown/testdata/deflists/07-extra-space.md new file mode 100644 index 0000000..70f2393 --- /dev/null +++ b/markdown/testdata/deflists/07-extra-space.md @@ -0,0 +1,2 @@ +Term + : Definition diff --git a/markdown/testdata/deflists/08-glossary.md b/markdown/testdata/deflists/08-glossary.md new file mode 100644 index 0000000..8be7017 --- /dev/null +++ b/markdown/testdata/deflists/08-glossary.md @@ -0,0 +1,11 @@ +API +: An application programming interface, a set of rules that allow different software applications to communicate with each other. + +Framework +: A software library that provides a standardized structure for application development. + +Git +: A distributed version control system used to track changes in a project development process. + +IDE +: An integrated development environment, a software application that includes a code editor and various development tools. diff --git a/markdown/testdata/deflists/09-api-params.md b/markdown/testdata/deflists/09-api-params.md new file mode 100644 index 0000000..9ac0947 --- /dev/null +++ b/markdown/testdata/deflists/09-api-params.md @@ -0,0 +1,14 @@ +user_id +: **Required** - The unique identifier for the user. +: Type: `integer` + +name +: **Optional** - The display name for the user. +: Type: `string` +: Default: `null` + +status +: **Optional** - The user status. +: Type: `string` +: Allowed values: `active`, `inactive`, `suspended` +: Default: `active` diff --git a/markdown/testdata/deflists/expected/01-basic.html b/markdown/testdata/deflists/expected/01-basic.html new file mode 100644 index 0000000..239e94a --- /dev/null +++ b/markdown/testdata/deflists/expected/01-basic.html @@ -0,0 +1 @@ +
Term
Definition content
\ No newline at end of file diff --git a/markdown/testdata/deflists/expected/02-multiple-defs.html b/markdown/testdata/deflists/expected/02-multiple-defs.html new file mode 100644 index 0000000..0de2d72 --- /dev/null +++ b/markdown/testdata/deflists/expected/02-multiple-defs.html @@ -0,0 +1 @@ +
Markdown
A lightweight markup language
Created by John Gruber in 2004
HTML
A standard markup language used to create web pages
Used to create web pages
\ No newline at end of file diff --git a/markdown/testdata/deflists/expected/03-multiline-def.html b/markdown/testdata/deflists/expected/03-multiline-def.html new file mode 100644 index 0000000..71bd233 --- /dev/null +++ b/markdown/testdata/deflists/expected/03-multiline-def.html @@ -0,0 +1,3 @@ +
Term
This is the first line of definition content
This is the second line, needs at least one space indentation + + This is a new paragraph, needs at least one space indentation and a preceding blank line diff --git a/markdown/testdata/deflists/expected/04-markdown-in-def.html b/markdown/testdata/deflists/expected/04-markdown-in-def.html new file mode 100644 index 0000000..825e1e4 --- /dev/null +++ b/markdown/testdata/deflists/expected/04-markdown-in-def.html @@ -0,0 +1,5 @@ +
Markdown Syntax
Markdown supports multiple text formatting:
- *Italic* and **Bold** + - [Link](https://www.markdownlang.com) + - `Inline code` + - > Block quote + - Lists and other elements diff --git a/markdown/testdata/deflists/expected/05-nested.html b/markdown/testdata/deflists/expected/05-nested.html new file mode 100644 index 0000000..e64ec82 --- /dev/null +++ b/markdown/testdata/deflists/expected/05-nested.html @@ -0,0 +1 @@ +
Outer Term
Outer Definition
Inner Term
Inner Definition
Another Inner Definition
\ No newline at end of file diff --git a/markdown/testdata/deflists/expected/06-compact.html b/markdown/testdata/deflists/expected/06-compact.html new file mode 100644 index 0000000..f9da622 --- /dev/null +++ b/markdown/testdata/deflists/expected/06-compact.html @@ -0,0 +1 @@ +Term: Definition diff --git a/markdown/testdata/deflists/expected/07-extra-space.html b/markdown/testdata/deflists/expected/07-extra-space.html new file mode 100644 index 0000000..0107828 --- /dev/null +++ b/markdown/testdata/deflists/expected/07-extra-space.html @@ -0,0 +1 @@ +
Term
Definition
\ No newline at end of file diff --git a/markdown/testdata/deflists/expected/08-glossary.html b/markdown/testdata/deflists/expected/08-glossary.html new file mode 100644 index 0000000..e9ca036 --- /dev/null +++ b/markdown/testdata/deflists/expected/08-glossary.html @@ -0,0 +1 @@ +
API
An application programming interface, a set of rules that allow different software applications to communicate with each other.
Framework
A software library that provides a standardized structure for application development.
Git
A distributed version control system used to track changes in a project development process.
IDE
An integrated development environment, a software application that includes a code editor and various development tools.
\ No newline at end of file diff --git a/markdown/testdata/deflists/expected/09-api-params.html b/markdown/testdata/deflists/expected/09-api-params.html new file mode 100644 index 0000000..27b62c3 --- /dev/null +++ b/markdown/testdata/deflists/expected/09-api-params.html @@ -0,0 +1 @@ +
user_id
Required - The unique identifier for the user.
Type: integer
name
Optional - The display name for the user.
Type: string
Default: null
status
Optional - The user status.
Type: string
Allowed values: active, inactive, suspended
Default: active
\ No newline at end of file diff --git a/markdown/testdata/deflists/reference/COMPARISON.md b/markdown/testdata/deflists/reference/COMPARISON.md new file mode 100644 index 0000000..35eca1f --- /dev/null +++ b/markdown/testdata/deflists/reference/COMPARISON.md @@ -0,0 +1,60 @@ +# 3-way behavior comparison: ours vs Pandoc vs Hugo + +Captured for later review (e.g. if we ever pursue "Option B" — full CommonMark +block semantics inside definitions). Compares `convert_deflists` output (Option A, +inline/flat) against Pandoc (`-f markdown -t html5`) and Hugo/Goldmark for the +four fixtures where behavior is interesting. Raw renderings live in `pandoc/` and +`hugo/`; ours is in `../expected/`. Whitespace/formatting differences ignored +below — only structural differences are described. + +## 03-multiline-def — definition with a continuation line + a second paragraph +*How much multi-line content belongs to the `
`?* +- **Ours:** only the **first line** goes in the `
`. The continuation line and + the second paragraph fall out of the `
` as **raw, unrendered markdown**. +- **Pandoc:** folds the continuation line **into** the `
` (lazy continuation), + then **ejects** the second paragraph as a sibling `

` after `

`. +- **Hugo:** keeps **everything inside** the `
` — continuation line as text, + second paragraph as a nested `

`. +- All three differ. We capture the least, Pandoc the middle, Hugo the most. + +## 04-markdown-in-def — definition containing a bullet list (with a blockquote) +*What happens to the nested list?* +- **Ours:** intro line becomes the `

`; the **entire bullet list falls out as + raw markdown** (unrendered `- …` lines after `
`). +- **Pandoc:** bullets **flattened into the `
`** as inline text — no `
    `, the + `-` become literal dashes. +- **Hugo:** bullets become a **real nested `
      `/`
    • `** (blockquote and all) + inside the `
      `. +- All three differ; same shape as 03. We drop it, Pandoc absorbs-but-flattens, + Hugo builds true structure. + +## 05-nested — a definition list nested under another definition +*Nest or flatten?* +- **Ours:** **flatten** — one flat `
      `; "Inner Term" is just another `
      `. +- **Pandoc:** **flatten** — structurally **identical to ours**. +- **Hugo:** **nests** a real inner `
      ` inside the outer `
      `. +- Ours == Pandoc; Hugo is the odd one out. + +## 07-extra-space — term then a 2-space-indented `: Definition` +*Is a 2-space colon a definition marker?* +- **Ours:** **yes** → clean `
      `. +- **Pandoc:** **yes** → structurally **identical to ours**. +- **Hugo:** **no** → rejects it, emits a plain `

      `. +- Ours == Pandoc; Hugo is the odd one out. + +## Pattern + +Two clean groups: + +- **Marker / flattening decisions (05, 07):** we land **exactly on Pandoc**. These + are the calls our CommonMark-marker + flat-inline design makes cleanly; Pandoc + agrees, Hugo diverges (nests / stricter marker). +- **Block-content-in-a-definition (03, 04):** we diverge from **both** tools, doing + **less** — we emit the `

      ` for the term/first line and let the rest fall + through as raw markdown, rather than absorbing it into the `
      ` (flattened like + Pandoc, or structured like Hugo). + +In short: our behavior is **"Pandoc, minus the ability to pull block content into +a `
      `."** Single inline-line definitions (01, 02, 05, 07, 08, 09) match a real +engine; genuinely multi-block definitions (03, 04) stop at the first line and pass +the remainder through. Closing that gap is what Option B would entail. diff --git a/markdown/testdata/deflists/reference/README.md b/markdown/testdata/deflists/reference/README.md new file mode 100644 index 0000000..ed6a26b --- /dev/null +++ b/markdown/testdata/deflists/reference/README.md @@ -0,0 +1,44 @@ +# Definition-list test fixtures + +Inputs (`../NN-name.md`) are the definition-list examples from +. + +`pandoc/` and `hugo/` hold each input rendered by Pandoc (`-f markdown -t html5`) +and Hugo (Goldmark with `markup.goldmark.extensions.definitionList = true`). +They are retained as reference for a possible future "Option B" (full CommonMark +block semantics inside definitions). They are NOT the test oracle for the current +implementation. + +`../expected/NN.html` is the oracle for `convert_deflists`: our inline fragment +format (no inter-tag newlines, no trailing newline), derived from the trusted +tool per the decisions below. + +## Decisions (Option A — simple / inline) + +Marker whitespace follows CommonMark: `:` with 0–3 spaces of indent is a +definition; 4+ (tab = 4) is a code block. Definition bodies are inline/flat. + +Every fixture has a concrete exact-match oracle in `../expected/NN.html`. + +| Fixture | Pandoc | Hugo | Decision | expected/ source | +|---------|--------|------|----------|------------------| +| 01-basic | `
      ` | `
      ` | support | hugo (agreed) | +| 02-multiple-defs | multi `
      ` | multi `
      ` | support | hugo (agreed) | +| 06-compact (`Term: Definition`) | `

      ` | `

      ` | NOT a deflist → passthrough | raw input | +| 07-extra-space (2-space `:`) | `

      ` | `

      ` | support (accept, CommonMark ≤3-space rule) | hand-written | +| 08-glossary | `

      ` | `
      ` | support | hugo (agreed) | +| 09-api-params | multi `
      ` | multi `
      ` | support | hugo (agreed) | +| 05-nested | flattens | nested `
      ` | flatten (matches Pandoc) | snapshot (== pandoc) | +| 03-multiline-def | ejects 2nd para | nests 2nd para in `
      ` | degrade | snapshot (current output) | +| 04-markdown-in-def | flattens list | nested `
        `/`
        ` | degrade | snapshot (current output) | + +A detailed 3-way behavior comparison (ours vs Pandoc vs Hugo) for the diverging +fixtures is in `COMPARISON.md`, kept for Option-B review. + +03/04/05 require block-level content inside `
        ` (multi-paragraph, nested +lists/blockquotes, nested `
        `), which the inline preprocessor does not +produce. 05 flattens exactly as Pandoc does, so its snapshot equals Pandoc's +output. 03/04 emit a `
        ` for the term and first line, then let the remaining +block content fall through as markdown; their `expected/` files snapshot that +current degraded behavior as a regression guard, not an endorsed rendering. +Revisit via Option B (full CommonMark block semantics) if real content needs it. diff --git a/markdown/testdata/deflists/reference/hugo/01-basic.html b/markdown/testdata/deflists/reference/hugo/01-basic.html new file mode 100644 index 0000000..e24961e --- /dev/null +++ b/markdown/testdata/deflists/reference/hugo/01-basic.html @@ -0,0 +1,4 @@ +
        +
        Term
        +
        Definition content
        +
        diff --git a/markdown/testdata/deflists/reference/hugo/02-multiple-defs.html b/markdown/testdata/deflists/reference/hugo/02-multiple-defs.html new file mode 100644 index 0000000..25956bb --- /dev/null +++ b/markdown/testdata/deflists/reference/hugo/02-multiple-defs.html @@ -0,0 +1,8 @@ +
        +
        Markdown
        +
        A lightweight markup language
        +
        Created by John Gruber in 2004
        +
        HTML
        +
        A standard markup language used to create web pages
        +
        Used to create web pages
        +
        diff --git a/markdown/testdata/deflists/reference/hugo/03-multiline-def.html b/markdown/testdata/deflists/reference/hugo/03-multiline-def.html new file mode 100644 index 0000000..76c591a --- /dev/null +++ b/markdown/testdata/deflists/reference/hugo/03-multiline-def.html @@ -0,0 +1,7 @@ +
        +
        Term
        +
        This is the first line of definition content +This is the second line, needs at least one space indentation +

        This is a new paragraph, needs at least one space indentation and a preceding blank line

        +
        +
        diff --git a/markdown/testdata/deflists/reference/hugo/04-markdown-in-def.html b/markdown/testdata/deflists/reference/hugo/04-markdown-in-def.html new file mode 100644 index 0000000..b4024b2 --- /dev/null +++ b/markdown/testdata/deflists/reference/hugo/04-markdown-in-def.html @@ -0,0 +1,16 @@ +
        +
        Markdown Syntax
        +
        Markdown supports multiple text formatting: +
          +
        • Italic and Bold
        • +
        • Link
        • +
        • Inline code
        • +
        • +
          +

          Block quote

          +
          +
        • +
        • Lists and other elements
        • +
        +
        +
        diff --git a/markdown/testdata/deflists/reference/hugo/05-nested.html b/markdown/testdata/deflists/reference/hugo/05-nested.html new file mode 100644 index 0000000..b5e9884 --- /dev/null +++ b/markdown/testdata/deflists/reference/hugo/05-nested.html @@ -0,0 +1,10 @@ +
        +
        Outer Term
        +
        Outer Definition +
        +
        Inner Term
        +
        Inner Definition
        +
        Another Inner Definition
        +
        +
        +
        diff --git a/markdown/testdata/deflists/reference/hugo/06-compact.html b/markdown/testdata/deflists/reference/hugo/06-compact.html new file mode 100644 index 0000000..a6e1349 --- /dev/null +++ b/markdown/testdata/deflists/reference/hugo/06-compact.html @@ -0,0 +1 @@ +

        Term: Definition

        diff --git a/markdown/testdata/deflists/reference/hugo/07-extra-space.html b/markdown/testdata/deflists/reference/hugo/07-extra-space.html new file mode 100644 index 0000000..e02ac3c --- /dev/null +++ b/markdown/testdata/deflists/reference/hugo/07-extra-space.html @@ -0,0 +1,2 @@ +

        Term +: Definition

        diff --git a/markdown/testdata/deflists/reference/hugo/08-glossary.html b/markdown/testdata/deflists/reference/hugo/08-glossary.html new file mode 100644 index 0000000..cc91f89 --- /dev/null +++ b/markdown/testdata/deflists/reference/hugo/08-glossary.html @@ -0,0 +1,10 @@ +
        +
        API
        +
        An application programming interface, a set of rules that allow different software applications to communicate with each other.
        +
        Framework
        +
        A software library that provides a standardized structure for application development.
        +
        Git
        +
        A distributed version control system used to track changes in a project development process.
        +
        IDE
        +
        An integrated development environment, a software application that includes a code editor and various development tools.
        +
        diff --git a/markdown/testdata/deflists/reference/hugo/09-api-params.html b/markdown/testdata/deflists/reference/hugo/09-api-params.html new file mode 100644 index 0000000..e1792c4 --- /dev/null +++ b/markdown/testdata/deflists/reference/hugo/09-api-params.html @@ -0,0 +1,14 @@ +
        +
        user_id
        +
        Required - The unique identifier for the user.
        +
        Type: integer
        +
        name
        +
        Optional - The display name for the user.
        +
        Type: string
        +
        Default: null
        +
        status
        +
        Optional - The user status.
        +
        Type: string
        +
        Allowed values: active, inactive, suspended
        +
        Default: active
        +
        diff --git a/markdown/testdata/deflists/reference/pandoc/01-basic.html b/markdown/testdata/deflists/reference/pandoc/01-basic.html new file mode 100644 index 0000000..a62b274 --- /dev/null +++ b/markdown/testdata/deflists/reference/pandoc/01-basic.html @@ -0,0 +1,6 @@ +
        +
        Term
        +
        +Definition content +
        +
        diff --git a/markdown/testdata/deflists/reference/pandoc/02-multiple-defs.html b/markdown/testdata/deflists/reference/pandoc/02-multiple-defs.html new file mode 100644 index 0000000..7f0d67f --- /dev/null +++ b/markdown/testdata/deflists/reference/pandoc/02-multiple-defs.html @@ -0,0 +1,16 @@ +
        +
        Markdown
        +
        +A lightweight markup language +
        +
        +Created by John Gruber in 2004 +
        +
        HTML
        +
        +A standard markup language used to create web pages +
        +
        +Used to create web pages +
        +
        diff --git a/markdown/testdata/deflists/reference/pandoc/03-multiline-def.html b/markdown/testdata/deflists/reference/pandoc/03-multiline-def.html new file mode 100644 index 0000000..cdc0d6b --- /dev/null +++ b/markdown/testdata/deflists/reference/pandoc/03-multiline-def.html @@ -0,0 +1,9 @@ +
        +
        Term
        +
        +This is the first line of definition content This is the second line, +needs at least one space indentation +
        +
        +

        This is a new paragraph, needs at least one space indentation and a +preceding blank line

        diff --git a/markdown/testdata/deflists/reference/pandoc/04-markdown-in-def.html b/markdown/testdata/deflists/reference/pandoc/04-markdown-in-def.html new file mode 100644 index 0000000..8e566cc --- /dev/null +++ b/markdown/testdata/deflists/reference/pandoc/04-markdown-in-def.html @@ -0,0 +1,9 @@ +
        +
        Markdown Syntax
        +
        +Markdown supports multiple text formatting: - +Italic and Bold - Link - Inline code +- > Block quote - Lists and other elements +
        +
        diff --git a/markdown/testdata/deflists/reference/pandoc/05-nested.html b/markdown/testdata/deflists/reference/pandoc/05-nested.html new file mode 100644 index 0000000..5434637 --- /dev/null +++ b/markdown/testdata/deflists/reference/pandoc/05-nested.html @@ -0,0 +1,13 @@ +
        +
        Outer Term
        +
        +Outer Definition +
        +
        Inner Term
        +
        +Inner Definition +
        +
        +Another Inner Definition +
        +
        diff --git a/markdown/testdata/deflists/reference/pandoc/06-compact.html b/markdown/testdata/deflists/reference/pandoc/06-compact.html new file mode 100644 index 0000000..a6e1349 --- /dev/null +++ b/markdown/testdata/deflists/reference/pandoc/06-compact.html @@ -0,0 +1 @@ +

        Term: Definition

        diff --git a/markdown/testdata/deflists/reference/pandoc/07-extra-space.html b/markdown/testdata/deflists/reference/pandoc/07-extra-space.html new file mode 100644 index 0000000..4d8c3fd --- /dev/null +++ b/markdown/testdata/deflists/reference/pandoc/07-extra-space.html @@ -0,0 +1,6 @@ +
        +
        Term
        +
        +Definition +
        +
        diff --git a/markdown/testdata/deflists/reference/pandoc/08-glossary.html b/markdown/testdata/deflists/reference/pandoc/08-glossary.html new file mode 100644 index 0000000..9e1f07a --- /dev/null +++ b/markdown/testdata/deflists/reference/pandoc/08-glossary.html @@ -0,0 +1,22 @@ +
        +
        API
        +
        +An application programming interface, a set of rules that allow +different software applications to communicate with each other. +
        +
        Framework
        +
        +A software library that provides a standardized structure for +application development. +
        +
        Git
        +
        +A distributed version control system used to track changes in a project +development process. +
        +
        IDE
        +
        +An integrated development environment, a software application that +includes a code editor and various development tools. +
        +
        diff --git a/markdown/testdata/deflists/reference/pandoc/09-api-params.html b/markdown/testdata/deflists/reference/pandoc/09-api-params.html new file mode 100644 index 0000000..8e6156f --- /dev/null +++ b/markdown/testdata/deflists/reference/pandoc/09-api-params.html @@ -0,0 +1,33 @@ +
        +
        user_id
        +
        +Required - The unique identifier for the user. +
        +
        +Type: integer +
        +
        name
        +
        +Optional - The display name for the user. +
        +
        +Type: string +
        +
        +Default: null +
        +
        status
        +
        +Optional - The user status. +
        +
        +Type: string +
        +
        +Allowed values: active, inactive, +suspended +
        +
        +Default: active +
        +