diff --git a/AGENTS.md b/AGENTS.md index e664e7c..17e58c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,7 +27,7 @@ thor/ ├── feed.odin # RSS + sitemap generation ├── vfs.odin # Union file system (defaults → modules → site) ├── assets.odin # VFS-based asset copying -├── html.odin # HTML helpers: strip_html_tags, unescape_html, generate_summary +├── html.odin # HTML helpers: strip_html_tags, unescape_html, generate_summary (word-count truncation), generate_description (scrub to plain text) ├── opengraph.odin # Open_Graph struct + og_for_site/og_for_page ├── frontmatter.odin # JSON frontmatter parser (supports nested og + lastmod) ├── defaults.odin # DEFAULTS_PATH constant (#directory) @@ -48,7 +48,7 @@ thor/ | `feed.odin` | RSS feed + sitemap XML. Uses `page.url` for canonical URLs. | | `vfs.odin` | Union file system: `VFS`, `build_vfs`, `mount_dir`, `mount_subdir`, `mount_recursive`, `vfs_get`, `vfs_get_entry`, `vfs_entry_data`. Layers defaults → modules → site. | | `assets.odin` | `copy_assets_dir` — iterates VFS entries with `assets/` prefix, minifies CSS, copies verbatim or via `os.copy_file`. | -| `html.odin` | `strip_html_tags` (moved from render.odin), `unescape_html`, `generate_summary` (Hugo-style body summary for OG descriptions). | +| `html.odin` | `strip_html_tags`, `unescape_html`, `generate_summary` (word-count truncation, zero-alloc), `generate_description` (HTML→plain text: strip tags, decode entities, collapse whitespace). | | `opengraph.odin` | `Open_Graph` struct (fields ordered per OGP spec, `is_article: Maybe(bool)`). `og_for_site(site)` for site defaults (from config + derived), `og_for_page(site_og, page)` for page-specific (overlay page.og + derive from page data). | | `frontmatter.odin` | JSON frontmatter parser (`{ }` delimited). Supports `layout`, `lastmod`, and nested `og` object (via `json_get_open_graph`). | | `defaults.odin` | `DEFAULTS_PATH` constant, resolved at compile time via `#directory` so bundled templates ship in the binary. | @@ -187,7 +187,7 @@ Content is **not yet in the VFS** — `scan_content` still uses direct filesyste - `is_article ← !page._is_index` - `section ← page.section` - `published_time / modified_time ← page.date / page.lastmod` -- `description ← page.description`, else body summary (via `generate_summary`) +- `description ← page.description`, else `generate_description(generate_summary(body_html))` (scrubbed plain text) Paths through maps (e.g. `params.*`) are silently allowed — not validated. Templates access via `{{og.url}}`, `{{og.title}}`, `{{#og.is_article}}`, etc. diff --git a/TODOS.md b/TODOS.md index 3af834c..c03eb2e 100644 --- a/TODOS.md +++ b/TODOS.md @@ -21,6 +21,8 @@ - [ ] return `src: cstring` from `load_query`. - [ ] Improve `unescape_html` with simd. - [ ] generate summary before syntax highlighting. +- [ ] generate summary before markdown to html conversion. +- [ ] mount_recursive is pretty significant ## Memory Management diff --git a/html.odin b/html.odin index 38b0f0d..38e6712 100644 --- a/html.odin +++ b/html.odin @@ -29,7 +29,7 @@ strip_html_tags :: proc(s: string, allocator := context.allocator) -> string { } unescape_html :: proc(s: string) -> string { - sb := strings.builder_make_len(len(s)) + sb := strings.builder_make_len_cap(0, len(s)) defer strings.builder_destroy(&sb) start := 0 @@ -72,73 +72,146 @@ unescape_html :: proc(s: string) -> string { return strings.to_string(sb) } -// generate_summary produces a plain-text summary of an HTML fragment. -// Blocks (paragraphs, headings, list items) are extracted, their tags -// stripped, entities decoded, and accumulated word-by-word until the -// max_words threshold is crossed — at which point the rest of the -// current block is included before stopping. Mirrors Hugo's default -// summary behavior. +// generate_summary truncates an HTML string to the first max_words words. +// Walks forward counting whitespace→text transitions, skipping tag interiors +// so spaces inside attributes don't count. Returns a substring of the +// original — zero allocation. Mirrors Hugo's default (70 words). generate_summary :: proc(html: string, max_words: int = 70) -> string { - separated, _ := strings.replace_all(html, "

", "\n\n", context.temp_allocator) - separated, _ = strings.replace_all(separated, "", "\n\n") - separated, _ = strings.replace_all(separated, "", "\n\n") - separated, _ = strings.replace_all(separated, "", "\n\n") - separated, _ = strings.replace_all(separated, "", "\n\n") - separated, _ = strings.replace_all(separated, "", "\n\n") - separated, _ = strings.replace_all(separated, "", "\n\n") - separated, _ = strings.replace_all(separated, "", "\n\n") - separated, _ = strings.replace_all(separated, "", "\n\n") - - stripped := strip_html_tags(separated, context.temp_allocator) - plain := unescape_html(stripped) - - blocks := strings.split(plain, "\n\n", allocator = context.temp_allocator) - defer delete(blocks) - - sb := strings.builder_make(context.temp_allocator) - defer strings.builder_destroy(&sb) - + if max_words <= 0 { + return "" + } word_count := 0 - first := true - for raw_block in blocks { - block := strings.trim_space(raw_block) - if len(block) == 0 { + in_word := false + in_tag := false + for i in 0 ..< len(html) { + c := html[i] + if in_tag { + if c == '>' { + in_tag = false + } continue } - - // Collapse internal whitespace to single spaces. - block_sb := strings.builder_make(context.temp_allocator) - has_content := false - in_space := true - for c in block { - if c == ' ' || c == '\t' || c == '\n' || c == '\r' { - in_space = true - } else { - if in_space && has_content { - strings.write_byte(&block_sb, ' ') + if c == '<' { + in_tag = true + if in_word { + word_count += 1 + if word_count >= max_words { + return html[:i] } - strings.write_rune(&block_sb, c) - in_space = false - has_content = true + in_word = false } + continue } - collapsed := strings.to_string(block_sb) - words := strings.split(collapsed, " ", allocator = context.temp_allocator) - - if !first && word_count > 0 { - strings.write_byte(&sb, ' ') - } - strings.write_string(&sb, collapsed) - word_count += len(words) - first = false - - delete(words) - - if word_count >= max_words { - break + is_space := c == ' ' || c == '\n' || c == '\t' || c == '\r' + if is_space { + if in_word { + word_count += 1 + if word_count >= max_words { + return html[:i] + } + in_word = false + } + } else { + in_word = true } } - - return strings.to_string(sb) + return html } +// generate_description converts an HTML fragment to plain text by stripping +// tags, decoding entities, and collapsing whitespace. Emits a space when +// exiting any tag so block-level boundaries aren't lost. Intended for OG +// descriptions — operate on the output of generate_summary for bounded input. +generate_description :: proc(html: string, allocator := context.temp_allocator) -> string { + sb := strings.builder_make_len_cap(0, len(html), allocator) + defer strings.builder_destroy(&sb) + + in_tag := false + prev_was_space := true + run_start := 0 + i := 0 + for i < len(html) { + c := html[i] + if in_tag { + if c == '>' { + in_tag = false + if !prev_was_space { + strings.write_byte(&sb, ' ') + prev_was_space = true + } + } + i += 1 + run_start = i + continue + } + if c == '<' { + if i > run_start { + strings.write_string(&sb, html[run_start:i]) + prev_was_space = false + } + in_tag = true + i += 1 + continue + } + if c == '&' { + if i > run_start { + strings.write_string(&sb, html[run_start:i]) + prev_was_space = false + } + semi := strings.index(html[i:], ";") + if semi > 0 && semi <= 5 { + entity := html[i:i + semi + 1] + replacement := "" + switch entity { + case "&": + replacement = "&" + case "<": + replacement = "<" + case ">": + replacement = ">" + case """: + replacement = "\"" + case "'", "'": + replacement = "'" + case: + replacement = "" + } + if replacement != "" { + strings.write_string(&sb, replacement) + prev_was_space = false + i += semi + 1 + run_start = i + continue + } + } + strings.write_byte(&sb, '&') + prev_was_space = false + i += 1 + run_start = i + continue + } + if c == ' ' || c == '\n' || c == '\t' || c == '\r' { + if i > run_start { + strings.write_string(&sb, html[run_start:i]) + prev_was_space = false + } + if !prev_was_space { + strings.write_byte(&sb, ' ') + prev_was_space = true + } + i += 1 + run_start = i + continue + } + i += 1 + } + if i > run_start && !in_tag { + strings.write_string(&sb, html[run_start:i]) + } + + result := strings.to_string(sb) + if len(result) > 0 && result[len(result) - 1] == ' ' { + result = result[:len(result) - 1] + } + return result +} diff --git a/html_test.odin b/html_test.odin new file mode 100644 index 0000000..2a88e53 --- /dev/null +++ b/html_test.odin @@ -0,0 +1,107 @@ +#+test +package main + +import "core:testing" + +// --- generate_summary --- + +@(test) +test_summary_short :: proc(t: ^testing.T) { + result := generate_summary("

Hello world

") + testing.expect_value(t, result, "

Hello world

") +} + +@(test) +test_summary_word_limit :: proc(t: ^testing.T) { + result := generate_summary("

one two three four five

", max_words = 3) + testing.expect_value(t, result, "

one two three") +} + +@(test) +test_summary_empty :: proc(t: ^testing.T) { + result := generate_summary("") + testing.expect_value(t, result, "") +} + +@(test) +test_summary_no_words :: proc(t: ^testing.T) { + result := generate_summary("

") + testing.expect_value(t, result, "

") +} + +@(test) +test_summary_tags_not_counted :: proc(t: ^testing.T) { + html := `
if x
` + result := generate_summary(html, max_words = 1) + testing.expect_value(t, result, `
if`)
+}
+
+// --- generate_description ---
+
+@(test)
+test_description_simple :: proc(t: ^testing.T) {
+	result := generate_description("

Hello world

") + testing.expect_value(t, result, "Hello world") +} + +@(test) +test_description_entities :: proc(t: ^testing.T) { + result := generate_description("

Cats & dogs <3

") + testing.expect_value(t, result, "Cats & dogs <3") +} + +@(test) +test_description_nested_tags :: proc(t: ^testing.T) { + result := generate_description("

Bold text

") + testing.expect_value(t, result, "Bold text") +} + +@(test) +test_description_block_boundary :: proc(t: ^testing.T) { + result := generate_description("

First

Second

") + testing.expect_value(t, result, "First Second") +} + +@(test) +test_description_whitespace_collapse :: proc(t: ^testing.T) { + result := generate_description("

Multiple spaces

") + testing.expect_value(t, result, "Multiple spaces") +} + +@(test) +test_description_empty :: proc(t: ^testing.T) { + result := generate_description("") + testing.expect_value(t, result, "") +} + +@(test) +test_description_plain_text :: proc(t: ^testing.T) { + result := generate_description("Just plain text") + testing.expect_value(t, result, "Just plain text") +} + +@(test) +test_description_highlighted_code :: proc(t: ^testing.T) { + result := generate_description( + `
if x
`, + ) + testing.expect_value(t, result, "if x") +} + +@(test) +test_description_list_items :: proc(t: ^testing.T) { + result := generate_description("
  • One
  • Two
") + testing.expect_value(t, result, "One Two") +} + +@(test) +test_description_headings :: proc(t: ^testing.T) { + result := generate_description("

Title

Body

") + testing.expect_value(t, result, "Title Body") +} + +@(test) +test_description_blockquote :: proc(t: ^testing.T) { + result := generate_description("
Quote
") + testing.expect_value(t, result, "Quote") +} diff --git a/opengraph.odin b/opengraph.odin index 20766cd..7a88007 100644 --- a/opengraph.odin +++ b/opengraph.odin @@ -71,7 +71,7 @@ og_for_page :: proc(site_og: Open_Graph, page: Page) -> Open_Graph { description_set = true } if !description_set && is_article && page.body_html != "" { - og.description = generate_summary(page.body_html) + og.description = generate_description(generate_summary(page.body_html)) description_set = true } if !description_set {