mirror of
https://github.com/sbrow/thor.git
synced 2026-08-26 11:23:32 -04:00
feat: generate_summary and generate_description match hugo better.
This commit is contained in:
Spencer Brower
@@ -27,7 +27,7 @@ thor/
|
||||
├── feed.odin # RSS + sitemap generation
|
||||
├── vfs.odin # Union file system (defaults → modules → site)
|
||||
├── assets.odin # VFS-based asset copying
|
||||
├── html.odin # HTML helpers: strip_html_tags, unescape_html, generate_summary
|
||||
├── html.odin # HTML helpers: strip_html_tags, unescape_html, generate_summary (word-count truncation), generate_description (scrub to plain text)
|
||||
├── opengraph.odin # Open_Graph struct + og_for_site/og_for_page
|
||||
├── frontmatter.odin # JSON frontmatter parser (supports nested og + lastmod)
|
||||
├── defaults.odin # DEFAULTS_PATH constant (#directory)
|
||||
@@ -48,7 +48,7 @@ thor/
|
||||
| `feed.odin` | RSS feed + sitemap XML. Uses `page.url` for canonical URLs. |
|
||||
| `vfs.odin` | Union file system: `VFS`, `build_vfs`, `mount_dir`, `mount_subdir`, `mount_recursive`, `vfs_get`, `vfs_get_entry`, `vfs_entry_data`. Layers defaults → modules → site. |
|
||||
| `assets.odin` | `copy_assets_dir` — iterates VFS entries with `assets/` prefix, minifies CSS, copies verbatim or via `os.copy_file`. |
|
||||
| `html.odin` | `strip_html_tags` (moved from render.odin), `unescape_html`, `generate_summary` (Hugo-style body summary for OG descriptions). |
|
||||
| `html.odin` | `strip_html_tags`, `unescape_html`, `generate_summary` (word-count truncation, zero-alloc), `generate_description` (HTML→plain text: strip tags, decode entities, collapse whitespace). |
|
||||
| `opengraph.odin` | `Open_Graph` struct (fields ordered per OGP spec, `is_article: Maybe(bool)`). `og_for_site(site)` for site defaults (from config + derived), `og_for_page(site_og, page)` for page-specific (overlay page.og + derive from page data). |
|
||||
| `frontmatter.odin` | JSON frontmatter parser (`{ }` delimited). Supports `layout`, `lastmod`, and nested `og` object (via `json_get_open_graph`). |
|
||||
| `defaults.odin` | `DEFAULTS_PATH` constant, resolved at compile time via `#directory` so bundled templates ship in the binary. |
|
||||
@@ -187,7 +187,7 @@ Content is **not yet in the VFS** — `scan_content` still uses direct filesyste
|
||||
- `is_article ← !page._is_index`
|
||||
- `section ← page.section`
|
||||
- `published_time / modified_time ← page.date / page.lastmod`
|
||||
- `description ← page.description`, else body summary (via `generate_summary`)
|
||||
- `description ← page.description`, else `generate_description(generate_summary(body_html))` (scrubbed plain text)
|
||||
|
||||
Paths through maps (e.g. `params.*`) are silently allowed — not validated. Templates access via `{{og.url}}`, `{{og.title}}`, `{{#og.is_article}}`, etc.
|
||||
|
||||
|
||||
@@ -21,6 +21,8 @@
|
||||
- [ ] return `src: cstring` from `load_query`.
|
||||
- [ ] Improve `unescape_html` with simd.
|
||||
- [ ] generate summary before syntax highlighting.
|
||||
- [ ] generate summary before markdown to html conversion.
|
||||
- [ ] mount_recursive is pretty significant
|
||||
|
||||
|
||||
## Memory Management
|
||||
|
||||
@@ -29,7 +29,7 @@ strip_html_tags :: proc(s: string, allocator := context.allocator) -> string {
|
||||
}
|
||||
|
||||
unescape_html :: proc(s: string) -> string {
|
||||
sb := strings.builder_make_len(len(s))
|
||||
sb := strings.builder_make_len_cap(0, len(s))
|
||||
defer strings.builder_destroy(&sb)
|
||||
|
||||
start := 0
|
||||
@@ -72,73 +72,146 @@ unescape_html :: proc(s: string) -> string {
|
||||
return strings.to_string(sb)
|
||||
}
|
||||
|
||||
// generate_summary produces a plain-text summary of an HTML fragment.
|
||||
// Blocks (paragraphs, headings, list items) are extracted, their tags
|
||||
// stripped, entities decoded, and accumulated word-by-word until the
|
||||
// max_words threshold is crossed — at which point the rest of the
|
||||
// current block is included before stopping. Mirrors Hugo's default
|
||||
// summary behavior.
|
||||
// generate_summary truncates an HTML string to the first max_words words.
|
||||
// Walks forward counting whitespace→text transitions, skipping tag interiors
|
||||
// so spaces inside attributes don't count. Returns a substring of the
|
||||
// original — zero allocation. Mirrors Hugo's default (70 words).
|
||||
generate_summary :: proc(html: string, max_words: int = 70) -> string {
|
||||
separated, _ := strings.replace_all(html, "</p>", "\n\n", context.temp_allocator)
|
||||
separated, _ = strings.replace_all(separated, "</h1>", "\n\n")
|
||||
separated, _ = strings.replace_all(separated, "</h2>", "\n\n")
|
||||
separated, _ = strings.replace_all(separated, "</h3>", "\n\n")
|
||||
separated, _ = strings.replace_all(separated, "</h4>", "\n\n")
|
||||
separated, _ = strings.replace_all(separated, "</h5>", "\n\n")
|
||||
separated, _ = strings.replace_all(separated, "</h6>", "\n\n")
|
||||
separated, _ = strings.replace_all(separated, "</li>", "\n\n")
|
||||
separated, _ = strings.replace_all(separated, "</blockquote>", "\n\n")
|
||||
|
||||
stripped := strip_html_tags(separated, context.temp_allocator)
|
||||
plain := unescape_html(stripped)
|
||||
|
||||
blocks := strings.split(plain, "\n\n", allocator = context.temp_allocator)
|
||||
defer delete(blocks)
|
||||
|
||||
sb := strings.builder_make(context.temp_allocator)
|
||||
defer strings.builder_destroy(&sb)
|
||||
|
||||
if max_words <= 0 {
|
||||
return ""
|
||||
}
|
||||
word_count := 0
|
||||
first := true
|
||||
for raw_block in blocks {
|
||||
block := strings.trim_space(raw_block)
|
||||
if len(block) == 0 {
|
||||
in_word := false
|
||||
in_tag := false
|
||||
for i in 0 ..< len(html) {
|
||||
c := html[i]
|
||||
if in_tag {
|
||||
if c == '>' {
|
||||
in_tag = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Collapse internal whitespace to single spaces.
|
||||
block_sb := strings.builder_make(context.temp_allocator)
|
||||
has_content := false
|
||||
in_space := true
|
||||
for c in block {
|
||||
if c == ' ' || c == '\t' || c == '\n' || c == '\r' {
|
||||
in_space = true
|
||||
} else {
|
||||
if in_space && has_content {
|
||||
strings.write_byte(&block_sb, ' ')
|
||||
}
|
||||
strings.write_rune(&block_sb, c)
|
||||
in_space = false
|
||||
has_content = true
|
||||
}
|
||||
}
|
||||
collapsed := strings.to_string(block_sb)
|
||||
words := strings.split(collapsed, " ", allocator = context.temp_allocator)
|
||||
|
||||
if !first && word_count > 0 {
|
||||
strings.write_byte(&sb, ' ')
|
||||
}
|
||||
strings.write_string(&sb, collapsed)
|
||||
word_count += len(words)
|
||||
first = false
|
||||
|
||||
delete(words)
|
||||
|
||||
if c == '<' {
|
||||
in_tag = true
|
||||
if in_word {
|
||||
word_count += 1
|
||||
if word_count >= max_words {
|
||||
break
|
||||
return html[:i]
|
||||
}
|
||||
in_word = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
is_space := c == ' ' || c == '\n' || c == '\t' || c == '\r'
|
||||
if is_space {
|
||||
if in_word {
|
||||
word_count += 1
|
||||
if word_count >= max_words {
|
||||
return html[:i]
|
||||
}
|
||||
in_word = false
|
||||
}
|
||||
} else {
|
||||
in_word = true
|
||||
}
|
||||
}
|
||||
|
||||
return strings.to_string(sb)
|
||||
return html
|
||||
}
|
||||
|
||||
// generate_description converts an HTML fragment to plain text by stripping
|
||||
// tags, decoding entities, and collapsing whitespace. Emits a space when
|
||||
// exiting any tag so block-level boundaries aren't lost. Intended for OG
|
||||
// descriptions — operate on the output of generate_summary for bounded input.
|
||||
generate_description :: proc(html: string, allocator := context.temp_allocator) -> string {
|
||||
sb := strings.builder_make_len_cap(0, len(html), allocator)
|
||||
defer strings.builder_destroy(&sb)
|
||||
|
||||
in_tag := false
|
||||
prev_was_space := true
|
||||
run_start := 0
|
||||
i := 0
|
||||
for i < len(html) {
|
||||
c := html[i]
|
||||
if in_tag {
|
||||
if c == '>' {
|
||||
in_tag = false
|
||||
if !prev_was_space {
|
||||
strings.write_byte(&sb, ' ')
|
||||
prev_was_space = true
|
||||
}
|
||||
}
|
||||
i += 1
|
||||
run_start = i
|
||||
continue
|
||||
}
|
||||
if c == '<' {
|
||||
if i > run_start {
|
||||
strings.write_string(&sb, html[run_start:i])
|
||||
prev_was_space = false
|
||||
}
|
||||
in_tag = true
|
||||
i += 1
|
||||
continue
|
||||
}
|
||||
if c == '&' {
|
||||
if i > run_start {
|
||||
strings.write_string(&sb, html[run_start:i])
|
||||
prev_was_space = false
|
||||
}
|
||||
semi := strings.index(html[i:], ";")
|
||||
if semi > 0 && semi <= 5 {
|
||||
entity := html[i:i + semi + 1]
|
||||
replacement := ""
|
||||
switch entity {
|
||||
case "&":
|
||||
replacement = "&"
|
||||
case "<":
|
||||
replacement = "<"
|
||||
case ">":
|
||||
replacement = ">"
|
||||
case """:
|
||||
replacement = "\""
|
||||
case "'", "'":
|
||||
replacement = "'"
|
||||
case:
|
||||
replacement = ""
|
||||
}
|
||||
if replacement != "" {
|
||||
strings.write_string(&sb, replacement)
|
||||
prev_was_space = false
|
||||
i += semi + 1
|
||||
run_start = i
|
||||
continue
|
||||
}
|
||||
}
|
||||
strings.write_byte(&sb, '&')
|
||||
prev_was_space = false
|
||||
i += 1
|
||||
run_start = i
|
||||
continue
|
||||
}
|
||||
if c == ' ' || c == '\n' || c == '\t' || c == '\r' {
|
||||
if i > run_start {
|
||||
strings.write_string(&sb, html[run_start:i])
|
||||
prev_was_space = false
|
||||
}
|
||||
if !prev_was_space {
|
||||
strings.write_byte(&sb, ' ')
|
||||
prev_was_space = true
|
||||
}
|
||||
i += 1
|
||||
run_start = i
|
||||
continue
|
||||
}
|
||||
i += 1
|
||||
}
|
||||
if i > run_start && !in_tag {
|
||||
strings.write_string(&sb, html[run_start:i])
|
||||
}
|
||||
|
||||
result := strings.to_string(sb)
|
||||
if len(result) > 0 && result[len(result) - 1] == ' ' {
|
||||
result = result[:len(result) - 1]
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
#+test
|
||||
package main
|
||||
|
||||
import "core:testing"
|
||||
|
||||
// --- generate_summary ---
|
||||
|
||||
@(test)
|
||||
test_summary_short :: proc(t: ^testing.T) {
|
||||
result := generate_summary("<p>Hello world</p>")
|
||||
testing.expect_value(t, result, "<p>Hello world</p>")
|
||||
}
|
||||
|
||||
@(test)
|
||||
test_summary_word_limit :: proc(t: ^testing.T) {
|
||||
result := generate_summary("<p>one two three four five</p>", max_words = 3)
|
||||
testing.expect_value(t, result, "<p>one two three")
|
||||
}
|
||||
|
||||
@(test)
|
||||
test_summary_empty :: proc(t: ^testing.T) {
|
||||
result := generate_summary("")
|
||||
testing.expect_value(t, result, "")
|
||||
}
|
||||
|
||||
@(test)
|
||||
test_summary_no_words :: proc(t: ^testing.T) {
|
||||
result := generate_summary("<p></p>")
|
||||
testing.expect_value(t, result, "<p></p>")
|
||||
}
|
||||
|
||||
@(test)
|
||||
test_summary_tags_not_counted :: proc(t: ^testing.T) {
|
||||
html := `<pre><code><span class="hl-keyword">if</span> x</code></pre>`
|
||||
result := generate_summary(html, max_words = 1)
|
||||
testing.expect_value(t, result, `<pre><code><span class="hl-keyword">if`)
|
||||
}
|
||||
|
||||
// --- generate_description ---
|
||||
|
||||
@(test)
|
||||
test_description_simple :: proc(t: ^testing.T) {
|
||||
result := generate_description("<p>Hello world</p>")
|
||||
testing.expect_value(t, result, "Hello world")
|
||||
}
|
||||
|
||||
@(test)
|
||||
test_description_entities :: proc(t: ^testing.T) {
|
||||
result := generate_description("<p>Cats & dogs <3</p>")
|
||||
testing.expect_value(t, result, "Cats & dogs <3")
|
||||
}
|
||||
|
||||
@(test)
|
||||
test_description_nested_tags :: proc(t: ^testing.T) {
|
||||
result := generate_description("<p><strong>Bold</strong> text</p>")
|
||||
testing.expect_value(t, result, "Bold text")
|
||||
}
|
||||
|
||||
@(test)
|
||||
test_description_block_boundary :: proc(t: ^testing.T) {
|
||||
result := generate_description("<p>First</p><p>Second</p>")
|
||||
testing.expect_value(t, result, "First Second")
|
||||
}
|
||||
|
||||
@(test)
|
||||
test_description_whitespace_collapse :: proc(t: ^testing.T) {
|
||||
result := generate_description("<p> Multiple spaces </p>")
|
||||
testing.expect_value(t, result, "Multiple spaces")
|
||||
}
|
||||
|
||||
@(test)
|
||||
test_description_empty :: proc(t: ^testing.T) {
|
||||
result := generate_description("")
|
||||
testing.expect_value(t, result, "")
|
||||
}
|
||||
|
||||
@(test)
|
||||
test_description_plain_text :: proc(t: ^testing.T) {
|
||||
result := generate_description("Just plain text")
|
||||
testing.expect_value(t, result, "Just plain text")
|
||||
}
|
||||
|
||||
@(test)
|
||||
test_description_highlighted_code :: proc(t: ^testing.T) {
|
||||
result := generate_description(
|
||||
`<pre><code><span class="hl-keyword">if</span> x</code></pre>`,
|
||||
)
|
||||
testing.expect_value(t, result, "if x")
|
||||
}
|
||||
|
||||
@(test)
|
||||
test_description_list_items :: proc(t: ^testing.T) {
|
||||
result := generate_description("<ul><li>One</li><li>Two</li></ul>")
|
||||
testing.expect_value(t, result, "One Two")
|
||||
}
|
||||
|
||||
@(test)
|
||||
test_description_headings :: proc(t: ^testing.T) {
|
||||
result := generate_description("<h1>Title</h1><p>Body</p>")
|
||||
testing.expect_value(t, result, "Title Body")
|
||||
}
|
||||
|
||||
@(test)
|
||||
test_description_blockquote :: proc(t: ^testing.T) {
|
||||
result := generate_description("<blockquote>Quote</blockquote>")
|
||||
testing.expect_value(t, result, "Quote")
|
||||
}
|
||||
+1
-1
@@ -71,7 +71,7 @@ og_for_page :: proc(site_og: Open_Graph, page: Page) -> Open_Graph {
|
||||
description_set = true
|
||||
}
|
||||
if !description_set && is_article && page.body_html != "" {
|
||||
og.description = generate_summary(page.body_html)
|
||||
og.description = generate_description(generate_summary(page.body_html))
|
||||
description_set = true
|
||||
}
|
||||
if !description_set {
|
||||
|
||||
Reference in New Issue
Block a user