From 0ba29e43edd7b599f2d64a33101f20aeecf39ce3 Mon Sep 17 00:00:00 2001
From: Spencer Brower <6729162+sbrow@users.noreply.github.com>
Date: Tue, 14 Jul 2026 11:26:05 -0400
Subject: [PATCH] feat: Added margin notes support.

---
 AGENTS.md           | 49 ++++++++++++++++++-----
 HUGO.md             | 54 +++++++++++++++++++++++++
 TODOS.md            |  3 ++
 content.odin        |  4 +-
 footnotes.odin      | 91 ++++++++++++++++++++++++++++++-----------
 footnotes_test.odin | 98 +++++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 262 insertions(+), 37 deletions(-)
 create mode 100644 HUGO.md
 create mode 100644 footnotes_test.odin

diff --git a/AGENTS.md b/AGENTS.md
index 0b1fc80..9e7541a 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -5,10 +5,10 @@ Thor is a static site generator written in [Odin](https://odin-lang.org), replac
 ## Architecture
 
 ```
-thor.json          ← site config (title, base_url, author, params)
+thor.json          ← site config (title, base_url, author, params, sectionate)
 content/           ← markdown and HTML content files
-layouts/           ← Mustache templates + partials
-assets/            ← CSS (Tufte-based) and JS
+layouts/           ← Mustache templates + partials (including icons)
+assets/            ← CSS (Tufte-based, no build step) and JS
 public/            ← build output (generated)
 ```
 
@@ -16,13 +16,16 @@ public/            ← build output (generated)
 
 | File | Responsibility |
 |---|---|
-| `main.odin` | Entry point. Calls `init_site`, `walk_content`, `render_site` |
+| `main.odin` | Entry point. Sets `context.logger`, calls `init_site`, `walk_content`, `render_site` |
 | `site.odin` | `Site` struct (config + arena), `init_site`, `load_site_config`, `site_merge`, `site_allocator`, `destroy_site` |
 | `frontmatter.odin` | JSON frontmatter parser (`{ }` delimited) |
-| `content.odin` | `Page` struct, content walker, page loader, cmark integration, footnote/alert/emoji pipeline |
-| `footnotes.odin` | Footnote definition stripping (pre-cmark) + sidenote injection (post-cmark) |
+| `content.odin` | `Page` struct, content walker, page loader, cmark integration, full markdown pipeline |
+| `footnotes.odin` | Note definition stripping (pre-cmark) + sidenote/marginnote injection (post-cmark) |
 | `alerts.odin` | GitHub alert post-processor (`> [!CAUTION]` → styled blockquote) |
 | `emoji.odin` | Emoji shortcode expander (`:shrug:` → `¯\_(ツ)_/¯`) |
+| `highlight.odin` | Post-cmark Tree-sitter syntax highlighter; loads grammars/queries from Helix, caches loaded grammars, reports syntax errors with file/line |
+| `tree_sitter.odin` | C FFI bindings for Tree-sitter (TSParser, TSQuery, TSQueryCursor, node traversal, dlopen/dlsym) |
+| `sectionate.odin` | `wrap_sections` proc — splits HTML at `<h2` into `<section>` wrappers |
 | `render.odin` | Mustache template rendering, all page types, RSS, sitemap, robots.txt, `load_partials` recursive scan |
 | `feed.odin` | RSS feed + sitemap XML generation |
 | `mustache/` | Vendored [odin-mustache](https://github.com/benjamindblock/odin-mustache) library |
@@ -34,20 +37,21 @@ Icon SVGs live as HTML partials in `layouts/partials/icons/` (home, github, rss,
 ```
 thor.json → init_site → Site (config + Dynamic_Arena)
                             ↓
-content/ → walk_content → []Page (with body_html from cmark pipeline)
+content/ → walk_content → []Page (with body_html from pipeline)
                             ↓
 layouts/*.html → render_site (Mustache render_in_layout) → public/
 ```
 
 ### Config system
 
-`Site` has `params: json.Value` for arbitrary user-defined data from `thor.json`. Social links and other template-only data live under `"params"`:
+`Site` has `params: json.Value` for arbitrary user-defined data from `thor.json`. Social links and other template-only data live under `"params"`. `sectionate: bool` controls automatic `<section>` wrapping at `<h2>` boundaries.
 
 ```json
 {
   "title": "...",
   "base_url": "...",
   "author": "...",
+  "sectionate": true,
   "params": {
     "social": [
       { "name": "github", "url": "...", "icon": "icons/github" }
@@ -69,10 +73,24 @@ raw markdown
   → cmark markdown_to_html (Unsafe mode for HTML passthrough)
   → inject_sidenotes      (post-cmark: [^id] → <label><input><span> markup)
   → inject_alerts         (post-cmark: [!TYPE] blockquotes → styled alerts)
+  → highlight_code        (post-cmark: tree-sitter per code block, with error reporting)
+  → wrap_sections         (post-cmark: if site.sectionate, wraps content in <section> at <h2>)
 ```
 
 `.html` content files skip cmark entirely — body is used as-is.
 
+### Syntax highlighting
+
+Build-time highlighting via Tree-sitter C FFI. No client-side JavaScript.
+
+- Grammars loaded via `dlopen` from Helix's compiled `.so` files
+- Highlight queries (`.scm`) loaded from Helix's runtime directory
+- Paths hardcoded in `tree_sitter.odin` (Nix store paths, Helix-version-dependent)
+- Capture names mapped to CSS classes: `keyword` → `.hl-keyword`, `constant.numeric.integer` → `.hl-constant-numeric-integer`, etc.
+- Atom-one-dark color theme in `main.css`
+- Failed grammar loads are cached (no retries) and logged via `log.warnf`
+- Syntax errors detected via `ts_node_has_error`, reported with file path and line number relative to code block
+
 ### Memory management
 
 - `Site` owns a `mem.Dynamic_Arena`
@@ -80,6 +98,7 @@ raw markdown
 - Config loading (flags + JSON) uses the arena allocator explicitly
 - `site_allocator(site)` returns the arena allocator for callers
 - `destroy_site` frees the arena
+- `main.odin` sets `context.logger = log.create_console_logger()` — without this, all `log.*` calls are silently dropped
 - **Not yet wired:** `context.allocator` is not set to the arena in `main.odin`, so rendering and content processing still use the heap allocator
 
 ## Building
@@ -90,12 +109,12 @@ raw markdown
 nix develop
 # From blog root:
 odin run ./thor -- -drafts
-cp assets/css/tufte.css public/css/tufte.css
+cp assets/css/main.css public/css/main.css
 cp assets/js/main.js public/js/main.js
 caddy run  # serves public/ on blog.localhost
 ```
 
-No CSS build step — `tufte.css` is static CSS, no preprocessor or compiler needed.
+No CSS build step — `main.css` is static Tufte-based CSS, no preprocessor or compiler needed.
 
 ### Production build
 
@@ -119,7 +138,7 @@ Four modifications to `mustache/mustache.odin`:
 
 2. **Layout partials** — `layout_template.partials = tmpl.partials` added so partials (`{{> nav}}`, `{{> footer}}`) work inside the base layout template.
 
-3. **Inline partial rendering** — replaced `template_insert_partial` (which injected tokens into the main token list, breaking section iteration) with `template_render_partial` (which lexes the partial, temporarily swaps `tmpl.lexer`, and recursively processes via `template_process_tokens`). Fixes partials inside sections.
+3. **Inline partial rendering** — replaced `template_insert_partial` (which injected tokens into the main token list, breaking section iteration) with `template_render_partial` (which lexes the partial, temporarily swaps `tmpl.lexer`, and recursively processes via `template_process_tokens`). Fixes partials inside sections. Standalone indentation applied to partial source before lexing.
 
 4. **Dynamic Names** — `{{>*key}}` support. When a partial token value starts with `*`, the remaining key is resolved from the data context stack, and the resolved string is used as the partial name. Enables per-item partial selection inside sections (e.g., `{{>* icon}}` resolves `icon` from each social link).
 
@@ -131,6 +150,14 @@ Extracted `template_process_tokens` from `template_eat_tokens` to separate ROOT
 - CSS/JS cache busting uses manual `?v=N` query params instead of content hashing.
 - The `shrug` emoji has a backslash that may not display correctly.
 - `json.Value` params require 64-byte aligned arena (workaround for `dynamic_arena_allocator_proc` ignoring per-allocation alignment).
+- Tree-sitter grammar/query paths hardcoded in `tree_sitter.odin` (Nix store hashes, Helix-version-dependent).
+- `TSQueryCapture` needs explicit `_padding: u32` field for C ABI compatibility (40-byte sizeof).
+- `{{&content}}` in layout template must have no leading whitespace — `template_insert_content_into_layout` indents all lines by preceding whitespace, which corrupts `<pre>` blocks.
+- highlight.js removed; syntax highlighting is build-time only (no fallback if Tree-sitter fails).
+
+## Design decisions
+
+See `HUGO.md` for analysis of why thor doesn't need Hugo's shortcode context isolation.
 
 ## TODO
 
diff --git a/HUGO.md b/HUGO.md
new file mode 100644
index 0000000..f5d97aa
--- /dev/null
+++ b/HUGO.md
@@ -0,0 +1,54 @@
+# Hugo vs Thor: Why Thor Doesn't Need Shortcode Context Isolation
+
+## Hugo's Shortcode Isolation
+
+Hugo strictly separates shortcode context from layout template context. Shortcodes run during markdown rendering, before layouts. They get a limited context (`.Page`, `.Site`, `.Params`), not the full layout rendering state.
+
+### Why Hugo does this
+
+1. **Circular dependencies** — Hugo shortcodes output into `.Content`. Layouts read `.Content` and compute derived properties (`.TableOfContents`, `.WordCount`, `.ReadingTime`). If shortcodes could access these computed properties, you'd get circular dependencies (shortcode output → computed property → shortcode reads it).
+
+2. **Multiple layouts/themes** — Hugo supports user-selectable themes and multiple layouts per section type. A shortcode must produce identical output regardless of which layout renders it. Isolating context guarantees portability.
+
+3. **Shared mutable page object** — Hugo builds a rich `Page` object that both shortcodes and layouts access. Context isolation prevents shortcodes from mutating layout-visible state.
+
+4. **Security** — Shortcodes live in content files (potentially user-authored). Full template access would let content authors inject arbitrary logic, access sensitive config, or break site structure.
+
+5. **Caching** — Hugo caches shortcode output independently. Predictable context = predictable cache behavior.
+
+## Why this doesn't apply to thor
+
+- **No computed page properties** — The content Mustache pass runs on raw markdown (before cmark). The layout Mustache pass runs on finished HTML (after cmark). There's no shared `.Content` object or derived properties. The two passes are at different pipeline stages with no shared mutable state.
+
+- **Single layout set** — Thor has one set of templates (`base.html` + page templates). No theme switching, no multiple layout variants per section. Portability across layouts isn't a concern.
+
+- **Explicit data passing** — Each `mustache.render` call receives its own data struct. Content pass gets frontmatter + params. Layout pass gets page data + site config. No shared mutable object between them.
+
+- **Single author** — Content is authored by the site owner. No untrusted user-generated content.
+
+- **No caching** — Thor rebuilds from scratch every time. No cache invalidation concerns.
+
+## Thor's approach
+
+Thor runs Mustache on content **before** cmark as a pre-processing step, then runs Mustache on layout templates **after** cmark as a post-processing step. These are two independent `mustache.render` calls:
+
+```
+content (markdown)
+  → mustache.render(content, content_data, partials)   ← content pass
+  → cmark markdown_to_html
+  → inject_sidenotes / inject_alerts / highlight_code
+  → mustache.render_in_layout(template, page_data, layout, partials)  ← layout pass
+  → final HTML
+```
+
+The content pass can use Mustache variables, partials (`{{> ./file}}`), sections, and conditionals freely. The layout pass wraps the rendered content in the page chrome. They share nothing except the data we explicitly choose to pass.
+
+## When this WOULD matter for thor
+
+If thor ever adds:
+- Computed page properties (table of contents, reading time, word count)
+- Multiple selectable themes/layouts
+- User-generated content / multi-author support
+- Partial template caching
+
+...then context isolation between the content and layout passes would become relevant. Until then, it's unnecessary complexity.
diff --git a/TODOS.md b/TODOS.md
index ead6ee8..0442b6f 100644
--- a/TODOS.md
+++ b/TODOS.md
@@ -3,12 +3,15 @@
 - [ ] Content-hash fingerprinting for CSS and JS cache busting
 - [ ] Enable configuration to  opt-out of features, particular pre/post processors.
 - [ ] proper date/time/now object.
+- [ ] mustache data keys for opengraph, etc.
 - [ ] Block attributes on code fences (`{ #ex-1 }`) — hello-world.md
 - [x] Emoji shortcodes (`:shrug:` etc.) — 2 instances
   - [ ] Backslash in shrug not visible.
 - [ ] include-code shortcode (`{{< include-code ... >}}`) — i-ported-fd-to-odin
 - [x] Nix build integration — main flake runs thor + tailwindcss instead of Hugo
 - [x] OpenGraph meta tags
+- [ ] copy all files from `static` to `public`.
+- [ ] ensure sidenote numbers render in display order and not in declaration order.
 - [ ] Search up for `thor.json` files.
 - [ ] OpenGraph meta tags — verify all fields match production site
 - [ ] Table of contents support.
diff --git a/content.odin b/content.odin
index c35ec1c..32b5b3a 100644
--- a/content.odin
+++ b/content.odin
@@ -177,9 +177,9 @@ load_page :: proc(
 		page.body_html = strings.clone(body)
 	} else {
 		expanded := expand_emoji(body)
-		clean_body, defs := strip_definitions(expanded)
+		clean_body, sn_defs, mn_defs := strip_definitions(expanded)
 		html := cm.markdown_to_html_from_string(clean_body, {.Unsafe})
-		html = highlight_code(inject_alerts(inject_sidenotes(html, defs)), file_path)
+		html = highlight_code(inject_alerts(inject_notes(html, sn_defs, mn_defs)), file_path)
 		if sectionate {
 			html = wrap_sections(html)
 		}
diff --git a/footnotes.odin b/footnotes.odin
index d31af10..85b6fb4 100644
--- a/footnotes.odin
+++ b/footnotes.odin
@@ -5,11 +5,18 @@ import cm "vendor:commonmark"
 import "core:fmt"
 import "core:strings"
 
-// strip_definitions scans markdown text for footnote definitions ([^id]: text),
-// removes them, and returns the cleaned text plus a map of id→definition.
+Note_Kind :: enum {
+	Sidenote,
+	Marginnote,
+}
+
+// strip_definitions scans markdown text for note definitions ([^id]: text for
+// sidenotes, [*id]: text for marginnotes), removes them, and returns the cleaned
+// text plus separate maps of id->definition for each kind.
 // Handles multi-line definitions with indented continuation lines.
-strip_definitions :: proc(body: string) -> (clean_body: string, defs: map[string]string) {
-	defs = make(map[string]string)
+strip_definitions :: proc(body: string) -> (clean_body: string, sn_defs, mn_defs: map[string]string) {
+	sn_defs = make(map[string]string)
+	mn_defs = make(map[string]string)
 
 	lines := strings.split(body, "\n")
 	output_lines: [dynamic]string
@@ -19,7 +26,7 @@ strip_definitions :: proc(body: string) -> (clean_body: string, defs: map[string
 	for i < len(lines) {
 		line := lines[i]
 
-		id, def_text, is_def := parse_def_line(line)
+		id, def_text, kind, is_def := parse_def_line(line)
 		if !is_def {
 			append(&output_lines, line)
 			i += 1
@@ -39,8 +46,8 @@ strip_definitions :: proc(body: string) -> (clean_body: string, defs: map[string
 			if len(next) == 0 {
 				break
 			}
-			// Stop at new footnote definitions
-			_, _, is_new_def := parse_def_line(next)
+			// Stop at new note definitions
+			_, _, _, is_new_def := parse_def_line(next)
 			if is_new_def {
 				break
 			}
@@ -53,7 +60,12 @@ strip_definitions :: proc(body: string) -> (clean_body: string, defs: map[string
 			i += 1
 		}
 
-		defs[id] = strings.join(def_parts[:], "\n")
+		joined := strings.join(def_parts[:], "\n")
+		if kind == .Marginnote {
+			mn_defs[id] = joined
+		} else {
+			sn_defs[id] = joined
+		}
 		delete(def_parts)
 	}
 
@@ -61,9 +73,18 @@ strip_definitions :: proc(body: string) -> (clean_body: string, defs: map[string
 	return
 }
 
-// parse_def_line checks if a line is a footnote definition: [^id]: text
-parse_def_line :: proc(line: string) -> (id: string, text: string, ok: bool) {
-	if len(line) < 5 || line[0] != '[' || line[1] != '^' {
+// parse_def_line checks if a line is a note definition: [^id]: text (sidenote)
+// or [*id]: text (marginnote).
+parse_def_line :: proc(line: string) -> (id: string, text: string, kind: Note_Kind, ok: bool) {
+	if len(line) < 5 || line[0] != '[' {
+		return
+	}
+
+	if line[1] == '^' {
+		kind = .Sidenote
+	} else if line[1] == '*' {
+		kind = .Marginnote
+	} else {
 		return
 	}
 
@@ -90,10 +111,11 @@ is_indented :: proc(line: string) -> bool {
 	return line[0] == ' ' || line[0] == '\t'
 }
 
-// inject_sidenotes finds [^id] references in rendered HTML and replaces them
-// with sidenote markup. Each definition is rendered through cmark separately.
-inject_sidenotes :: proc(html: string, defs: map[string]string) -> string {
-	if len(defs) == 0 {
+// inject_notes finds [^id] (sidenote) and [*id] (marginnote) references in
+// rendered HTML and replaces them with the appropriate margin markup. Each
+// definition is rendered through cmark separately.
+inject_notes :: proc(html: string, sn_defs, mn_defs: map[string]string) -> string {
+	if len(sn_defs) == 0 && len(mn_defs) == 0 {
 		return html
 	}
 
@@ -103,13 +125,20 @@ inject_sidenotes :: proc(html: string, defs: map[string]string) -> string {
 	remaining := html
 
 	for {
-		pos := strings.index(remaining, "[^")
+		sn_pos := strings.index(remaining, "[^")
+		mn_pos := strings.index(remaining, "[*")
+
+		is_margin := mn_pos >= 0 && (sn_pos < 0 || mn_pos < sn_pos)
+		pos := sn_pos
+		if is_margin {
+			pos = mn_pos
+		}
 		if pos < 0 {
 			append(&parts, remaining)
 			break
 		}
 
-		// Append text before [^
+		// Append text before the reference
 		append(&parts, remaining[:pos])
 
 		close := strings.index(remaining[pos + 2:], "]")
@@ -121,6 +150,10 @@ inject_sidenotes :: proc(html: string, defs: map[string]string) -> string {
 		id := remaining[pos + 2 : pos + 2 + close]
 		ref_end := pos + 2 + close + 1
 
+		defs := sn_defs
+		if is_margin {
+			defs = mn_defs
+		}
 		def_text, found := defs[id]
 		if !found {
 			// No definition found, leave as literal text
@@ -133,13 +166,23 @@ inject_sidenotes :: proc(html: string, defs: map[string]string) -> string {
 		def_html := cm.markdown_to_html_from_string(def_text, {.Unsafe})
 		def_html = strip_p_tags(def_html)
 
-		sidenote := fmt.aprintf(
-			`<label for="fn-%s" class="margin-toggle sidenote-number"></label><input type="checkbox" id="fn-%s" class="margin-toggle"><span class="sidenote">%s</span>`,
-			id,
-			id,
-			def_html,
-		)
-		append(&parts, sidenote)
+		note: string
+		if is_margin {
+			note = fmt.aprintf(
+				`<label for="mn-%s" class="margin-toggle"></label><input type="checkbox" id="mn-%s" class="margin-toggle"><span class="marginnote">%s</span>`,
+				id,
+				id,
+				def_html,
+			)
+		} else {
+			note = fmt.aprintf(
+				`<label for="fn-%s" class="margin-toggle sidenote-number"></label><input type="checkbox" id="fn-%s" class="margin-toggle"><span class="sidenote">%s</span>`,
+				id,
+				id,
+				def_html,
+			)
+		}
+		append(&parts, note)
 
 		remaining = remaining[ref_end:]
 	}
diff --git a/footnotes_test.odin b/footnotes_test.odin
new file mode 100644
index 0000000..40ff4bd
--- /dev/null
+++ b/footnotes_test.odin
@@ -0,0 +1,98 @@
+#+feature dynamic-literals
+#+test
+package main
+
+import "core:strings"
+import "core:testing"
+
+@(test)
+test_parse_def_line :: proc(t: ^testing.T) {
+	// sidenote definition
+	id, text, kind, ok := parse_def_line("[^foo]: bar baz")
+	testing.expect(t, ok)
+	testing.expect(t, id == "foo")
+	testing.expect(t, text == "bar baz")
+	testing.expect(t, kind == .Sidenote)
+
+	// marginnote definition
+	id, text, kind, ok = parse_def_line("[*baz]: qux")
+	testing.expect(t, ok)
+	testing.expect(t, id == "baz")
+	testing.expect(t, text == "qux")
+	testing.expect(t, kind == .Marginnote)
+
+	// plain text is not a definition
+	_, _, _, ok = parse_def_line("regular text")
+	testing.expect(t, !ok)
+
+	// a bracket that is not a note (e.g. reference-link style) is ignored
+	_, _, _, ok = parse_def_line("[link]: url")
+	testing.expect(t, !ok)
+}
+
+@(test)
+test_strip_definitions :: proc(t: ^testing.T) {
+	body := "Intro[^a] and [*b].\n\n[^a]: a side\n\n[*b]: a margin\n"
+	clean, sn_defs, mn_defs := strip_definitions(body)
+
+	// references stay; definitions are removed
+	testing.expect(t, strings.contains(clean, "Intro[^a]"))
+	testing.expect(t, strings.contains(clean, "[*b]"))
+	testing.expect(t, !strings.contains(clean, "[^a]:"))
+	testing.expect(t, !strings.contains(clean, "[*b]:"))
+
+	// routed to the correct map by sigil
+	sa, sa_ok := sn_defs["a"]
+	testing.expect(t, sa_ok)
+	testing.expect(t, sa == "a side")
+
+	mb, mb_ok := mn_defs["b"]
+	testing.expect(t, mb_ok)
+	testing.expect(t, mb == "a margin")
+
+	// ids do not leak across maps
+	_, leaked := sn_defs["b"]
+	testing.expect(t, !leaked)
+	_, leaked2 := mn_defs["a"]
+	testing.expect(t, !leaked2)
+}
+
+@(test)
+test_inject_notes :: proc(t: ^testing.T) {
+	html := "Text[^a] more [*b] end."
+	sn_defs := map[string]string{"a" = "side note"}
+	mn_defs := map[string]string{"b" = "margin note"}
+
+	out := inject_notes(html, sn_defs, mn_defs)
+
+	// sidenote: numbered, fn- prefix, .sidenote span, rendered text
+	testing.expect(t, strings.contains(out, `for="fn-a" class="margin-toggle sidenote-number"></label>`))
+	testing.expect(t, strings.contains(out, `class="sidenote"`))
+	testing.expect(t, strings.contains(out, "side note"))
+
+	// marginnote: unnumbered, mn- prefix, .marginnote span, rendered text
+	testing.expect(t, strings.contains(out, `for="mn-b" class="margin-toggle"></label>`))
+	testing.expect(t, strings.contains(out, `class="marginnote"`))
+	testing.expect(t, strings.contains(out, "margin note"))
+}
+
+@(test)
+test_inject_notes_no_defs :: proc(t: ^testing.T) {
+	html := "No notes here."
+	sn := make(map[string]string)
+	mn := make(map[string]string)
+	testing.expect(t, inject_notes(html, sn, mn) == html)
+}
+
+@(test)
+test_inject_notes_missing_ref :: proc(t: ^testing.T) {
+	html := "Ref[^missing] and [*missing] end."
+	sn := map[string]string{"other" = "x"}
+	mn := map[string]string{"other2" = "y"}
+
+	out := inject_notes(html, sn, mn)
+
+	// references with no matching definition are left as literal text
+	testing.expect(t, strings.contains(out, "[^missing]"))
+	testing.expect(t, strings.contains(out, "[*missing]"))
+}