fix: Fixed race-conditions related to treesitter parsers.

GRAMMAR_CACHE_THREADING.md

@@ -1,9 +1,32 @@
 # Grammar cache thread-safety & allocator lifetime
 
-Status: **investigation complete, fix deferred.** This document records what we
+Status: **resolved.** All three issues below are fixed. The cache was redesigned
-found and the direction we chose, so the next person doesn't have to rediscover
+so the library caches only the immutable, shareable objects and callers own the
-it. No behavior has been changed yet — only documentation comments were added to
+per-thread ones. The rest of this document is kept as the rationale.
-`treesitter/treesitter.odin`.
+
+## Resolution (what shipped)
+
+The `Grammar_Cache` (which mixed shareable `language`/`query` with per-thread
+`parser`/`cursor`) was split:
+
+- **`Grammar`** holds only the immutable `language` + `query`. It lives in a
+  process-lifetime **`registry`** (`map[string]^Grammar`) guarded by one mutex.
+- **`grammar(lang)`** is the thread-safe, self-initializing, lazy loader — fixes
+  **#1a** (all map access under the lock) and lets tests run with no setup.
+- The registry allocator is pinned with `os.heap_allocator()` — fixes **#2**; the
+  cache no longer captures a transient `context.allocator`.
+- Parsers/cursors are **caller-owned**: `open_parser(g)` mints one, the caller
+  `defer`s `parser_delete` / `query_cursor_delete`. No parser is shared, so
+  **#1b** is gone. `open_parser` is the seam where a future pool / thread-local
+  strategy plugs in without touching call sites.
+- Deleted: `ensure_parser`, `load_grammar`, `preload_grammar`, `grammar_worker`,
+  `cache_mutex`; `preload_grammars` is now a serial registry warm-up. The minify
+  tests dropped their `parse_mu` serialization and preload scaffolding entirely.
+
+The **later** decision — pool vs per-thread parser reuse for genuine parallel
+parsing — is still open, but intentionally so: it only matters once we parse in
+parallel, and it now changes only `open_parser`, not the rest of the library.
+The original investigation follows.
 
 ## How we got here
 

TODOS.md

@@ -2,12 +2,15 @@
 
 ## Priority Unclear (Evaluate Later)
 
-- [ ] Decide whether the grammar cache should support multi-threaded parsing.
+- [ ] Decide the parallel-parsing strategy for the grammar cache (pool vs
-  - For now it stays a single global cache used single-threaded; the plan is to
+      thread-local parsers) — deferred until the parallel usage shape is clear.
-    guard the non-thread-safe parser/cursor use with a mutex when tests (or any
+  - Resolved already: the registry (`grammar()`) is thread-safe and self-
-    future parallel caller) need it. A `TSParser`/`TSQueryCursor` is one-per-thread.
+    initializing, its allocator is pinned to the heap, and parsers/cursors are now
-  - Alternative later: give each thread its own parser+cursor for real parallelism.
+    caller-owned (`open_parser`) instead of cached — so the library holds no
-  - Also pin `init_persistent` to the heap allocator (lifetime bug, independent).
+    single-thread-use state and the multi-threaded tests pass without locks.
+  - Remaining choice only matters once we actually parse in parallel: reuse
+    parsers via a pool or per-thread storage, plugged in behind `open_parser`
+    without touching call sites. A `TSParser`/`TSQueryCursor` is one-per-thread.
   - Full write-up: `GRAMMAR_CACHE_THREADING.md`.
 - [ ] instead of `warnings: [dynamic]Error` we should use `warnings: [dynamic; 8]Error`
   - when limit reached, the template fails and stops rendering.

assets.odin

@@ -5,7 +5,20 @@ import "core:log"
 import "core:os"
 import "core:strings"
 
+import ts "treesitter"
+
 copy_assets_dir :: proc(vfs: ^VFS, output_dir: string, features: bit_set[Feature]) {
+	// One CSS parser, reused for every .css asset (single-threaded). Owned here.
+	css_parser: ts.Parser
+	if .Minify in features {
+		if g, ok := ts.grammar("css"); ok {
+			css_parser = ts.open_parser(g)
+		}
+	}
+	defer if css_parser != nil {
+		ts.parser_delete(css_parser)
+	}
+
 	for virtual_path, entry in vfs.files {
 		if !strings.has_prefix(virtual_path, "assets/") {
 			continue
@@ -24,7 +37,11 @@ copy_assets_dir :: proc(vfs: ^VFS, output_dir: string, features: bit_set[Feature
 		if .Minify in features && strings.has_suffix(rel, ".css") {
 			data, ok := vfs_get(vfs, virtual_path)
 			if ok {
-				write_file(dest, minify_css(string(data)))
+				if m, mok := minify_css(css_parser, string(data)); mok {
+					write_file(dest, m)
+				} else {
+					write_file(dest, string(data))
+				}
 			}
 		} else if entry.data != nil {
 			if err := os.write_entire_file(dest, entry.data); err != nil {

markdown/highlight.odin

@@ -125,16 +125,21 @@ unescape_html :: proc(s: string) -> string {
 }
 
 highlight_block :: proc(code: string, lang: string, file_path: string) -> string {
-	gc := ts.load_grammar(lang)
+	g, ok := ts.grammar(lang)
-	if gc == nil {
+	if !ok {
 		return code
 	}
+	parser := ts.open_parser(g)
+	if parser == nil {
+		return code
+	}
+	defer ts.parser_delete(parser)
 
 	raw_code := unescape_html(code)
 	raw_c := strings.clone_to_cstring(raw_code)
 	defer delete(raw_c)
 
-	tree := ts.parser_parse_string(gc.parser, nil, raw_c, u32(len(raw_code)))
+	tree := ts.parser_parse_string(parser, nil, raw_c, u32(len(raw_code)))
 	if tree == nil {
 		return code
 	}
@@ -156,12 +161,13 @@ highlight_block :: proc(code: string, lang: string, file_path: string) -> string
 		}
 	}
 
-	cursor := gc.cursor
+	if g.query == nil {
-	if cursor == nil {
 		return code
 	}
+	cursor := ts.query_cursor_new()
+	defer ts.query_cursor_delete(cursor)
 
-	ts.query_cursor_exec(cursor, gc.query, root)
+	ts.query_cursor_exec(cursor, g.query, root)
 
 	captures := make([dynamic]Capture, 0, 64, context.temp_allocator)
 	defer delete(captures)
@@ -174,7 +180,7 @@ highlight_block :: proc(code: string, lang: string, file_path: string) -> string
 		}
 		cap := match.captures[capture_idx]
 		name_len: u32
-		name_c := ts.query_capture_name_for_id(gc.query, cap.index, &name_len)
+		name_c := ts.query_capture_name_for_id(g.query, cap.index, &name_len)
 		if name_c == nil {
 			continue
 		}

markdown/highlight_test.odin

@@ -0,0 +1,83 @@
+#+test
+package markdown
+
+import ts "../treesitter"
+
+import "core:mem"
+import "core:strings"
+import "core:testing"
+
+// highlight_block returns the input string unchanged (borrowed) when it can't
+// highlight — no grammar, no query, or zero captures — and a freshly-allocated
+// string when it does. Free only the latter; identity (raw_data) tells them
+// apart regardless of content.
+@(private = "file")
+free_highlight :: proc(out, code: string) {
+	if raw_data(out) != raw_data(code) {
+		delete(out)
+	}
+}
+
+// A builtin grammar (css) highlights into hl- spans, preserving the source text.
+@(test)
+test_highlight_block_css :: proc(t: ^testing.T) {
+	code := "a{color:red}"
+	out := highlight_block(code, "css", "test")
+	defer free_highlight(out, code)
+
+	testing.expect(t, strings.contains(out, "<span class=\"hl-"), "css should be highlighted")
+	testing.expect(t, strings.contains(out, "</span>"))
+	testing.expect(t, strings.contains(out, "color"))
+	testing.expect(t, strings.contains(out, "red"))
+}
+
+// The other builtin grammar (html) highlights, and the source's own '<'/'>' are
+// HTML-escaped in the output (so they can't be confused with the wrapping spans).
+@(test)
+test_highlight_block_html_escapes_source :: proc(t: ^testing.T) {
+	code := "<p>hi</p>"
+	out := highlight_block(code, "html", "test")
+	defer free_highlight(out, code)
+
+	testing.expect(t, strings.contains(out, "<span class=\"hl-"), "html should be highlighted")
+	testing.expect(t, strings.contains(out, "hi"))
+	// The literal angle brackets of the source are escaped in the output.
+	testing.expect(t, strings.contains(out, "&lt;"))
+	testing.expect(t, strings.contains(out, "&gt;"))
+}
+
+// A language with no available grammar returns the input verbatim (best-effort:
+// highlighting is optional). The returned string must be the same buffer, so
+// free_highlight leaves it alone.
+@(test)
+test_highlight_block_unknown_lang_passthrough :: proc(t: ^testing.T) {
+	code := "fn main() {}"
+	out := highlight_block(code, "no-such-language", "test")
+	defer free_highlight(out, code)
+
+	testing.expect_value(t, out, code)
+	testing.expect(t, raw_data(out) == raw_data(code), "passthrough must return the input buffer")
+}
+
+// End-to-end: highlight_code finds a fenced code block in rendered HTML and
+// rewrites its body with highlight spans while preserving the <pre><code> shell.
+// Run under a scratch arena (as production does) so the intermediate allocations
+// highlight_code doesn't individually free are reclaimed in one shot.
+@(test)
+test_highlight_code_wraps_css_block :: proc(t: ^testing.T) {
+	arena: mem.Dynamic_Arena
+	mem.dynamic_arena_init(&arena)
+	defer mem.dynamic_arena_destroy(&arena)
+	context.allocator = mem.dynamic_arena_allocator(&arena)
+
+	html := `<pre><code class="language-css">a{color:red}</code></pre>`
+	out := highlight_code(html, "test")
+
+	testing.expect(t, strings.contains(out, `<pre><code class="language-css">`))
+	testing.expect(t, strings.contains(out, "</code></pre>"))
+	testing.expect(
+		t,
+		strings.contains(out, "<span class=\"hl-"),
+		"block body should be highlighted",
+	)
+}

minify.odin

@@ -12,18 +12,31 @@ Range :: struct {
 	end:   u32,
 }
 
-minify_html :: proc(source: string) -> string {
+// minify_html collapses insignificant whitespace, strips comments, and minifies
-	gc := ts.ensure_parser("html")
+// inline <style> bodies (via minify_css). The caller owns html_parser and
-	if gc == nil {
+// css_parser and passes them in; both may be reused across calls. On success it
-		return source
+// returns (owned string, true) and the caller must free the string. On failure
+// (nil html_parser, parse failure, or syntax errors) it returns ("", false) and
+// the caller should keep the original source unchanged.
+minify_html :: proc(
+	html_parser: ts.Parser,
+	css_parser: ts.Parser,
+	source: string,
+) -> (
+	minified: string,
+	ok: bool,
+) {
+	if html_parser == nil {
+		return minified, ok
 	}
 
 	source_c := strings.clone_to_cstring(source)
 	defer delete(source_c)
 
-	tree := ts.parser_parse_string(gc.parser, nil, source_c, u32(len(source)))
+	tree := ts.parser_parse_string(html_parser, nil, source_c, u32(len(source)))
 	if tree == nil {
-		return source
+		ok = false
+		return minified, ok
 	}
 	defer ts.tree_delete(tree)
 
@@ -31,7 +44,8 @@ minify_html :: proc(source: string) -> string {
 
 	if ts.node_has_error(root) {
 		log.warnf("minify: HTML parse errors, skipping minification")
-		return source
+		ok = false
+		return minified, ok
 	}
 
 	comments: [dynamic]Range
@@ -66,14 +80,20 @@ minify_html :: proc(source: string) -> string {
 
 		if si < len(styles) && u32(i) >= styles[si].start {
 			s := styles[si]
-			// NOTE: minify_css may return either a fresh allocation or the
+			seg := source[i:s.end]
-			// input slice unchanged (on parser failure), so the result is not
+			if minified, mok := minify_css(css_parser, seg); mok {
-			// freed here — matching this package's one-shot allocation style.
-			minified := minify_css(source[i:s.end])
 				strings.write_string(&sb, minified)
 				if len(minified) > 0 {
 					last_written = minified[len(minified) - 1]
 				}
+				delete(minified)
+			} else {
+				// CSS minify skipped/failed: keep the style body verbatim.
+				strings.write_string(&sb, seg)
+				if len(seg) > 0 {
+					last_written = seg[len(seg) - 1]
+				}
+			}
 			i = int(s.end)
 			si += 1
 			continue
@@ -111,7 +131,10 @@ minify_html :: proc(source: string) -> string {
 		}
 	}
 
-	return strings.to_string(sb)
+
+	minified, ok = strings.to_string(sb), true
+
+	return minified, ok
 }
 
 collect_html_ranges :: proc(
@@ -136,10 +159,7 @@ collect_html_ranges :: proc(
 			// child) via the CSS-aware pass.
 			raw, ok := style_raw_text(child)
 			if ok {
-				append(
+				append(styles, Range{start = ts.node_start_byte(raw), end = ts.node_end_byte(raw)})
-					styles,
-					Range{start = ts.node_start_byte(raw), end = ts.node_end_byte(raw)},
-				)
 			}
 		} else if type_str == "script_element" {
 			append(
@@ -210,18 +230,23 @@ is_css_delim :: proc(c: u8) -> bool {
 	return false
 }
 
-minify_css :: proc(source: string) -> string {
+// minify_css collapses insignificant whitespace and strips comments. The caller
-	gc := ts.ensure_parser("css")
+// owns `parser` and passes it in; it may be reused across calls. On success it
-	if gc == nil {
+// returns (owned string, true) and the caller must free the string. On failure
-		return source
+// (nil parser, parse failure, or syntax errors) it returns ("", false) and the
+// caller should keep the original source unchanged.
+minify_css :: proc(parser: ts.Parser, source: string) -> (minified: string, ok: bool) {
+	if parser == nil {
+		return minified, ok
 	}
 
 	source_c := strings.clone_to_cstring(source)
 	defer delete(source_c)
 
-	tree := ts.parser_parse_string(gc.parser, nil, source_c, u32(len(source)))
+	tree := ts.parser_parse_string(parser, nil, source_c, u32(len(source)))
 	if tree == nil {
-		return source
+		ok = false
+		return minified, ok
 	}
 	defer ts.tree_delete(tree)
 
@@ -229,7 +254,8 @@ minify_css :: proc(source: string) -> string {
 
 	if ts.node_has_error(root) {
 		log.warnf("minify: CSS parse errors, skipping minification")
-		return source
+		ok = false
+		return minified, ok
 	}
 
 	comments: [dynamic]Range
@@ -270,8 +296,10 @@ minify_css :: proc(source: string) -> string {
 			// (last_written == 0), a doubled space (last_written == ' ',
 			// which happens when a stripped comment sits between two
 			// whitespace runs), or one adjacent to a delimiter.
-			if last_written != 0 && last_written != ' ' &&
+			if last_written != 0 &&
-			   !is_css_delim(last_written) && !is_css_delim(next) {
+			   last_written != ' ' &&
+			   !is_css_delim(last_written) &&
+			   !is_css_delim(next) {
 				strings.write_byte(&sb, ' ')
 				last_written = ' '
 			}
@@ -283,7 +311,8 @@ minify_css :: proc(source: string) -> string {
 		}
 	}
 
-	return strings.to_string(sb)
+	minified, ok = strings.to_string(sb), true
+	return minified, ok
 }
 
 collect_css_comments :: proc(node: ts.Node, comments: ^[dynamic]Range) {

minify_test.odin

@@ -1,59 +1,49 @@
 #+test
 package main
 
+import ts "treesitter"
+
 import "core:strings"
-import "core:sync"
 import "core:testing"
-import "treesitter"
 
-// The treesitter parser store is normally initialized by main() via
+// The treesitter registry is thread-safe and self-initializing (grammar() lazily
-// treesitter.init_persistent(); the test runner never calls main. init_once
+// binds it to the heap), and parsers are caller-owned. Each test opens its own
-// ensures it happens exactly once no matter which test thread arrives first,
+// parser(s) via ts.open_parser, passes them to minify_*, and frees the returned
-// and preloads the builtin parsers on that single thread — after which the
+// string — so the multi-threaded test runner needs no shared state or locking.
-// concurrent tests only *read* the cache (the unlocked writes in ensure_parser
+
-// are what corrupt it, not reads).
 @(private = "file")
-init_once: sync.Once
+open_parsers :: proc(t: ^testing.T) -> (html_parser, css_parser: ts.Parser) {
-
+	hg, hok := ts.grammar("html")
-setup_treesitter :: proc() {
+	cg, cok := ts.grammar("css")
-	sync.once_do(&init_once, proc() {
+	testing.expect(t, hok, "html grammar should load")
-		treesitter.init_persistent()
+	testing.expect(t, cok, "css grammar should load")
-		treesitter.ensure_parser("html")
+	return ts.open_parser(hg), ts.open_parser(cg)
-		treesitter.ensure_parser("css")
-	})
 }
 
-// The builtin parsers are cached and shared per-language, and tree-sitter
-// parsers are not reentrant — concurrent parser_parse_string corrupts their
-// internal subtree pool. The multi-threaded test runner would otherwise race,
-// so serialize parser use behind this mutex. (Production renders on a single
-// thread, so it needs no such lock.)
 @(private = "file")
-parse_mu: sync.Mutex
+expect_css :: proc(t: ^testing.T, parser: ts.Parser, input, want: string) {
-
+	got, ok := minify_css(parser, input)
-ts_minify_html :: proc(s: string) -> string {
+	testing.expect(t, ok)
-	sync.mutex_lock(&parse_mu)
+	defer delete(got)
-	defer sync.mutex_unlock(&parse_mu)
+	testing.expect_value(t, got, want)
-	return minify_html(s)
-}
-
-ts_minify_css :: proc(s: string) -> string {
-	sync.mutex_lock(&parse_mu)
-	defer sync.mutex_unlock(&parse_mu)
-	return minify_css(s)
 }
 
 // --- minify_html: inline <style> bodies get CSS-minified ---
 
 @(test)
 test_minify_html_inline_style :: proc(t: ^testing.T) {
-	setup_treesitter()
+	hp, cp := open_parsers(t)
+	defer ts.parser_delete(hp)
+	defer ts.parser_delete(cp)
+
 	input := `<html><head><style>
 body {
   color:  red;
 }
 </style></head><body><p>hi</p></body></html>`
-	result := ts_minify_html(input)
+	result, ok := minify_html(hp, cp, input)
+	testing.expect(t, ok)
+	defer delete(result)
 
 	// The CSS body is collapsed by minify_css (delimiters lose surrounding
 	// whitespace)...
@@ -65,10 +55,15 @@ body {
 
 @(test)
 test_minify_html_empty_style :: proc(t: ^testing.T) {
-	setup_treesitter()
+	hp, cp := open_parsers(t)
+	defer ts.parser_delete(hp)
+	defer ts.parser_delete(cp)
+
 	// An empty <style></style> has no raw_text child; must not crash.
 	input := `<html><head><style></style></head><body></body></html>`
-	result := ts_minify_html(input)
+	result, ok := minify_html(hp, cp, input)
+	testing.expect(t, ok)
+	defer delete(result)
 	testing.expect(t, strings.contains(result, "<style></style>"))
 }
 
@@ -76,19 +71,28 @@ test_minify_html_empty_style :: proc(t: ^testing.T) {
 
 @(test)
 test_minify_css_leading_and_comment :: proc(t: ^testing.T) {
-	setup_treesitter()
+	cg, cok := ts.grammar("css")
+	testing.expect(t, cok, "css grammar should load")
+	cp := ts.open_parser(cg)
+	defer ts.parser_delete(cp)
+
 	// Leading whitespace and a comment between two whitespace runs must not
 	// leave a leading space or a doubled space.
-	testing.expect_value(t, ts_minify_css("\n/* c */\na {}"), "a{}")
+	expect_css(t, cp, "\n/* c */\na {}", "a{}")
-	testing.expect_value(t, ts_minify_css("  a {}"), "a{}")
+	expect_css(t, cp, "  a {}", "a{}")
-	testing.expect_value(t, ts_minify_css("a {}\n/* trailing */"), "a{}")
+	expect_css(t, cp, "a {}\n/* trailing */", "a{}")
 }
 
 @(test)
 test_minify_html_script_preserved :: proc(t: ^testing.T) {
-	setup_treesitter()
+	hp, cp := open_parsers(t)
+	defer ts.parser_delete(hp)
+	defer ts.parser_delete(cp)
+
 	// <script> stays byte-for-byte (whitespace-significant JS).
 	input := "<html><body><script>\nconst s = \"a    b\";\n</script></body></html>"
-	result := ts_minify_html(input)
+	result, ok := minify_html(hp, cp, input)
+	testing.expect(t, ok)
+	defer delete(result)
 	testing.expect(t, strings.contains(result, "const s = \"a    b\";"))
 }

render.odin

@@ -3,6 +3,7 @@ package main
 import "mustache"
 
 import diags "diagnostics"
+import ts "treesitter"
 
 import "core:encoding/json"
 import "core:fmt"
@@ -205,6 +206,24 @@ render_site :: proc(site: ^Site) {
 		}
 	}
 
+	// Parsers for minification, created once and reused across every page
+	// (render is single-threaded). Owned here; freed when render_site returns.
+	html_parser, css_parser: ts.Parser
+	if .Minify in site.features {
+		if g, gok := ts.grammar("html"); gok {
+			html_parser = ts.open_parser(g)
+		}
+		if g, gok := ts.grammar("css"); gok {
+			css_parser = ts.open_parser(g)
+		}
+	}
+	defer if html_parser != nil {
+		ts.parser_delete(html_parser)
+	}
+	defer if css_parser != nil {
+		ts.parser_delete(css_parser)
+	}
+
 	// Render individual content pages (skip all index pages)
 	for page in pages {
 		if page._is_index {
@@ -213,7 +232,9 @@ render_site :: proc(site: ^Site) {
 		tpl := get_template(&site.vfs, page.layout, &template_cache)
 		html := render_page_html(page, site, tpl, partials, ctx, &errors)
 		if .Minify in site.features {
-			html = minify_html(html)
+			if m, mok := minify_html(html_parser, css_parser, html); mok {
+				html = m
+			}
 		}
 		write_page(site.output_dir, page.permalink, html)
 	}
@@ -243,7 +264,9 @@ render_site :: proc(site: ^Site) {
 			&errors,
 		)
 		if .Minify in site.features {
-			html = minify_html(html)
+			if m, mok := minify_html(html_parser, css_parser, html); mok {
+				html = m
+			}
 		}
 		write_page(site.output_dir, fmt.aprintf("/%s/", section), html)
 	}
@@ -253,7 +276,9 @@ render_site :: proc(site: ^Site) {
 		home_tpl := get_template(&site.vfs, "home", &template_cache)
 		home_html := render_home_html(home, site, home_tpl, partials, ctx, &errors)
 		if .Minify in site.features {
-			home_html = minify_html(home_html)
+			if m, mok := minify_html(html_parser, css_parser, home_html); mok {
+				home_html = m
+			}
 		}
 		write_file(fmt.tprintf("%s/index.html", site.output_dir), home_html)
 	}
@@ -445,4 +470,3 @@ write_file :: proc(path: string, html: string) {
 		log.errorf("cannot write %s: %v", path, err)
 	}
 }
-

treesitter/treesitter.odin

@@ -7,7 +7,6 @@ import "core:mem"
 import "core:os"
 import "core:strings"
 import "core:sync"
-import "core:thread"
 
 grammar_dir: string
 query_dir: string
@@ -120,17 +119,15 @@ foreign css_grammar {
 	tree_sitter_css :: proc() -> Language ---
 }
 
-// THREAD-SAFETY: a Grammar_Cache bundles shareable tree-sitter objects
+// A Grammar bundles only the immutable, shareable tree-sitter objects for a
-// (`language`, `query` — immutable, safe to read from many threads) with
+// language: `language` (from tree_sitter_x() or a dlopen'd .so) and `query` (the
-// per-thread ones (`parser`, `cursor` — a TSParser mutates its internal
+// compiled highlight query). Both are immutable after construction and safe to
-// subtree pool during a parse, and a TSQueryCursor is likewise stateful).
+// read from any number of threads. The stateful, single-thread-use objects —
-// A cached Grammar_Cache is therefore SINGLE-THREADED-USE: exactly one thread
+// parser and query cursor — are deliberately NOT stored here; the caller creates
-// may drive `parser`/`cursor` at a time. See GRAMMAR_CACHE_THREADING.md.
+// and owns those (see `open_parser`). See GRAMMAR_CACHE_THREADING.md.
-Grammar_Cache :: struct {
+Grammar :: struct {
 	language:     Language,
-	parser:       Parser,
 	query:        Query,
-	cursor:       Query_Cursor,
 	query_failed: bool,
 }
 
@@ -138,31 +135,94 @@ Get_Language_Proc :: #type proc() -> Language
 
 SPALL :: #config(SPALL, false)
 
-grammar_store: Grammar_Store
+// registry is the process-lifetime cache of immutable Grammars, keyed by language
-
+// name. It is the library's ONLY shared mutable state; `mu` serializes all access.
-// cache_mutex guards writes to grammar_store.cache. NOTE: today it is only
+// A nil value memoizes "tried and unavailable" so misses don't re-dlopen. Parsers
-// taken on the parallel preload path (grammar_worker), NOT in ensure_parser /
+// live in the caller, so the registry holds nothing single-thread-use.
-// load_grammar. Those lazy paths are intentionally unlocked because production
+Grammar_Registry :: struct {
-// only ever calls them single-threaded. See GRAMMAR_CACHE_THREADING.md before
+	mu:        sync.Mutex,
-// relying on this cache from multiple threads.
+	grammars:  map[string]^Grammar,
-cache_mutex: sync.Mutex
-
-Grammar_Store :: struct {
-	cache:     map[string]^Grammar_Cache,
 	allocator: mem.Allocator,
 }
 
-// KNOWN ISSUE (allocator lifetime): grammar_store is a process-lifetime global,
+registry: Grammar_Registry
-// but this captures whatever `context.allocator` is live at call time. In main
+
-// that is deliberately the heap (init_persistent runs before the site arena is
+// init_persistent is optional — grammar() lazily initializes the registry on first
-// installed), so it works. But it is fragile: called from any other context
+// use — but callers (main) may call it to bind the cache early. The allocator is
-// (e.g. a per-test tracking allocator, or lazily once the site arena is active)
+// pinned to the OS heap, NOT context.allocator, so the cache survives any transient
-// it would bind the cache to a transient allocator and leave it dangling once
+// allocator (per-build arena, per-test tracking) that happens to be live.
-// that allocator is torn down. The intended fix is to pin the heap allocator
-// explicitly (runtime.heap_allocator()). See GRAMMAR_CACHE_THREADING.md.
 init_persistent :: proc() {
-	grammar_store.allocator = context.allocator
+	sync.mutex_lock(&registry.mu)
-	grammar_store.cache = make(map[string]^Grammar_Cache, grammar_store.allocator)
+	defer sync.mutex_unlock(&registry.mu)
+	ensure_registry()
+}
+
+// ensure_registry binds the registry to the heap on first use. Caller must hold mu.
+@(private)
+ensure_registry :: proc() {
+	if registry.grammars == nil {
+		registry.allocator = os.heap_allocator()
+		registry.grammars = make(map[string]^Grammar, registry.allocator)
+	}
+}
+
+// grammar returns the immutable Grammar for `lang`, loading it on first use.
+// Thread-safe: concurrent callers serialize on `mu`. Returns (nil, false) when the
+// grammar cannot be loaded. The returned pointer is immutable — callers may cache
+// and read it (its `language`/`query`) without any further locking.
+grammar :: proc(lang: string) -> (^Grammar, bool) {
+	sync.mutex_lock(&registry.mu)
+	defer sync.mutex_unlock(&registry.mu)
+	ensure_registry()
+	if g, seen := registry.grammars[lang]; seen {
+		return g, g != nil
+	}
+	g := build_grammar(lang)
+	registry.grammars[lang] = g
+	return g, g != nil
+}
+
+// build_grammar loads a language and compiles its highlight query. Caller must
+// hold mu. Returns nil if the language itself cannot be loaded; a grammar whose
+// query is absent/failed is still returned (parsing works, highlighting does not).
+@(private)
+build_grammar :: proc(lang: string) -> ^Grammar {
+	language, ok := load_language(lang)
+	if !ok {
+		return nil
+	}
+	g := new(Grammar, registry.allocator)
+	g.language = language
+	if query, qok := compile_query(lang, language); qok {
+		g.query = query
+	} else {
+		g.query_failed = true
+	}
+	return g
+}
+
+// open_parser creates a fresh parser bound to the grammar's language. The CALLER
+// owns the returned parser and must free it with parser_delete. This is the seam
+// for parser lifetime: today callers open one and reuse it across a batch of
+// parses on a single thread (render opens an html+css pair per render_site;
+// assets opens one per copy_assets_dir), which is safe because nothing is shared.
+// A future pooling or thread-local strategy can be dropped in here without
+// touching any call site. Returns nil on failure.
+open_parser :: proc(g: ^Grammar) -> Parser {
+	if g == nil {
+		return nil
+	}
+	parser := parser_new()
+	if parser == nil {
+		log.errorf("treesitter: cannot create parser")
+		return nil
+	}
+	if !parser_set_language(parser, g.language) {
+		log.errorf("treesitter: ABI mismatch setting parser language")
+		parser_delete(parser)
+		return nil
+	}
+	return parser
 }
 
 when SPALL {
@@ -190,7 +250,7 @@ builtin_language :: proc(lang: string) -> (language: Language, ok: bool) {
 // load_query returns the highlight query source for a language. Builtin
 // languages (html/css) are baked into the binary via `#load`; all others are
 // read from the runtime `query_dir`. `path` is the on-disk location for
-// diagnostics ("(builtin)" for embedded queries). Mirrors `ensure_parser`.
+// diagnostics ("(builtin)" for embedded queries). Mirrors `load_language`.
 load_query :: proc(lang: string) -> (src: string, path: string, ok: bool) {
 	switch lang {
 	case "html":
@@ -239,50 +299,7 @@ load_language :: proc(lang: string) -> (language: Language, ok: bool) {
 	return
 }
 
-// THREAD-SAFETY: ensure_parser is NOT safe to call concurrently. It reads and
+compile_query :: proc(lang: string, language: Language) -> (query: Query, ok: bool) {
-// writes grammar_store.cache without taking cache_mutex, and it returns a shared
-// per-language Grammar_Cache whose `parser` is not reentrant. It is written for
-// single-threaded lazy use (minify, syntax highlighting). Callers that need
-// parsing on multiple threads must serialize with a lock or give each thread
-// its own parser. See GRAMMAR_CACHE_THREADING.md.
-ensure_parser :: proc(lang: string) -> ^Grammar_Cache {
-	if cached, ok := grammar_store.cache[lang]; ok {
-		return cached
-	}
-
-	grammar_store.cache[lang] = nil
-
-	language, ok := load_language(lang)
-	if !ok {
-		return nil
-	}
-
-	parser := parser_new()
-	if parser == nil {
-		log.errorf("treesitter: cannot create parser for %s", lang)
-		return nil
-	}
-	if !parser_set_language(parser, language) {
-		log.errorf("treesitter: ABI mismatch for %s grammar", lang)
-		parser_delete(parser)
-		return nil
-	}
-
-	gc := new(Grammar_Cache, grammar_store.allocator)
-	gc.language = language
-	gc.parser = parser
-	grammar_store.cache[lang] = gc
-	return gc
-}
-
-compile_query :: proc(
-	lang: string,
-	language: Language,
-) -> (
-	query: Query,
-	cursor: Query_Cursor,
-	ok: bool,
-) {
 	query_src, query_path, qok := load_query(lang)
 	if !qok {
 		return
@@ -344,112 +361,20 @@ compile_query :: proc(
 		return
 	}
 
-	cursor = query_cursor_new()
 	ok = true
 	return
 }
 
-load_grammar :: proc(lang: string) -> ^Grammar_Cache {
+// preload_grammars warms the registry for the given languages ahead of the render
-	gc := ensure_parser(lang)
+// loop, so the first page that needs one doesn't pay the load cost inline. Builtins
-	if gc == nil {
+// are always available lazily and are skipped. Runs on the calling thread; grammar()
-		return nil
+// is itself thread-safe if a caller ever wants to fan this out.
-	}
-	if gc.query != nil {
-		return gc
-	}
-	if gc.query_failed {
-		return nil
-	}
-
-	query, cursor, ok := compile_query(lang, gc.language)
-	if !ok {
-		gc.query_failed = true
-		return nil
-	}
-
-	gc.query = query
-	gc.cursor = cursor
-	return gc
-}
-
-preload_grammar :: proc(lang: string) -> ^Grammar_Cache {
-	language, ok := load_language(lang)
-	if !ok {
-		return nil
-	}
-
-	parser := parser_new()
-	if parser == nil {
-		log.errorf("treesitter: cannot create parser for %s", lang)
-		return nil
-	}
-	if !parser_set_language(parser, language) {
-		log.errorf("treesitter: ABI mismatch for %s grammar", lang)
-		parser_delete(parser)
-		return nil
-	}
-
-	gc := new(Grammar_Cache, grammar_store.allocator)
-	gc.language = language
-	gc.parser = parser
-
-	query, cursor, qok := compile_query(lang, language)
-	if !qok {
-		gc.query_failed = true
-		return gc
-	}
-
-	gc.query = query
-	gc.cursor = cursor
-	return gc
-}
-
 preload_grammars :: proc(languages: []string) {
-	if len(languages) == 0 {
-		return
-	}
-
-	// Filter out already-loaded languages (watch mode reuse)
-	to_load := make([dynamic]string, 0, len(languages), context.temp_allocator)
 	for lang in languages {
-		if cached, ok := grammar_store.cache[lang]; ok && cached != nil {
-			continue
-		}
 		if _, bok := builtin_language(lang); bok {
 			continue
 		}
-		append(&to_load, lang)
+		grammar(lang)
-	}
-
-	if len(to_load) == 0 {
-		return
-	}
-
-	threads := make([]^thread.Thread, len(to_load), context.temp_allocator)
-	for i in 0 ..< len(to_load) {
-		threads[i] = thread.create_and_start_with_poly_data(to_load[i], grammar_worker)
-	}
-	for t in threads {
-		thread.join(t)
-		thread.destroy(t)
-	}
-}
-
-grammar_worker :: proc(lang: string) {
-	when SPALL {
-		if _thread_init != nil {
-			_thread_init()
-		}
-		defer if _thread_cleanup != nil {
-			_thread_cleanup()
-		}
-	}
-
-	gc := preload_grammar(lang)
-	if gc != nil {
-		sync.mutex_lock(&cache_mutex)
-		grammar_store.cache[lang] = gc
-		sync.mutex_unlock(&cache_mutex)
 	}
 }