chore: Added plan for dynamically loading grammars.

2026-08-26 11:23:32 -04:00 · 2026-07-24 13:54:24 -04:00
parent 91623e8992
commit 961b3f41b0
2 changed files with 121 additions and 61 deletions
@@ -0,0 +1,74 @@
+# Tree-sitter Grammar Discovery
+
+## Problem
+
+Grammar (`.so`) and query (`.scm`) paths are hardcoded to the developer's machine:
+
+```odin
+GRAPHS_PATH: string = "/home/spencer/.config/helix/runtime/grammars"
+QUERIES_PATH: string = "/nix/store/n9da8d...-helix-25.07.1/lib/runtime/queries"
+```
+
+Only works on one machine. CI and other users get no syntax highlighting.
+
+## Design
+
+### Config
+
+`thor.json` gets two optional directory paths:
+
+```json
+{
+  "grammar_dir": "~/.local/share/thor/grammars",
+  "queries_dir": "~/.local/share/thor/queries"
+}
+```
+
+These are thor's managed cache directories. Thor looks here first, and hardlinks discovered files into them.
+
+### Discovery flow (per language, lazy)
+
+When a code block with language X is encountered:
+
+1. **Check configured dir**: `grammar_dir/X.so` — if exists, use it
+2. **Search standard locations** (if not in configured dir):
+   - `$HELIX_RUNTIME/grammars/X.so`
+   - `~/.config/helix/runtime/grammars/X.so`
+   - `~/.local/share/nvim/site/parser/X.so`
+   - `/usr/lib/tree-sitter/X.so`
+   - `/usr/local/lib/X.so`
+3. **Hardlink** found file into `grammar_dir/X.so`
+4. **Cache** in `grammar_cache` (in-memory, per-run)
+
+Same flow for queries: `queries_dir/X/highlights.scm`, searching:
+- `$HELIX_RUNTIME/queries/X/highlights.scm`
+- `~/.config/helix/runtime/queries/X/highlights.scm`
+
+### Subsequent runs
+
+`grammar_dir/X.so` exists → skip search entirely. Fast cold start.
+
+### Staleness
+
+- User deletes file from `grammar_dir` → re-search on next run
+- Source file changes (Helix update) → hardlink still points to old inode until source is deleted (Nix GC) or user manually clears
+- Hardlink fails (cross-filesystem) → fall back to copy or symlink (TBD)
+
+### HTML/CSS
+
+Already statically linked via `mkGrammarStaticLib` in the flake. No change needed — `builtin_language()` handles them before the search path logic.
+
+## Open questions
+
+1. **Default location**: `~/.local/share/thor/grammars` (XDG) or project-local `.thor/grammars`?
+2. **Hardlink fallback**: copy vs symlink when cross-filesystem?
+3. **Nix integration**: Flake sets `grammar_dir`/`queries_dir` in derivation env, or user configures manually?
+4. **Per-language override**: Should `thor.json` support per-language paths in addition to the directory? (e.g., `"grammars": {"odin": "/custom/path/odin.so"}`)
+
+## Files changed
+
+| File | Change |
+|---|---|
+| `treesitter/treesitter.odin` | Delete `GRAPHS_PATH`/`QUERIES_PATH` globals. Add `find_grammar(lang)` and `find_query(lang)` search procs. Update `ensure_parser` and `load_grammar` to use them. Add hardlink-to-configured-dir logic. |
+| `site.odin` | `Config_File` and `Site` get `grammar_dir`/`queries_dir` fields. |
+| `thor.json` | Optional `grammar_dir`/`queries_dir` fields. |
@@ -56,26 +56,21 @@ foreign import libdl "system:dl"
 foreign import html_grammar "system:tree-sitter-html"
 foreign import css_grammar "system:tree-sitter-css"

-@(link_prefix="ts_")
+@(link_prefix = "ts_")
 foreign lib {
 	parser_new :: proc() -> Parser ---
 	parser_delete :: proc(self: Parser) ---
 	parser_set_language :: proc(self: Parser, language: Language) -> bool ---
-	parser_parse_string :: proc(
-		self: Parser,
-		old_tree: Tree,
-		string: cstring,
-		length: u32,
-	) -> Tree ---
+	parser_parse_string :: proc(self: Parser, old_tree: Tree, string: cstring, length: u32) -> Tree ---
 }

-@(link_prefix="ts_")
+@(link_prefix = "ts_")
 foreign lib {
 	tree_root_node :: proc(self: Tree) -> Node ---
 	tree_delete :: proc(self: Tree) ---
 }

-@(link_prefix="ts_")
+@(link_prefix = "ts_")
 foreign lib {
 	node_start_byte :: proc(self: Node) -> u32 ---
 	node_end_byte :: proc(self: Node) -> u32 ---
@@ -90,37 +85,19 @@ foreign lib {
 	node_parent :: proc(self: Node) -> Node ---
 }

-@(link_prefix="ts_")
+@(link_prefix = "ts_")
 foreign lib {
-	 query_new :: proc(
-		language: Language,
-		source: cstring,
-		source_len: u32,
-		error_offset: ^u32,
-		error_type: ^Query_Error,
-	) -> Query ---
+	query_new :: proc(language: Language, source: cstring, source_len: u32, error_offset: ^u32, error_type: ^Query_Error) -> Query ---
 	query_delete :: proc(self: Query) ---
-	query_capture_name_for_id :: proc(
-		self: Query,
-		index: u32,
-		length: ^u32,
-	) -> cstring ---
+	query_capture_name_for_id :: proc(self: Query, index: u32, length: ^u32) -> cstring ---
 }

-@(link_prefix="ts_")
+@(link_prefix = "ts_")
 foreign lib {
 	query_cursor_new :: proc() -> Query_Cursor ---
 	query_cursor_delete :: proc(self: Query_Cursor) ---
-	query_cursor_exec :: proc(
-		self: Query_Cursor,
-		query: Query,
-		node: Node,
-	) ---
-	query_cursor_next_capture :: proc(
-		self: Query_Cursor,
-		match: ^Query_Match,
-		capture_index: ^u32,
-	) -> bool ---
+	query_cursor_exec :: proc(self: Query_Cursor, query: Query, node: Node) ---
+	query_cursor_next_capture :: proc(self: Query_Cursor, match: ^Query_Match, capture_index: ^u32) -> bool ---
 }

 foreign libdl {
@@ -250,22 +227,25 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {

 	err_offset: u32
 	err_type: Query_Error
-	query := query_new(
-		gc.language,
-		query_c,
-		u32(len(query_src)),
-		&err_offset,
-		&err_type,
-	)
+	query := query_new(gc.language, query_c, u32(len(query_src)), &err_offset, &err_type)
 	if query == nil {
 		tok := extract_query_token(query_src, err_offset)
 		cause := fmt.tprintf("query error at byte %d (type %v)", err_offset, err_type)
 		#partial switch err_type {
 		case .NodeType:
 			if tok != "" {
-				cause = fmt.tprintf("query references unknown node type '%s' (byte %d); the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions", tok, err_offset, lang)
+				cause = fmt.tprintf(
+					"query references unknown node type '%s' (byte %d); the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions",
+					tok,
+					err_offset,
+					lang,
+				)
 			} else {
-				cause = fmt.tprintf("query references an unknown node type at byte %d; the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions", err_offset, lang)
+				cause = fmt.tprintf(
+					"query references an unknown node type at byte %d; the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions",
+					err_offset,
+					lang,
+				)
 			}
 		case .Field:
 			cause = fmt.tprintf("query references unknown field '%s' at byte %d", tok, err_offset)
@@ -308,8 +288,13 @@ extract_query_token :: proc(src: []byte, offset: u32) -> string {
 	end := offset
 	for int(end) < len(src) {
 		c := src[end]
-		is_ident := (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') ||
-			(c >= '0' && c <= '9') || c == '_' || c == '-' || c == '.'
+		is_ident :=
+			(c >= 'A' && c <= 'Z') ||
+			(c >= 'a' && c <= 'z') ||
+			(c >= '0' && c <= '9') ||
+			c == '_' ||
+			c == '-' ||
+			c == '.'
 		if !is_ident do break
 		end += 1
 	}
@@ -331,3 +316,4 @@ helix_version_from_path :: proc(path: string) -> string {
 	if end <= start do return ""
 	return path[start:end]
 }
+