spencer/thor
1
0
Fork 0
mirror of https://github.com/sbrow/thor.git synced 2026-08-26 11:23:32 -04:00
Code Issues Packages Projects Releases Wiki Activity

chore: Added plan for dynamically loading grammars.

Browse Source
This commit is contained in:
Spencer Brower
2026-07-24 13:54:24 -04:00
parent 91623e8992
commit 961b3f41b0
2 changed files with 121 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
GRAMMARS.md
+74
View File
@@ -0,0 +1,74 @@
# Tree-sitter Grammar Discovery
## Problem
Grammar (`.so`) and query (`.scm`) paths are hardcoded to the developer's machine:
```odin
GRAPHS_PATH: string = "/home/spencer/.config/helix/runtime/grammars"
QUERIES_PATH: string = "/nix/store/n9da8d...-helix-25.07.1/lib/runtime/queries"
```
Only works on one machine. CI and other users get no syntax highlighting.
## Design
### Config
`thor.json` gets two optional directory paths:
```json
{
"grammar_dir": "~/.local/share/thor/grammars",
"queries_dir": "~/.local/share/thor/queries"
}
```
These are thor's managed cache directories. Thor looks here first, and hardlinks discovered files into them.
### Discovery flow (per language, lazy)
When a code block with language X is encountered:
1. **Check configured dir**: `grammar_dir/X.so` — if exists, use it
2. **Search standard locations** (if not in configured dir):
- `$HELIX_RUNTIME/grammars/X.so`
- `~/.config/helix/runtime/grammars/X.so`
- `~/.local/share/nvim/site/parser/X.so`
- `/usr/lib/tree-sitter/X.so`
- `/usr/local/lib/X.so`
3. **Hardlink** found file into `grammar_dir/X.so`
4. **Cache** in `grammar_cache` (in-memory, per-run)
Same flow for queries: `queries_dir/X/highlights.scm`, searching:
- `$HELIX_RUNTIME/queries/X/highlights.scm`
- `~/.config/helix/runtime/queries/X/highlights.scm`
### Subsequent runs
`grammar_dir/X.so` exists → skip search entirely. Fast cold start.
### Staleness
- User deletes file from `grammar_dir` → re-search on next run
- Source file changes (Helix update) → hardlink still points to old inode until source is deleted (Nix GC) or user manually clears
- Hardlink fails (cross-filesystem) → fall back to copy or symlink (TBD)
### HTML/CSS
Already statically linked via `mkGrammarStaticLib` in the flake. No change needed — `builtin_language()` handles them before the search path logic.
## Open questions
1. **Default location**: `~/.local/share/thor/grammars` (XDG) or project-local `.thor/grammars`?
2. **Hardlink fallback**: copy vs symlink when cross-filesystem?
3. **Nix integration**: Flake sets `grammar_dir`/`queries_dir` in derivation env, or user configures manually?
4. **Per-language override**: Should `thor.json` support per-language paths in addition to the directory? (e.g., `"grammars": {"odin": "/custom/path/odin.so"}`)
## Files changed
| File | Change |
|---|---|
| `treesitter/treesitter.odin` | Delete `GRAPHS_PATH`/`QUERIES_PATH` globals. Add `find_grammar(lang)` and `find_query(lang)` search procs. Update `ensure_parser` and `load_grammar` to use them. Add hardlink-to-configured-dir logic. |
| `site.odin` | `Config_File` and `Site` get `grammar_dir`/`queries_dir` fields. |
| `thor.json` | Optional `grammar_dir`/`queries_dir` fields. |
treesitter/treesitter.odin
+47 -61
View File
@@ -21,9 +21,9 @@ Point :: struct {
}
Node :: struct {
ctx: [4]u32,
id: rawptr,
tree: rawptr,
ctx: [4]u32,
id: rawptr,
tree: rawptr,
}
Query_Capture :: struct {
@@ -40,7 +40,7 @@ Query_Match :: struct {
}
Query_Error :: enum c.int {
None = 0,
None = 0,
Syntax,
NodeType,
Field,
@@ -56,71 +56,48 @@ foreign import libdl "system:dl"
foreign import html_grammar "system:tree-sitter-html"
foreign import css_grammar "system:tree-sitter-css"
@(link_prefix="ts_")
@(link_prefix = "ts_")
foreign lib {
parser_new :: proc() -> Parser ---
parser_delete :: proc(self: Parser) ---
parser_set_language :: proc(self: Parser, language: Language) -> bool ---
parser_parse_string :: proc(
self: Parser,
old_tree: Tree,
string: cstring,
length: u32,
) -> Tree ---
parser_parse_string :: proc(self: Parser, old_tree: Tree, string: cstring, length: u32) -> Tree ---
}
@(link_prefix="ts_")
@(link_prefix = "ts_")
foreign lib {
tree_root_node :: proc(self: Tree) -> Node ---
tree_delete :: proc(self: Tree) ---
tree_root_node :: proc(self: Tree) -> Node ---
tree_delete :: proc(self: Tree) ---
}
@(link_prefix="ts_")
@(link_prefix = "ts_")
foreign lib {
node_start_byte :: proc(self: Node) -> u32 ---
node_end_byte :: proc(self: Node) -> u32 ---
node_has_error :: proc(self: Node) -> bool ---
node_is_error :: proc(self: Node) -> bool ---
node_child_count :: proc(self: Node) -> u32 ---
node_child :: proc(self: Node, child_index: u32) -> Node ---
node_named_child_count :: proc(self: Node) -> u32 ---
node_named_child :: proc(self: Node, child_index: u32) -> Node ---
node_start_point :: proc(self: Node) -> Point ---
node_type :: proc(self: Node) -> cstring ---
node_parent :: proc(self: Node) -> Node ---
node_start_byte :: proc(self: Node) -> u32 ---
node_end_byte :: proc(self: Node) -> u32 ---
node_has_error :: proc(self: Node) -> bool ---
node_is_error :: proc(self: Node) -> bool ---
node_child_count :: proc(self: Node) -> u32 ---
node_child :: proc(self: Node, child_index: u32) -> Node ---
node_named_child_count :: proc(self: Node) -> u32 ---
node_named_child :: proc(self: Node, child_index: u32) -> Node ---
node_start_point :: proc(self: Node) -> Point ---
node_type :: proc(self: Node) -> cstring ---
node_parent :: proc(self: Node) -> Node ---
}
@(link_prefix="ts_")
@(link_prefix = "ts_")
foreign lib {
query_new :: proc(
language: Language,
source: cstring,
source_len: u32,
error_offset: ^u32,
error_type: ^Query_Error,
) -> Query ---
query_new :: proc(language: Language, source: cstring, source_len: u32, error_offset: ^u32, error_type: ^Query_Error) -> Query ---
query_delete :: proc(self: Query) ---
query_capture_name_for_id :: proc(
self: Query,
index: u32,
length: ^u32,
) -> cstring ---
query_capture_name_for_id :: proc(self: Query, index: u32, length: ^u32) -> cstring ---
}
@(link_prefix="ts_")
@(link_prefix = "ts_")
foreign lib {
query_cursor_new :: proc() -> Query_Cursor ---
query_cursor_delete :: proc(self: Query_Cursor) ---
query_cursor_exec :: proc(
self: Query_Cursor,
query: Query,
node: Node,
) ---
query_cursor_next_capture :: proc(
self: Query_Cursor,
match: ^Query_Match,
capture_index: ^u32,
) -> bool ---
query_cursor_exec :: proc(self: Query_Cursor, query: Query, node: Node) ---
query_cursor_next_capture :: proc(self: Query_Cursor, match: ^Query_Match, capture_index: ^u32) -> bool ---
}
foreign libdl {
@@ -250,22 +227,25 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {
err_offset: u32
err_type: Query_Error
query := query_new(
gc.language,
query_c,
u32(len(query_src)),
&err_offset,
&err_type,
)
query := query_new(gc.language, query_c, u32(len(query_src)), &err_offset, &err_type)
if query == nil {
tok := extract_query_token(query_src, err_offset)
cause := fmt.tprintf("query error at byte %d (type %v)", err_offset, err_type)
#partial switch err_type {
case .NodeType:
if tok != "" {
cause = fmt.tprintf("query references unknown node type '%s' (byte %d); the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions", tok, err_offset, lang)
cause = fmt.tprintf(
"query references unknown node type '%s' (byte %d); the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions",
tok,
err_offset,
lang,
)
} else {
cause = fmt.tprintf("query references an unknown node type at byte %d; the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions", err_offset, lang)
cause = fmt.tprintf(
"query references an unknown node type at byte %d; the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions",
err_offset,
lang,
)
}
case .Field:
cause = fmt.tprintf("query references unknown field '%s' at byte %d", tok, err_offset)
@@ -308,8 +288,13 @@ extract_query_token :: proc(src: []byte, offset: u32) -> string {
end := offset
for int(end) < len(src) {
c := src[end]
is_ident := (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') ||
(c >= '0' && c <= '9') || c == '_' || c == '-' || c == '.'
is_ident :=
(c >= 'A' && c <= 'Z') ||
(c >= 'a' && c <= 'z') ||
(c >= '0' && c <= '9') ||
c == '_' ||
c == '-' ||
c == '.'
if !is_ident do break
end += 1
}
@@ -331,3 +316,4 @@ helix_version_from_path :: proc(path: string) -> string {
if end <= start do return ""
return path[start:end]
}