From 4f0b5c929db66765eaf3070477e0d10425704ab7 Mon Sep 17 00:00:00 2001 From: Spencer Brower <6729162+sbrow@users.noreply.github.com> Date: Fri, 24 Jul 2026 16:50:33 -0400 Subject: [PATCH] feat: The queries for HTML and CSS are now baked in to the `thor` binary. --- GRAMMARS.md | 74 ------------------------- TODOS.md | 14 +++++ treesitter/queries/css/highlights.scm | 76 ++++++++++++++++++++++++++ treesitter/queries/html/highlights.scm | 13 +++++ treesitter/queries/html/injections.scm | 7 +++ treesitter/treesitter.odin | 44 ++++++++++----- 6 files changed, 141 insertions(+), 87 deletions(-) delete mode 100644 GRAMMARS.md create mode 100644 treesitter/queries/css/highlights.scm create mode 100644 treesitter/queries/html/highlights.scm create mode 100644 treesitter/queries/html/injections.scm diff --git a/GRAMMARS.md b/GRAMMARS.md deleted file mode 100644 index 14c6a19..0000000 --- a/GRAMMARS.md +++ /dev/null @@ -1,74 +0,0 @@ -# Tree-sitter Grammar Discovery - -## Problem - -Grammar (`.so`) and query (`.scm`) paths are hardcoded to the developer's machine: - -```odin -GRAPHS_PATH: string = "/home/spencer/.config/helix/runtime/grammars" -QUERIES_PATH: string = "/nix/store/n9da8d...-helix-25.07.1/lib/runtime/queries" -``` - -Only works on one machine. CI and other users get no syntax highlighting. - -## Design - -### Config - -`thor.json` gets two optional directory paths: - -```json -{ - "grammar_dir": "~/.local/share/thor/grammars", - "queries_dir": "~/.local/share/thor/queries" -} -``` - -These are thor's managed cache directories. Thor looks here first, and hardlinks discovered files into them. - -### Discovery flow (per language, lazy) - -When a code block with language X is encountered: - -1. **Check configured dir**: `grammar_dir/X.so` — if exists, use it -2. **Search standard locations** (if not in configured dir): - - `$HELIX_RUNTIME/grammars/X.so` - - `~/.config/helix/runtime/grammars/X.so` - - `~/.local/share/nvim/site/parser/X.so` - - `/usr/lib/tree-sitter/X.so` - - `/usr/local/lib/X.so` -3. **Hardlink** found file into `grammar_dir/X.so` -4. **Cache** in `grammar_cache` (in-memory, per-run) - -Same flow for queries: `queries_dir/X/highlights.scm`, searching: -- `$HELIX_RUNTIME/queries/X/highlights.scm` -- `~/.config/helix/runtime/queries/X/highlights.scm` - -### Subsequent runs - -`grammar_dir/X.so` exists → skip search entirely. Fast cold start. - -### Staleness - -- User deletes file from `grammar_dir` → re-search on next run -- Source file changes (Helix update) → hardlink still points to old inode until source is deleted (Nix GC) or user manually clears -- Hardlink fails (cross-filesystem) → fall back to copy or symlink (TBD) - -### HTML/CSS - -Already statically linked via `mkGrammarStaticLib` in the flake. No change needed — `builtin_language()` handles them before the search path logic. - -## Open questions - -1. **Default location**: `~/.local/share/thor/grammars` (XDG) or project-local `.thor/grammars`? -2. **Hardlink fallback**: copy vs symlink when cross-filesystem? -3. **Nix integration**: Flake sets `grammar_dir`/`queries_dir` in derivation env, or user configures manually? -4. **Per-language override**: Should `thor.json` support per-language paths in addition to the directory? (e.g., `"grammars": {"odin": "/custom/path/odin.so"}`) - -## Files changed - -| File | Change | -|---|---| -| `treesitter/treesitter.odin` | Delete `GRAPHS_PATH`/`QUERIES_PATH` globals. Add `find_grammar(lang)` and `find_query(lang)` search procs. Update `ensure_parser` and `load_grammar` to use them. Add hardlink-to-configured-dir logic. | -| `site.odin` | `Config_File` and `Site` get `grammar_dir`/`queries_dir` fields. | -| `thor.json` | Optional `grammar_dir`/`queries_dir` fields. | diff --git a/TODOS.md b/TODOS.md index fca3db7..b46eed4 100644 --- a/TODOS.md +++ b/TODOS.md @@ -53,6 +53,20 @@ ## General - [ ] get rid of the global variables in the `treesitter` package. +- [ ] Consider using `or_else` when applying default values to structs. i.e. +```odin +package main + +X :: struct { + foo: string +} + +main :: proc () { + x: X + + x.foo = x.foo or_else "bar" +} +``` - [ ] Integrity hash - Allows users to verify their output didn't change after upgrading to a new version - [ ] Content-hash fingerprinting for CSS and JS cache busting diff --git a/treesitter/queries/css/highlights.scm b/treesitter/queries/css/highlights.scm new file mode 100644 index 0000000..40c6586 --- /dev/null +++ b/treesitter/queries/css/highlights.scm @@ -0,0 +1,76 @@ +(comment) @comment + +(tag_name) @tag +(nesting_selector) @tag +(universal_selector) @tag + +"~" @operator +">" @operator +"+" @operator +"-" @operator +"*" @operator +"/" @operator +"=" @operator +"^=" @operator +"|=" @operator +"~=" @operator +"$=" @operator +"*=" @operator + +"and" @operator +"or" @operator +"not" @operator +"only" @operator + +(attribute_selector (plain_value) @string) + +((property_name) @variable + (#match? @variable "^--")) +((plain_value) @variable + (#match? @variable "^--")) + +(class_name) @property +(id_name) @property +(namespace_name) @property +(property_name) @property +(feature_name) @property + +(pseudo_element_selector (tag_name) @attribute) +(pseudo_class_selector (class_name) @attribute) +(attribute_name) @attribute + +(function_name) @function + +"@media" @keyword +"@import" @keyword +"@charset" @keyword +"@namespace" @keyword +"@supports" @keyword +"@keyframes" @keyword +(at_keyword) @keyword +(to) @keyword +(from) @keyword +(important) @keyword + +(string_value) @string +(color_value) @string.special + +(integer_value) @number +(float_value) @number +(unit) @type + +[ + "#" + "," + "." + ":" + "::" + ";" +] @punctuation.delimiter + +[ + "{" + ")" + "(" + "}" +] @punctuation.bracket diff --git a/treesitter/queries/html/highlights.scm b/treesitter/queries/html/highlights.scm new file mode 100644 index 0000000..ea0ff4e --- /dev/null +++ b/treesitter/queries/html/highlights.scm @@ -0,0 +1,13 @@ +(tag_name) @tag +(erroneous_end_tag_name) @tag.error +(doctype) @constant +(attribute_name) @attribute +(attribute_value) @string +(comment) @comment + +[ + "<" + ">" + "" +] @punctuation.bracket diff --git a/treesitter/queries/html/injections.scm b/treesitter/queries/html/injections.scm new file mode 100644 index 0000000..71e7c3a --- /dev/null +++ b/treesitter/queries/html/injections.scm @@ -0,0 +1,7 @@ +((script_element + (raw_text) @injection.content) + (#set! injection.language "javascript")) + +((style_element + (raw_text) @injection.content) + (#set! injection.language "css")) diff --git a/treesitter/treesitter.odin b/treesitter/treesitter.odin index 86a8b10..eaa7d17 100644 --- a/treesitter/treesitter.odin +++ b/treesitter/treesitter.odin @@ -9,6 +9,9 @@ import "core:strings" grammar_dir: string query_dir: string +HTML_HIGHLIGHTS :: #load(#directory + "queries/html/highlights.scm", string) +CSS_HIGHLIGHTS :: #load(#directory + "queries/css/highlights.scm", string) + Language :: distinct rawptr Parser :: distinct rawptr Tree :: distinct rawptr @@ -137,6 +140,30 @@ builtin_language :: proc(lang: string) -> (language: Language, ok: bool) { return } +// load_query returns the highlight query source for a language. Builtin +// languages (html/css) are baked into the binary via `#load`; all others are +// read from the runtime `query_dir`. `path` is the on-disk location for +// diagnostics ("(builtin)" for embedded queries). Mirrors `ensure_parser`. +load_query :: proc(lang: string) -> (src: string, path: string, ok: bool) { + switch lang { + case "html": + return HTML_HIGHLIGHTS, "(builtin)", true + case "css": + return CSS_HIGHLIGHTS, "(builtin)", true + } + if query_dir == "" { + log.warnf("treesitter: no query path set, skipping %s", lang) + return "", "", false + } + path = fmt.tprintf("%s/%s/highlights.scm", query_dir, lang) + raw, err := os.read_entire_file_from_path(path, context.allocator) + if err != nil { + log.warnf("treesitter: cannot load query %s", path) + return "", "", false + } + return string(raw), path, true +} + ensure_parser :: proc(lang: string) -> ^Grammar_Cache { if grammar_cache == nil { grammar_cache = make(map[string]^Grammar_Cache) @@ -208,28 +235,19 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache { return nil } - if query_dir == "" { - log.warnf("treesitter: no query path set, skipping %s", lang) + query_src, query_path, ok := load_query(lang) + if !ok { gc.query_failed = true return nil } - - query_path := fmt.tprintf("%s/%s/highlights.scm", query_dir, lang) - query_src, err := os.read_entire_file_from_path(query_path, context.allocator) - if err != nil { - log.warnf("treesitter: cannot load query %s", query_path) - gc.query_failed = true - return nil - } - query_str := string(query_src) - query_c := strings.clone_to_cstring(query_str) + query_c := strings.clone_to_cstring(query_src) defer delete(query_c) err_offset: u32 err_type: Query_Error query := query_new(gc.language, query_c, u32(len(query_src)), &err_offset, &err_type) if query == nil { - tok := extract_query_token(query_src, err_offset) + tok := extract_query_token(transmute([]byte)query_src, err_offset) cause := fmt.tprintf("query error at byte %d (type %v)", err_offset, err_type) #partial switch err_type { case .NodeType: