mirror of
https://github.com/sbrow/thor.git
synced 2026-08-26 11:23:32 -04:00
feat: The queries for HTML and CSS are now baked in to the thor binary.
This commit is contained in:
Spencer Brower
-74
@@ -1,74 +0,0 @@
|
||||
# Tree-sitter Grammar Discovery
|
||||
|
||||
## Problem
|
||||
|
||||
Grammar (`.so`) and query (`.scm`) paths are hardcoded to the developer's machine:
|
||||
|
||||
```odin
|
||||
GRAPHS_PATH: string = "/home/spencer/.config/helix/runtime/grammars"
|
||||
QUERIES_PATH: string = "/nix/store/n9da8d...-helix-25.07.1/lib/runtime/queries"
|
||||
```
|
||||
|
||||
Only works on one machine. CI and other users get no syntax highlighting.
|
||||
|
||||
## Design
|
||||
|
||||
### Config
|
||||
|
||||
`thor.json` gets two optional directory paths:
|
||||
|
||||
```json
|
||||
{
|
||||
"grammar_dir": "~/.local/share/thor/grammars",
|
||||
"queries_dir": "~/.local/share/thor/queries"
|
||||
}
|
||||
```
|
||||
|
||||
These are thor's managed cache directories. Thor looks here first, and hardlinks discovered files into them.
|
||||
|
||||
### Discovery flow (per language, lazy)
|
||||
|
||||
When a code block with language X is encountered:
|
||||
|
||||
1. **Check configured dir**: `grammar_dir/X.so` — if exists, use it
|
||||
2. **Search standard locations** (if not in configured dir):
|
||||
- `$HELIX_RUNTIME/grammars/X.so`
|
||||
- `~/.config/helix/runtime/grammars/X.so`
|
||||
- `~/.local/share/nvim/site/parser/X.so`
|
||||
- `/usr/lib/tree-sitter/X.so`
|
||||
- `/usr/local/lib/X.so`
|
||||
3. **Hardlink** found file into `grammar_dir/X.so`
|
||||
4. **Cache** in `grammar_cache` (in-memory, per-run)
|
||||
|
||||
Same flow for queries: `queries_dir/X/highlights.scm`, searching:
|
||||
- `$HELIX_RUNTIME/queries/X/highlights.scm`
|
||||
- `~/.config/helix/runtime/queries/X/highlights.scm`
|
||||
|
||||
### Subsequent runs
|
||||
|
||||
`grammar_dir/X.so` exists → skip search entirely. Fast cold start.
|
||||
|
||||
### Staleness
|
||||
|
||||
- User deletes file from `grammar_dir` → re-search on next run
|
||||
- Source file changes (Helix update) → hardlink still points to old inode until source is deleted (Nix GC) or user manually clears
|
||||
- Hardlink fails (cross-filesystem) → fall back to copy or symlink (TBD)
|
||||
|
||||
### HTML/CSS
|
||||
|
||||
Already statically linked via `mkGrammarStaticLib` in the flake. No change needed — `builtin_language()` handles them before the search path logic.
|
||||
|
||||
## Open questions
|
||||
|
||||
1. **Default location**: `~/.local/share/thor/grammars` (XDG) or project-local `.thor/grammars`?
|
||||
2. **Hardlink fallback**: copy vs symlink when cross-filesystem?
|
||||
3. **Nix integration**: Flake sets `grammar_dir`/`queries_dir` in derivation env, or user configures manually?
|
||||
4. **Per-language override**: Should `thor.json` support per-language paths in addition to the directory? (e.g., `"grammars": {"odin": "/custom/path/odin.so"}`)
|
||||
|
||||
## Files changed
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `treesitter/treesitter.odin` | Delete `GRAPHS_PATH`/`QUERIES_PATH` globals. Add `find_grammar(lang)` and `find_query(lang)` search procs. Update `ensure_parser` and `load_grammar` to use them. Add hardlink-to-configured-dir logic. |
|
||||
| `site.odin` | `Config_File` and `Site` get `grammar_dir`/`queries_dir` fields. |
|
||||
| `thor.json` | Optional `grammar_dir`/`queries_dir` fields. |
|
||||
@@ -53,6 +53,20 @@
|
||||
|
||||
## General
|
||||
- [ ] get rid of the global variables in the `treesitter` package.
|
||||
- [ ] Consider using `or_else` when applying default values to structs. i.e.
|
||||
```odin
|
||||
package main
|
||||
|
||||
X :: struct {
|
||||
foo: string
|
||||
}
|
||||
|
||||
main :: proc () {
|
||||
x: X
|
||||
|
||||
x.foo = x.foo or_else "bar"
|
||||
}
|
||||
```
|
||||
- [ ] Integrity hash
|
||||
- Allows users to verify their output didn't change after upgrading to a new version
|
||||
- [ ] Content-hash fingerprinting for CSS and JS cache busting
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
(comment) @comment
|
||||
|
||||
(tag_name) @tag
|
||||
(nesting_selector) @tag
|
||||
(universal_selector) @tag
|
||||
|
||||
"~" @operator
|
||||
">" @operator
|
||||
"+" @operator
|
||||
"-" @operator
|
||||
"*" @operator
|
||||
"/" @operator
|
||||
"=" @operator
|
||||
"^=" @operator
|
||||
"|=" @operator
|
||||
"~=" @operator
|
||||
"$=" @operator
|
||||
"*=" @operator
|
||||
|
||||
"and" @operator
|
||||
"or" @operator
|
||||
"not" @operator
|
||||
"only" @operator
|
||||
|
||||
(attribute_selector (plain_value) @string)
|
||||
|
||||
((property_name) @variable
|
||||
(#match? @variable "^--"))
|
||||
((plain_value) @variable
|
||||
(#match? @variable "^--"))
|
||||
|
||||
(class_name) @property
|
||||
(id_name) @property
|
||||
(namespace_name) @property
|
||||
(property_name) @property
|
||||
(feature_name) @property
|
||||
|
||||
(pseudo_element_selector (tag_name) @attribute)
|
||||
(pseudo_class_selector (class_name) @attribute)
|
||||
(attribute_name) @attribute
|
||||
|
||||
(function_name) @function
|
||||
|
||||
"@media" @keyword
|
||||
"@import" @keyword
|
||||
"@charset" @keyword
|
||||
"@namespace" @keyword
|
||||
"@supports" @keyword
|
||||
"@keyframes" @keyword
|
||||
(at_keyword) @keyword
|
||||
(to) @keyword
|
||||
(from) @keyword
|
||||
(important) @keyword
|
||||
|
||||
(string_value) @string
|
||||
(color_value) @string.special
|
||||
|
||||
(integer_value) @number
|
||||
(float_value) @number
|
||||
(unit) @type
|
||||
|
||||
[
|
||||
"#"
|
||||
","
|
||||
"."
|
||||
":"
|
||||
"::"
|
||||
";"
|
||||
] @punctuation.delimiter
|
||||
|
||||
[
|
||||
"{"
|
||||
")"
|
||||
"("
|
||||
"}"
|
||||
] @punctuation.bracket
|
||||
@@ -0,0 +1,13 @@
|
||||
(tag_name) @tag
|
||||
(erroneous_end_tag_name) @tag.error
|
||||
(doctype) @constant
|
||||
(attribute_name) @attribute
|
||||
(attribute_value) @string
|
||||
(comment) @comment
|
||||
|
||||
[
|
||||
"<"
|
||||
">"
|
||||
"</"
|
||||
"/>"
|
||||
] @punctuation.bracket
|
||||
@@ -0,0 +1,7 @@
|
||||
((script_element
|
||||
(raw_text) @injection.content)
|
||||
(#set! injection.language "javascript"))
|
||||
|
||||
((style_element
|
||||
(raw_text) @injection.content)
|
||||
(#set! injection.language "css"))
|
||||
+31
-13
@@ -9,6 +9,9 @@ import "core:strings"
|
||||
grammar_dir: string
|
||||
query_dir: string
|
||||
|
||||
HTML_HIGHLIGHTS :: #load(#directory + "queries/html/highlights.scm", string)
|
||||
CSS_HIGHLIGHTS :: #load(#directory + "queries/css/highlights.scm", string)
|
||||
|
||||
Language :: distinct rawptr
|
||||
Parser :: distinct rawptr
|
||||
Tree :: distinct rawptr
|
||||
@@ -137,6 +140,30 @@ builtin_language :: proc(lang: string) -> (language: Language, ok: bool) {
|
||||
return
|
||||
}
|
||||
|
||||
// load_query returns the highlight query source for a language. Builtin
|
||||
// languages (html/css) are baked into the binary via `#load`; all others are
|
||||
// read from the runtime `query_dir`. `path` is the on-disk location for
|
||||
// diagnostics ("(builtin)" for embedded queries). Mirrors `ensure_parser`.
|
||||
load_query :: proc(lang: string) -> (src: string, path: string, ok: bool) {
|
||||
switch lang {
|
||||
case "html":
|
||||
return HTML_HIGHLIGHTS, "(builtin)", true
|
||||
case "css":
|
||||
return CSS_HIGHLIGHTS, "(builtin)", true
|
||||
}
|
||||
if query_dir == "" {
|
||||
log.warnf("treesitter: no query path set, skipping %s", lang)
|
||||
return "", "", false
|
||||
}
|
||||
path = fmt.tprintf("%s/%s/highlights.scm", query_dir, lang)
|
||||
raw, err := os.read_entire_file_from_path(path, context.allocator)
|
||||
if err != nil {
|
||||
log.warnf("treesitter: cannot load query %s", path)
|
||||
return "", "", false
|
||||
}
|
||||
return string(raw), path, true
|
||||
}
|
||||
|
||||
ensure_parser :: proc(lang: string) -> ^Grammar_Cache {
|
||||
if grammar_cache == nil {
|
||||
grammar_cache = make(map[string]^Grammar_Cache)
|
||||
@@ -208,28 +235,19 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {
|
||||
return nil
|
||||
}
|
||||
|
||||
if query_dir == "" {
|
||||
log.warnf("treesitter: no query path set, skipping %s", lang)
|
||||
query_src, query_path, ok := load_query(lang)
|
||||
if !ok {
|
||||
gc.query_failed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
query_path := fmt.tprintf("%s/%s/highlights.scm", query_dir, lang)
|
||||
query_src, err := os.read_entire_file_from_path(query_path, context.allocator)
|
||||
if err != nil {
|
||||
log.warnf("treesitter: cannot load query %s", query_path)
|
||||
gc.query_failed = true
|
||||
return nil
|
||||
}
|
||||
query_str := string(query_src)
|
||||
query_c := strings.clone_to_cstring(query_str)
|
||||
query_c := strings.clone_to_cstring(query_src)
|
||||
defer delete(query_c)
|
||||
|
||||
err_offset: u32
|
||||
err_type: Query_Error
|
||||
query := query_new(gc.language, query_c, u32(len(query_src)), &err_offset, &err_type)
|
||||
if query == nil {
|
||||
tok := extract_query_token(query_src, err_offset)
|
||||
tok := extract_query_token(transmute([]byte)query_src, err_offset)
|
||||
cause := fmt.tprintf("query error at byte %d (type %v)", err_offset, err_type)
|
||||
#partial switch err_type {
|
||||
case .NodeType:
|
||||
|
||||
Reference in New Issue
Block a user