package treesitter import "core:c" import "core:fmt" import "core:log" import "core:mem" import "core:os" import "core:strings" import "core:sync" grammar_dir: string query_dir: string HTML_HIGHLIGHTS :: #load(#directory + "queries/html/highlights.scm", string) CSS_HIGHLIGHTS :: #load(#directory + "queries/css/highlights.scm", string) Language :: distinct rawptr Parser :: distinct rawptr Tree :: distinct rawptr Query :: distinct rawptr Query_Cursor :: distinct rawptr Point :: struct { row: u32, column: u32, } Node :: struct { ctx: [4]u32, id: rawptr, tree: rawptr, } Query_Capture :: struct { node: Node, index: u32, _: u32, } Query_Match :: struct { id: u32, pattern_index: u16, capture_count: u16, captures: [^]Query_Capture, } Query_Error :: enum c.int { None = 0, Syntax, NodeType, Field, Capture, Structure, Language, } RTLD_LAZY :: c.int(1) foreign import lib "system:tree-sitter" foreign import libdl "system:dl" foreign import html_grammar "system:tree-sitter-html" foreign import css_grammar "system:tree-sitter-css" @(link_prefix = "ts_") foreign lib { parser_new :: proc() -> Parser --- parser_delete :: proc(self: Parser) --- parser_set_language :: proc(self: Parser, language: Language) -> bool --- parser_parse_string :: proc(self: Parser, old_tree: Tree, string: cstring, length: u32) -> Tree --- } @(link_prefix = "ts_") foreign lib { tree_root_node :: proc(self: Tree) -> Node --- tree_delete :: proc(self: Tree) --- } @(link_prefix = "ts_") foreign lib { node_start_byte :: proc(self: Node) -> u32 --- node_end_byte :: proc(self: Node) -> u32 --- node_has_error :: proc(self: Node) -> bool --- node_is_error :: proc(self: Node) -> bool --- node_child_count :: proc(self: Node) -> u32 --- node_child :: proc(self: Node, child_index: u32) -> Node --- node_named_child_count :: proc(self: Node) -> u32 --- node_named_child :: proc(self: Node, child_index: u32) -> Node --- node_start_point :: proc(self: Node) -> Point --- node_type :: proc(self: Node) -> cstring --- node_parent :: proc(self: Node) -> Node --- } @(link_prefix = "ts_") foreign lib { query_new :: proc(language: Language, source: cstring, source_len: u32, error_offset: ^u32, error_type: ^Query_Error) -> Query --- query_delete :: proc(self: Query) --- query_capture_name_for_id :: proc(self: Query, index: u32, length: ^u32) -> cstring --- } @(link_prefix = "ts_") foreign lib { query_cursor_new :: proc() -> Query_Cursor --- query_cursor_delete :: proc(self: Query_Cursor) --- query_cursor_exec :: proc(self: Query_Cursor, query: Query, node: Node) --- query_cursor_next_capture :: proc(self: Query_Cursor, match: ^Query_Match, capture_index: ^u32) -> bool --- } foreign libdl { dlopen :: proc(filename: cstring, flags: c.int) -> rawptr --- dlsym :: proc(handle: rawptr, symbol: cstring) -> rawptr --- dlclose :: proc(handle: rawptr) -> c.int --- } foreign html_grammar { tree_sitter_html :: proc() -> Language --- } foreign css_grammar { tree_sitter_css :: proc() -> Language --- } // A Grammar bundles only the immutable, shareable tree-sitter objects for a // language: `language` (from tree_sitter_x() or a dlopen'd .so) and `query` (the // compiled highlight query). Both are immutable after construction and safe to // read from any number of threads. The stateful, single-thread-use objects — // parser and query cursor — are deliberately NOT stored here; the caller creates // and owns those (see `open_parser`). See GRAMMAR_CACHE_THREADING.md. Grammar :: struct { language: Language, query: Query, query_failed: bool, } Get_Language_Proc :: #type proc() -> Language SPALL :: #config(SPALL, false) // registry is the process-lifetime cache of immutable Grammars, keyed by language // name. It is the library's ONLY shared mutable state; `mu` serializes all access. // A nil value memoizes "tried and unavailable" so misses don't re-dlopen. Parsers // live in the caller, so the registry holds nothing single-thread-use. Grammar_Registry :: struct { mu: sync.Mutex, grammars: map[string]^Grammar, allocator: mem.Allocator, } registry: Grammar_Registry // init_persistent is optional — grammar() lazily initializes the registry on first // use — but callers (main) may call it to bind the cache early. The allocator is // pinned to the OS heap, NOT context.allocator, so the cache survives any transient // allocator (per-build arena, per-test tracking) that happens to be live. init_persistent :: proc() { sync.mutex_lock(®istry.mu) defer sync.mutex_unlock(®istry.mu) ensure_registry() } // ensure_registry binds the registry to the heap on first use. Caller must hold mu. @(private) ensure_registry :: proc() { if registry.grammars == nil { registry.allocator = os.heap_allocator() registry.grammars = make(map[string]^Grammar, registry.allocator) } } // grammar returns the immutable Grammar for `lang`, loading it on first use. // Thread-safe: concurrent callers serialize on `mu`. Returns (nil, false) when the // grammar cannot be loaded. The returned pointer is immutable — callers may cache // and read it (its `language`/`query`) without any further locking. grammar :: proc(lang: string) -> (^Grammar, bool) { sync.mutex_lock(®istry.mu) defer sync.mutex_unlock(®istry.mu) ensure_registry() if g, seen := registry.grammars[lang]; seen { return g, g != nil } g := build_grammar(lang) registry.grammars[lang] = g return g, g != nil } // build_grammar loads a language and compiles its highlight query. Caller must // hold mu. Returns nil if the language itself cannot be loaded; a grammar whose // query is absent/failed is still returned (parsing works, highlighting does not). @(private) build_grammar :: proc(lang: string) -> ^Grammar { language, ok := load_language(lang) if !ok { return nil } g := new(Grammar, registry.allocator) g.language = language if query, qok := compile_query(lang, language); qok { g.query = query } else { g.query_failed = true } return g } // open_parser creates a fresh parser bound to the grammar's language. The CALLER // owns the returned parser and must free it with parser_delete. This is the seam // for parser lifetime: today callers open one and reuse it across a batch of // parses on a single thread (render opens an html+css pair per render_site; // assets opens one per copy_assets_dir), which is safe because nothing is shared. // A future pooling or thread-local strategy can be dropped in here without // touching any call site. Returns nil on failure. open_parser :: proc(g: ^Grammar) -> Parser { if g == nil { return nil } parser := parser_new() if parser == nil { log.errorf("treesitter: cannot create parser") return nil } if !parser_set_language(parser, g.language) { log.errorf("treesitter: ABI mismatch setting parser language") parser_delete(parser) return nil } return parser } when SPALL { _thread_init: proc() = nil _thread_cleanup: proc() = nil set_thread_callbacks :: proc(init: proc() = nil, cleanup: proc() = nil) { _thread_init = init _thread_cleanup = cleanup } } builtin_language :: proc(lang: string) -> (language: Language, ok: bool) { switch lang { case "html": language = tree_sitter_html() ok = true case "css": language = tree_sitter_css() ok = true } return } // load_query returns the highlight query source for a language. Builtin // languages (html/css) are baked into the binary via `#load`; all others are // read from the runtime `query_dir`. `path` is the on-disk location for // diagnostics ("(builtin)" for embedded queries). Mirrors `load_language`. load_query :: proc(lang: string) -> (src: string, path: string, ok: bool) { switch lang { case "html": return HTML_HIGHLIGHTS, "(builtin)", true case "css": return CSS_HIGHLIGHTS, "(builtin)", true } if query_dir == "" { log.warnf("treesitter: no query path set, skipping %s", lang) return "", "", false } path = fmt.tprintf("%s/%s/highlights.scm", query_dir, lang) raw, err := os.read_entire_file_from_path(path, context.allocator) if err != nil { log.warnf("treesitter: cannot load query %s", path) return "", "", false } return string(raw), path, true } load_language :: proc(lang: string) -> (language: Language, ok: bool) { if builtin, bok := builtin_language(lang); bok { language = builtin ok = true return } if grammar_dir == "" { log.warnf("treesitter: no grammar path set, skipping %s", lang) return } so_path := fmt.caprintf("%s/%s.so", grammar_dir, lang, allocator = context.temp_allocator) handle := dlopen(so_path, RTLD_LAZY) if handle == nil { log.warnf("treesitter: cannot load grammar %s (%s)", lang, so_path) return } sym_name := fmt.caprintf("tree_sitter_%s", lang, allocator = context.temp_allocator) sym := dlsym(handle, sym_name) if sym == nil { log.errorf("treesitter: cannot find symbol %s in %s", sym_name, so_path) return } get_language := transmute(Get_Language_Proc)(sym) language = get_language() ok = true return } compile_query :: proc(lang: string, language: Language) -> (query: Query, ok: bool) { query_src, query_path, qok := load_query(lang) if !qok { return } query_c := strings.clone_to_cstring(query_src, context.temp_allocator) err_offset: u32 err_type: Query_Error query = query_new(language, query_c, u32(len(query_src)), &err_offset, &err_type) if query == nil { tok := extract_query_token(transmute([]byte)query_src, err_offset) cause := fmt.tprintf("query error at byte %d (type %v)", err_offset, err_type) switch err_type { case .NodeType: if tok != "" { cause = fmt.tprintf( "query references unknown node type '%s' (byte %d); the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions", tok, err_offset, lang, ) } else { cause = fmt.tprintf( "query references an unknown node type at byte %d; the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions", err_offset, lang, ) } case .Field: cause = fmt.tprintf("query references unknown field '%s' at byte %d", tok, err_offset) case .Capture: cause = fmt.tprintf("query uses an invalid capture '%s' at byte %d", tok, err_offset) case .Syntax: cause = fmt.tprintf("query has a syntax error at byte %d", err_offset) case .Structure: cause = fmt.tprintf("query has an illegal pattern structure at byte %d", err_offset) case .Language: cause = "grammar language is null (broken grammar .so)" case .None: } log.errorf("treesitter: %s query failed: %s", lang, cause) _, is_builtin := builtin_language(lang) if !is_builtin { so_path := fmt.tprintf("%s/%s.so", grammar_dir, lang) gram_v := helix_version_from_path(so_path) query_v := helix_version_from_path(query_path) gram_note := "(version unknown)" if gram_v != "" do gram_note = fmt.tprintf("helix %s", gram_v) query_note := "(version unknown)" if query_v != "" do query_note = fmt.tprintf("helix %s", query_v) log.errorf(" grammar: %s [%s]", so_path, gram_note) log.errorf(" query: %s [%s]", query_path, query_note) if gram_v != "" && query_v != "" && gram_v != query_v { log.errorf(" >> helix VERSION MISMATCH: grammar %s vs query %s", gram_v, query_v) } } return } ok = true return } // preload_grammars warms the registry for the given languages ahead of the render // loop, so the first page that needs one doesn't pay the load cost inline. Builtins // are always available lazily and are skipped. Runs on the calling thread; grammar() // is itself thread-safe if a caller ever wants to fan this out. preload_grammars :: proc(languages: []string) { for lang in languages { if _, bok := builtin_language(lang); bok { continue } grammar(lang) } } extract_query_token :: proc(src: []byte, offset: u32) -> string { end := offset for int(end) < len(src) { c := src[end] is_ident := (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '_' || c == '-' || c == '.' if !is_ident do break end += 1 } if end <= offset do return "" return string(src[offset:end]) } helix_version_from_path :: proc(path: string) -> string { tag := "-helix-" idx := strings.index(path, tag) if idx < 0 do return "" start := idx + len(tag) end := start for end < len(path) { c := path[end] if !((c >= '0' && c <= '9') || c == '.') do break end += 1 } if end <= start do return "" return path[start:end] }