diff --git a/highlight.odin b/highlight.odin index d60c513..4d3ffef 100644 --- a/highlight.odin +++ b/highlight.odin @@ -1,218 +1,24 @@ package main +import ts "treesitter" + import "core:fmt" import "core:log" -import "core:os" import "core:strings" -Grammar_Cache :: struct { - language: TSLanguage, - parser: TSParser, - query: TSQuery, - query_failed: bool, -} - -Get_Language_Proc :: #type proc() -> TSLanguage - -grammar_cache: map[string]^Grammar_Cache - -builtin_language :: proc(lang: string) -> (language: TSLanguage, ok: bool) { - switch lang { - case "html": - language = tree_sitter_html() - ok = true - case "css": - language = tree_sitter_css() - ok = true - } - return -} - -ensure_parser :: proc(lang: string) -> ^Grammar_Cache { - if grammar_cache == nil { - grammar_cache = make(map[string]^Grammar_Cache) - } - if cached, ok := grammar_cache[lang]; ok { - return cached - } - - grammar_cache[lang] = nil - - language: TSLanguage - - if builtin, ok := builtin_language(lang); ok { - language = builtin - } else { - if GRAPHS_PATH == "" { - log.warnf("highlight: no grammars path set, skipping %s", lang) - return nil - } - - so_path := fmt.tprintf("%s/%s.so", GRAPHS_PATH, lang) - so_c := strings.clone_to_cstring(so_path) - defer delete(so_c) - handle := dlopen(so_c, RTLD_LAZY) - if handle == nil { - log.warnf("highlight: cannot load grammar %s (%s)", lang, so_path) - return nil - } - - sym_name := fmt.tprintf("tree_sitter_%s", lang) - sym_c := strings.clone_to_cstring(sym_name) - defer delete(sym_c) - sym := dlsym(handle, sym_c) - if sym == nil { - log.errorf("highlight: cannot find symbol %s in %s", sym_name, so_path) - return nil - } - get_language := transmute(Get_Language_Proc)(sym) - language = get_language() - } - - parser := ts_parser_new() - if parser == nil { - log.errorf("highlight: cannot create parser for %s", lang) - return nil - } - if !ts_parser_set_language(parser, language) { - log.errorf("highlight: ABI mismatch for %s grammar", lang) - ts_parser_delete(parser) - return nil - } - - gc := new(Grammar_Cache) - gc.language = language - gc.parser = parser - grammar_cache[lang] = gc - return gc -} - -load_grammar :: proc(lang: string) -> ^Grammar_Cache { - gc := ensure_parser(lang) - if gc == nil { - return nil - } - if gc.query != nil { - return gc - } - if gc.query_failed { - return nil - } - - if QUERIES_PATH == "" { - log.warnf("highlight: no queries path set, skipping %s", lang) - gc.query_failed = true - return nil - } - - query_path := fmt.tprintf("%s/%s/highlights.scm", QUERIES_PATH, lang) - query_src, err := os.read_entire_file_from_path(query_path, context.allocator) - if err != nil { - log.warnf("highlight: cannot load query %s", query_path) - gc.query_failed = true - return nil - } - query_str := string(query_src) - query_c := strings.clone_to_cstring(query_str) - defer delete(query_c) - - err_offset: u32 - err_type: TSQueryError - query := ts_query_new( - gc.language, - query_c, - u32(len(query_src)), - &err_offset, - &err_type, - ) - if query == nil { - tok := extract_query_token(query_src, err_offset) - cause := fmt.tprintf("query error at byte %d (type %v)", err_offset, err_type) - #partial switch err_type { - case .NodeType: - if tok != "" { - cause = fmt.tprintf("query references unknown node type '%s' (byte %d); the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions", tok, err_offset, lang) - } else { - cause = fmt.tprintf("query references an unknown node type at byte %d; the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions", err_offset, lang) - } - case .Field: - cause = fmt.tprintf("query references unknown field '%s' at byte %d", tok, err_offset) - case .Capture: - cause = fmt.tprintf("query uses an invalid capture '%s' at byte %d", tok, err_offset) - case .Syntax: - cause = fmt.tprintf("query has a syntax error at byte %d", err_offset) - case .Structure: - cause = fmt.tprintf("query has an illegal pattern structure at byte %d", err_offset) - case .Language: - cause = "grammar language is null (broken grammar .so)" - } - log.errorf("highlight: %s query failed: %s", lang, cause) - - _, is_builtin := builtin_language(lang) - if !is_builtin { - so_path := fmt.tprintf("%s/%s.so", GRAPHS_PATH, lang) - gram_v := helix_version_from_path(so_path) - query_v := helix_version_from_path(query_path) - gram_note := "(version unknown)" - if gram_v != "" do gram_note = fmt.tprintf("helix %s", gram_v) - query_note := "(version unknown)" - if query_v != "" do query_note = fmt.tprintf("helix %s", query_v) - log.errorf(" grammar: %s [%s]", so_path, gram_note) - log.errorf(" query: %s [%s]", query_path, query_note) - if gram_v != "" && query_v != "" && gram_v != query_v { - log.errorf(" >> helix VERSION MISMATCH: grammar %s vs query %s", gram_v, query_v) - } - } - - gc.query_failed = true - return nil - } - - gc.query = query - return gc -} - -extract_query_token :: proc(src: []byte, offset: u32) -> string { - end := offset - for int(end) < len(src) { - c := src[end] - is_ident := (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || - (c >= '0' && c <= '9') || c == '_' || c == '-' || c == '.' - if !is_ident do break - end += 1 - } - if end <= offset do return "" - return string(src[offset:end]) -} - -helix_version_from_path :: proc(path: string) -> string { - tag := "-helix-" - idx := strings.index(path, tag) - if idx < 0 do return "" - start := idx + len(tag) - end := start - for end < len(path) { - c := path[end] - if !((c >= '0' && c <= '9') || c == '.') do break - end += 1 - } - if end <= start do return "" - return path[start:end] -} - Capture :: struct { start: u32, end: u32, name: string, } -find_first_error_line :: proc(root: TSNode) -> int { - if ts_node_is_error(root) { - return int(ts_node_start_point(root).row) + 1 +find_first_error_line :: proc(root: ts.Node) -> int { + if ts.node_is_error(root) { + return int(ts.node_start_point(root).row) + 1 } - for i in 0.. 0 { return line @@ -302,7 +108,7 @@ unescape_html :: proc(s: string) -> string { } highlight_block :: proc(code: string, lang: string, file_path: string) -> string { - gc := load_grammar(lang) + gc := ts.load_grammar(lang) if gc == nil { return code } @@ -311,15 +117,15 @@ highlight_block :: proc(code: string, lang: string, file_path: string) -> string raw_c := strings.clone_to_cstring(raw_code) defer delete(raw_c) - tree := ts_parser_parse_string(gc.parser, nil, raw_c, u32(len(raw_code))) + tree := ts.parser_parse_string(gc.parser, nil, raw_c, u32(len(raw_code))) if tree == nil { return code } - defer ts_tree_delete(tree) + defer ts.tree_delete(tree) - root := ts_tree_root_node(tree) + root := ts.tree_root_node(tree) - if ts_node_has_error(root) { + if ts.node_has_error(root) { line := find_first_error_line(root) if line > 0 { log.warnf("highlight: syntax errors in %s code block at line %d (%s)", lang, line, file_path) @@ -328,26 +134,26 @@ highlight_block :: proc(code: string, lang: string, file_path: string) -> string } } - cursor := ts_query_cursor_new() + cursor := ts.query_cursor_new() if cursor == nil { return code } - defer ts_query_cursor_delete(cursor) + defer ts.query_cursor_delete(cursor) - ts_query_cursor_exec(cursor, gc.query, root) + ts.query_cursor_exec(cursor, gc.query, root) captures: [dynamic]Capture defer delete(captures) - match: TSQueryMatch + match: ts.Query_Match capture_idx: u32 - for ts_query_cursor_next_capture(cursor, &match, &capture_idx) { + for ts.query_cursor_next_capture(cursor, &match, &capture_idx) { if capture_idx >= u32(match.capture_count) { continue } cap := match.captures[capture_idx] name_len: u32 - name_c := ts_query_capture_name_for_id(gc.query, cap.index, &name_len) + name_c := ts.query_capture_name_for_id(gc.query, cap.index, &name_len) if name_c == nil { continue } @@ -357,8 +163,8 @@ highlight_block :: proc(code: string, lang: string, file_path: string) -> string name = name_full[:int(name_len)] } append(&captures, Capture{ - start = ts_node_start_byte(cap.node), - end = ts_node_end_byte(cap.node), + start = ts.node_start_byte(cap.node), + end = ts.node_end_byte(cap.node), name = name, }) } diff --git a/minify.odin b/minify.odin index a68fd9f..33e7a2f 100644 --- a/minify.odin +++ b/minify.odin @@ -1,5 +1,7 @@ package main +import ts "treesitter" + import "core:log" import "core:strings" @@ -11,7 +13,7 @@ Range :: struct { } minify_html :: proc(source: string) -> string { - gc := ensure_parser("html") + gc := ts.ensure_parser("html") if gc == nil { return source } @@ -19,15 +21,15 @@ minify_html :: proc(source: string) -> string { source_c := strings.clone_to_cstring(source) defer delete(source_c) - tree := ts_parser_parse_string(gc.parser, nil, source_c, u32(len(source))) + tree := ts.parser_parse_string(gc.parser, nil, source_c, u32(len(source))) if tree == nil { return source } - defer ts_tree_delete(tree) + defer ts.tree_delete(tree) - root := ts_tree_root_node(tree) + root := ts.tree_root_node(tree) - if ts_node_has_error(root) { + if ts.node_has_error(root) { log.warnf("minify: HTML parse errors, skipping minification") return source } @@ -95,32 +97,32 @@ minify_html :: proc(source: string) -> string { } collect_html_ranges :: proc( - node: TSNode, + node: ts.Node, source: string, comments: ^[dynamic]Range, preserves: ^[dynamic]Range, ) { - child_count := ts_node_named_child_count(node) + child_count := ts.node_named_child_count(node) for i in 0.. string { - child_count := ts_node_named_child_count(element) +html_tag_name :: proc(element: ts.Node, source: string) -> string { + child_count := ts.node_named_child_count(element) for i in 0.. bool { } minify_css :: proc(source: string) -> string { - gc := ensure_parser("css") + gc := ts.ensure_parser("css") if gc == nil { return source } @@ -175,15 +177,15 @@ minify_css :: proc(source: string) -> string { source_c := strings.clone_to_cstring(source) defer delete(source_c) - tree := ts_parser_parse_string(gc.parser, nil, source_c, u32(len(source))) + tree := ts.parser_parse_string(gc.parser, nil, source_c, u32(len(source))) if tree == nil { return source } - defer ts_tree_delete(tree) + defer ts.tree_delete(tree) - root := ts_tree_root_node(tree) + root := ts.tree_root_node(tree) - if ts_node_has_error(root) { + if ts.node_has_error(root) { log.warnf("minify: CSS parse errors, skipping minification") return source } @@ -237,14 +239,14 @@ minify_css :: proc(source: string) -> string { return strings.to_string(sb) } -collect_css_comments :: proc(node: TSNode, comments: ^[dynamic]Range) { - child_count := ts_node_named_child_count(node) +collect_css_comments :: proc(node: ts.Node, comments: ^[dynamic]Range) { + child_count := ts.node_named_child_count(node) for i in 0.. TSParser --- - ts_parser_delete :: proc(self: TSParser) --- - ts_parser_set_language :: proc(self: TSParser, language: TSLanguage) -> bool --- - ts_parser_parse_string :: proc( - self: TSParser, - old_tree: TSTree, - string: cstring, - length: u32, - ) -> TSTree --- -} - -foreign lib { - ts_tree_root_node :: proc(self: TSTree) -> TSNode --- - ts_tree_delete :: proc(self: TSTree) --- -} - -foreign lib { - ts_node_start_byte :: proc(self: TSNode) -> u32 --- - ts_node_end_byte :: proc(self: TSNode) -> u32 --- - ts_node_has_error :: proc(self: TSNode) -> bool --- - ts_node_is_error :: proc(self: TSNode) -> bool --- - ts_node_child_count :: proc(self: TSNode) -> u32 --- - ts_node_child :: proc(self: TSNode, child_index: u32) -> TSNode --- - ts_node_named_child_count :: proc(self: TSNode) -> u32 --- - ts_node_named_child :: proc(self: TSNode, child_index: u32) -> TSNode --- - ts_node_start_point :: proc(self: TSNode) -> TSPoint --- - ts_node_type :: proc(self: TSNode) -> cstring --- - ts_node_parent :: proc(self: TSNode) -> TSNode --- -} - -foreign lib { - ts_query_new :: proc( - language: TSLanguage, - source: cstring, - source_len: u32, - error_offset: ^u32, - error_type: ^TSQueryError, - ) -> TSQuery --- - ts_query_delete :: proc(self: TSQuery) --- - ts_query_capture_name_for_id :: proc( - self: TSQuery, - index: u32, - length: ^u32, - ) -> cstring --- -} - -foreign lib { - ts_query_cursor_new :: proc() -> TSQueryCursor --- - ts_query_cursor_delete :: proc(self: TSQueryCursor) --- - ts_query_cursor_exec :: proc( - self: TSQueryCursor, - query: TSQuery, - node: TSNode, - ) --- - ts_query_cursor_next_capture :: proc( - self: TSQueryCursor, - match: ^TSQueryMatch, - capture_index: ^u32, - ) -> bool --- -} - -foreign libdl { - dlopen :: proc(filename: cstring, flags: c.int) -> rawptr --- - dlsym :: proc(handle: rawptr, symbol: cstring) -> rawptr --- - dlclose :: proc(handle: rawptr) -> c.int --- -} - -foreign html_grammar { - tree_sitter_html :: proc() -> TSLanguage --- -} - -foreign css_grammar { - tree_sitter_css :: proc() -> TSLanguage --- -} diff --git a/treesitter/treesitter.odin b/treesitter/treesitter.odin new file mode 100644 index 0000000..8059e52 --- /dev/null +++ b/treesitter/treesitter.odin @@ -0,0 +1,333 @@ +package treesitter + +import "core:c" +import "core:fmt" +import "core:log" +import "core:os" +import "core:strings" + +GRAPHS_PATH: string = "/home/spencer/.config/helix/runtime/grammars" +QUERIES_PATH: string = "/nix/store/n9da8d007ygbgsx983jr3ar3wb1fsh6q-helix-25.07.1/lib/runtime/queries" + +Language :: distinct rawptr +Parser :: distinct rawptr +Tree :: distinct rawptr +Query :: distinct rawptr +Query_Cursor :: distinct rawptr + +Point :: struct { + row: u32, + column: u32, +} + +Node :: struct { + ctx: [4]u32, + id: rawptr, + tree: rawptr, +} + +Query_Capture :: struct { + node: Node, + index: u32, + _: u32, +} + +Query_Match :: struct { + id: u32, + pattern_index: u16, + capture_count: u16, + captures: [^]Query_Capture, +} + +Query_Error :: enum c.int { + None = 0, + Syntax, + NodeType, + Field, + Capture, + Structure, + Language, +} + +RTLD_LAZY :: c.int(1) + +foreign import lib "system:tree-sitter" +foreign import libdl "system:dl" +foreign import html_grammar "system:tree-sitter-html" +foreign import css_grammar "system:tree-sitter-css" + +@(link_prefix="ts_") +foreign lib { + parser_new :: proc() -> Parser --- + parser_delete :: proc(self: Parser) --- + parser_set_language :: proc(self: Parser, language: Language) -> bool --- + parser_parse_string :: proc( + self: Parser, + old_tree: Tree, + string: cstring, + length: u32, + ) -> Tree --- +} + +@(link_prefix="ts_") +foreign lib { + tree_root_node :: proc(self: Tree) -> Node --- + tree_delete :: proc(self: Tree) --- +} + +@(link_prefix="ts_") +foreign lib { + node_start_byte :: proc(self: Node) -> u32 --- + node_end_byte :: proc(self: Node) -> u32 --- + node_has_error :: proc(self: Node) -> bool --- + node_is_error :: proc(self: Node) -> bool --- + node_child_count :: proc(self: Node) -> u32 --- + node_child :: proc(self: Node, child_index: u32) -> Node --- + node_named_child_count :: proc(self: Node) -> u32 --- + node_named_child :: proc(self: Node, child_index: u32) -> Node --- + node_start_point :: proc(self: Node) -> Point --- + node_type :: proc(self: Node) -> cstring --- + node_parent :: proc(self: Node) -> Node --- +} + +@(link_prefix="ts_") +foreign lib { + query_new :: proc( + language: Language, + source: cstring, + source_len: u32, + error_offset: ^u32, + error_type: ^Query_Error, + ) -> Query --- + query_delete :: proc(self: Query) --- + query_capture_name_for_id :: proc( + self: Query, + index: u32, + length: ^u32, + ) -> cstring --- +} + +@(link_prefix="ts_") +foreign lib { + query_cursor_new :: proc() -> Query_Cursor --- + query_cursor_delete :: proc(self: Query_Cursor) --- + query_cursor_exec :: proc( + self: Query_Cursor, + query: Query, + node: Node, + ) --- + query_cursor_next_capture :: proc( + self: Query_Cursor, + match: ^Query_Match, + capture_index: ^u32, + ) -> bool --- +} + +foreign libdl { + dlopen :: proc(filename: cstring, flags: c.int) -> rawptr --- + dlsym :: proc(handle: rawptr, symbol: cstring) -> rawptr --- + dlclose :: proc(handle: rawptr) -> c.int --- +} + +foreign html_grammar { + tree_sitter_html :: proc() -> Language --- +} + +foreign css_grammar { + tree_sitter_css :: proc() -> Language --- +} + +Grammar_Cache :: struct { + language: Language, + parser: Parser, + query: Query, + query_failed: bool, +} + +Get_Language_Proc :: #type proc() -> Language + +grammar_cache: map[string]^Grammar_Cache + +builtin_language :: proc(lang: string) -> (language: Language, ok: bool) { + switch lang { + case "html": + language = tree_sitter_html() + ok = true + case "css": + language = tree_sitter_css() + ok = true + } + return +} + +ensure_parser :: proc(lang: string) -> ^Grammar_Cache { + if grammar_cache == nil { + grammar_cache = make(map[string]^Grammar_Cache) + } + if cached, ok := grammar_cache[lang]; ok { + return cached + } + + grammar_cache[lang] = nil + + language: Language + + if builtin, ok := builtin_language(lang); ok { + language = builtin + } else { + if GRAPHS_PATH == "" { + log.warnf("treesitter: no grammars path set, skipping %s", lang) + return nil + } + + so_path := fmt.tprintf("%s/%s.so", GRAPHS_PATH, lang) + so_c := strings.clone_to_cstring(so_path) + defer delete(so_c) + handle := dlopen(so_c, RTLD_LAZY) + if handle == nil { + log.warnf("treesitter: cannot load grammar %s (%s)", lang, so_path) + return nil + } + + sym_name := fmt.tprintf("tree_sitter_%s", lang) + sym_c := strings.clone_to_cstring(sym_name) + defer delete(sym_c) + sym := dlsym(handle, sym_c) + if sym == nil { + log.errorf("treesitter: cannot find symbol %s in %s", sym_name, so_path) + return nil + } + get_language := transmute(Get_Language_Proc)(sym) + language = get_language() + } + + parser := parser_new() + if parser == nil { + log.errorf("treesitter: cannot create parser for %s", lang) + return nil + } + if !parser_set_language(parser, language) { + log.errorf("treesitter: ABI mismatch for %s grammar", lang) + parser_delete(parser) + return nil + } + + gc := new(Grammar_Cache) + gc.language = language + gc.parser = parser + grammar_cache[lang] = gc + return gc +} + +load_grammar :: proc(lang: string) -> ^Grammar_Cache { + gc := ensure_parser(lang) + if gc == nil { + return nil + } + if gc.query != nil { + return gc + } + if gc.query_failed { + return nil + } + + if QUERIES_PATH == "" { + log.warnf("treesitter: no queries path set, skipping %s", lang) + gc.query_failed = true + return nil + } + + query_path := fmt.tprintf("%s/%s/highlights.scm", QUERIES_PATH, lang) + query_src, err := os.read_entire_file_from_path(query_path, context.allocator) + if err != nil { + log.warnf("treesitter: cannot load query %s", query_path) + gc.query_failed = true + return nil + } + query_str := string(query_src) + query_c := strings.clone_to_cstring(query_str) + defer delete(query_c) + + err_offset: u32 + err_type: Query_Error + query := query_new( + gc.language, + query_c, + u32(len(query_src)), + &err_offset, + &err_type, + ) + if query == nil { + tok := extract_query_token(query_src, err_offset) + cause := fmt.tprintf("query error at byte %d (type %v)", err_offset, err_type) + #partial switch err_type { + case .NodeType: + if tok != "" { + cause = fmt.tprintf("query references unknown node type '%s' (byte %d); the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions", tok, err_offset, lang) + } else { + cause = fmt.tprintf("query references an unknown node type at byte %d; the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions", err_offset, lang) + } + case .Field: + cause = fmt.tprintf("query references unknown field '%s' at byte %d", tok, err_offset) + case .Capture: + cause = fmt.tprintf("query uses an invalid capture '%s' at byte %d", tok, err_offset) + case .Syntax: + cause = fmt.tprintf("query has a syntax error at byte %d", err_offset) + case .Structure: + cause = fmt.tprintf("query has an illegal pattern structure at byte %d", err_offset) + case .Language: + cause = "grammar language is null (broken grammar .so)" + } + log.errorf("treesitter: %s query failed: %s", lang, cause) + + _, is_builtin := builtin_language(lang) + if !is_builtin { + so_path := fmt.tprintf("%s/%s.so", GRAPHS_PATH, lang) + gram_v := helix_version_from_path(so_path) + query_v := helix_version_from_path(query_path) + gram_note := "(version unknown)" + if gram_v != "" do gram_note = fmt.tprintf("helix %s", gram_v) + query_note := "(version unknown)" + if query_v != "" do query_note = fmt.tprintf("helix %s", query_v) + log.errorf(" grammar: %s [%s]", so_path, gram_note) + log.errorf(" query: %s [%s]", query_path, query_note) + if gram_v != "" && query_v != "" && gram_v != query_v { + log.errorf(" >> helix VERSION MISMATCH: grammar %s vs query %s", gram_v, query_v) + } + } + + gc.query_failed = true + return nil + } + + gc.query = query + return gc +} + +extract_query_token :: proc(src: []byte, offset: u32) -> string { + end := offset + for int(end) < len(src) { + c := src[end] + is_ident := (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || c == '_' || c == '-' || c == '.' + if !is_ident do break + end += 1 + } + if end <= offset do return "" + return string(src[offset:end]) +} + +helix_version_from_path :: proc(path: string) -> string { + tag := "-helix-" + idx := strings.index(path, tag) + if idx < 0 do return "" + start := idx + len(tag) + end := start + for end < len(path) { + c := path[end] + if !((c >= '0' && c <= '9') || c == '.') do break + end += 1 + } + if end <= start do return "" + return path[start:end] +}