From d23eec448b3ce3b5451fb119773ca0def32c777c Mon Sep 17 00:00:00 2001 From: Spencer Brower <6729162+sbrow@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:51:54 -0400 Subject: [PATCH] feat: Added server-side syntax-highlighting (via tree-sitter). --- TODOS.md | 17 +- content.odin | 2 +- flake.nix | 2 + highlight.odin | 426 +++++++++++++++++++++++++++++++++++++++++++++++ main.odin | 1 + tree_sitter.odin | 115 +++++++++++++ 6 files changed, 555 insertions(+), 8 deletions(-) create mode 100644 highlight.odin create mode 100644 tree_sitter.odin diff --git a/TODOS.md b/TODOS.md index 13f1a96..7f7d6c7 100644 --- a/TODOS.md +++ b/TODOS.md @@ -1,19 +1,22 @@ -- [ ] add content-hash fingerprinting for tailwind cache busting. -- [ ] Evaluate Tufte CSS — borrow sidenote CSS or replace TailwindCSS entirely - - Option B: Steal Tufte's sidenote/margin-note CSS (adapt for dark theme), keep TailwindCSS - - Option C: Full Tufte CSS — drop TailwindCSS, no build step, semantic HTML, customize for dark theme + Roboto - - Our sidenote HTML pattern already matches Tufte's exactly +- README.md + - [ ] "Zero is beautiful" +- [ ] Content-hash fingerprinting for CSS and JS cache busting +- [ ] proper date/time/now object. - [ ] Block attributes on code fences (`{ #ex-1 }`) — hello-world.md - [x] Emoji shortcodes (`:shrug:` etc.) — 2 instances - [ ] Backslash in shrug not visible. - [ ] include-code shortcode (`{{< include-code ... >}}`) — i-ported-fd-to-odin - [x] Nix build integration — main flake runs thor + tailwindcss instead of Hugo -- [ ] Fix copy-to-clipboard button x-axis positioning inside code blocks -- [ ] Content-hash fingerprinting for CSS and JS cache busting - [x] OpenGraph meta tags - [ ] Search up for `thor.json` files. - [ ] OpenGraph meta tags — verify all fields match production site +- [ ] Table of contents support. - [ ] Nav items should be active when the current page is selected. +- [ ] Theme selector for syntax highlighting. + - use http://github.com/helix-editor/helix/tree/master/runtime/themes) as a + guide +- [ ] Search for grammars in multiple places +- [ ] Download missing grammars. - [ ] Review every file in thor - [ ] Review alerts.odin - [ ] Review content.odin diff --git a/content.odin b/content.odin index 1f0b802..eb3fa00 100644 --- a/content.odin +++ b/content.odin @@ -178,7 +178,7 @@ load_page :: proc( expanded := expand_emoji(body) clean_body, defs := strip_definitions(expanded) html := cm.markdown_to_html_from_string(clean_body, {.Unsafe}) - page.body_html = inject_alerts(inject_sidenotes(html, defs)) + page.body_html = highlight_code(inject_alerts(inject_sidenotes(html, defs)), file_path) } switch page_type { diff --git a/flake.nix b/flake.nix index 0ed4d42..7cb486a 100644 --- a/flake.nix +++ b/flake.nix @@ -81,6 +81,7 @@ buildInputs = [ pkgs.git pkgs.cmark + pkgs.tree-sitter ]; doCheck = true; @@ -111,6 +112,7 @@ odin ols cmark + tree-sitter # IDE unstable.helix diff --git a/highlight.odin b/highlight.odin new file mode 100644 index 0000000..e09db44 --- /dev/null +++ b/highlight.odin @@ -0,0 +1,426 @@ +package main + +import "core:fmt" +import "core:log" +import "core:os" +import "core:strings" + +Grammar_Cache :: struct { + language: TSLanguage, + parser: TSParser, + query: TSQuery, +} + +Get_Language_Proc :: #type proc() -> TSLanguage + +grammar_cache: map[string]^Grammar_Cache + +load_grammar :: proc(lang: string) -> ^Grammar_Cache { + if grammar_cache == nil { + grammar_cache = make(map[string]^Grammar_Cache) + } + if cached, ok := grammar_cache[lang]; ok { + return cached + } + + // Cache nil by default so failures aren't retried. + grammar_cache[lang] = nil + + if GRAPHS_PATH == "" { + log.warnf("highlight: no grammars path set, skipping %s", lang) + return nil + } + + so_path := fmt.tprintf("%s/%s.so", GRAPHS_PATH, lang) + so_c := strings.clone_to_cstring(so_path) + defer delete(so_c) + handle := dlopen(so_c, RTLD_LAZY) + if handle == nil { + log.warnf("highlight: cannot load grammar %s (%s)", lang, so_path) + return nil + } + + sym_name := fmt.tprintf("tree_sitter_%s", lang) + sym_c := strings.clone_to_cstring(sym_name) + defer delete(sym_c) + sym := dlsym(handle, sym_c) + if sym == nil { + log.errorf("highlight: cannot find symbol %s in %s", sym_name, so_path) + return nil + } + get_language := transmute(Get_Language_Proc)(sym) + language := get_language() + + parser := ts_parser_new() + if parser == nil { + log.errorf("highlight: cannot create parser for %s", lang) + return nil + } + if !ts_parser_set_language(parser, language) { + log.errorf("highlight: ABI mismatch for %s grammar", lang) + ts_parser_delete(parser) + return nil + } + + if QUERIES_PATH == "" { + log.warnf("highlight: no queries path set, skipping %s", lang) + ts_parser_delete(parser) + return nil + } + + query_path := fmt.tprintf("%s/%s/highlights.scm", QUERIES_PATH, lang) + query_src, err := os.read_entire_file_from_path(query_path, context.allocator) + if err != nil { + log.warnf("highlight: cannot load query %s", query_path) + ts_parser_delete(parser) + return nil + } + query_str := string(query_src) + query_c := strings.clone_to_cstring(query_str) + defer delete(query_c) + + err_offset: u32 + err_type: TSQueryError + query := ts_query_new( + language, + query_c, + u32(len(query_src)), + &err_offset, + &err_type, + ) + if query == nil { + tok := extract_query_token(query_src, err_offset) + cause := fmt.tprintf("query error at byte %d (type %v)", err_offset, err_type) + #partial switch err_type { + case .NodeType: + if tok != "" { + cause = fmt.tprintf("query references unknown node type '%s' (byte %d); the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions", tok, err_offset, lang) + } else { + cause = fmt.tprintf("query references an unknown node type at byte %d; the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions", err_offset, lang) + } + case .Field: + cause = fmt.tprintf("query references unknown field '%s' at byte %d", tok, err_offset) + case .Capture: + cause = fmt.tprintf("query uses an invalid capture '%s' at byte %d", tok, err_offset) + case .Syntax: + cause = fmt.tprintf("query has a syntax error at byte %d", err_offset) + case .Structure: + cause = fmt.tprintf("query has an illegal pattern structure at byte %d", err_offset) + case .Language: + cause = "grammar language is null (broken grammar .so)" + } + log.errorf("highlight: %s query failed: %s", lang, cause) + + gram_v := helix_version_from_path(so_path) + query_v := helix_version_from_path(query_path) + gram_note := "(version unknown)" + if gram_v != "" do gram_note = fmt.tprintf("helix %s", gram_v) + query_note := "(version unknown)" + if query_v != "" do query_note = fmt.tprintf("helix %s", query_v) + log.errorf(" grammar: %s [%s]", so_path, gram_note) + log.errorf(" query: %s [%s]", query_path, query_note) + if gram_v != "" && query_v != "" && gram_v != query_v { + log.errorf(" >> helix VERSION MISMATCH: grammar %s vs query %s", gram_v, query_v) + } + + ts_parser_delete(parser) + return nil + } + + gc := new(Grammar_Cache) + gc.language = language + gc.parser = parser + gc.query = query + grammar_cache[lang] = gc + return gc +} + +extract_query_token :: proc(src: []byte, offset: u32) -> string { + end := offset + for int(end) < len(src) { + c := src[end] + is_ident := (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || c == '_' || c == '-' || c == '.' + if !is_ident do break + end += 1 + } + if end <= offset do return "" + return string(src[offset:end]) +} + +helix_version_from_path :: proc(path: string) -> string { + tag := "-helix-" + idx := strings.index(path, tag) + if idx < 0 do return "" + start := idx + len(tag) + end := start + for end < len(path) { + c := path[end] + if !((c >= '0' && c <= '9') || c == '.') do break + end += 1 + } + if end <= start do return "" + return path[start:end] +} + +Capture :: struct { + start: u32, + end: u32, + name: string, +} + +find_first_error_line :: proc(root: TSNode) -> int { + if ts_node_is_error(root) { + return int(ts_node_start_point(root).row) + 1 + } + for i in 0.. 0 { + return line + } + } + } + return 0 +} + +capture_name_to_css :: proc(name: string) -> string { + sb := strings.builder_make() + strings.write_string(&sb, "hl-") + for i in 0.. string { + parts: [dynamic]string + defer delete(parts) + start := 0 + for i in 0.. start do append(&parts, s[start:i]) + append(&parts, "&") + start = i + 1 + case '<': + if i > start do append(&parts, s[start:i]) + append(&parts, "<") + start = i + 1 + case '>': + if i > start do append(&parts, s[start:i]) + append(&parts, ">") + start = i + 1 + case '"': + if i > start do append(&parts, s[start:i]) + append(&parts, """) + start = i + 1 + } + } + if start < len(s) do append(&parts, s[start:]) + if len(parts) == 0 do return s + return strings.join(parts[:], "") +} + +unescape_html :: proc(s: string) -> string { + parts: [dynamic]string + defer delete(parts) + start := 0 + for i in 0.. start do append(&parts, s[start:i]) + append(&parts, replacement) + start = i + semi + 1 + } + if start < len(s) do append(&parts, s[start:]) + if len(parts) == 0 do return s + return strings.join(parts[:], "") +} + +highlight_block :: proc(code: string, lang: string, file_path: string) -> string { + gc := load_grammar(lang) + if gc == nil { + return code + } + + raw_code := unescape_html(code) + raw_c := strings.clone_to_cstring(raw_code) + defer delete(raw_c) + + tree := ts_parser_parse_string(gc.parser, nil, raw_c, u32(len(raw_code))) + if tree == nil { + return code + } + defer ts_tree_delete(tree) + + root := ts_tree_root_node(tree) + + if ts_node_has_error(root) { + line := find_first_error_line(root) + if line > 0 { + log.warnf("highlight: syntax errors in %s code block at line %d (%s)", lang, line, file_path) + } else { + log.warnf("highlight: syntax errors in %s code block (%s)", lang, file_path) + } + } + + cursor := ts_query_cursor_new() + if cursor == nil { + return code + } + defer ts_query_cursor_delete(cursor) + + ts_query_cursor_exec(cursor, gc.query, root) + + captures: [dynamic]Capture + defer delete(captures) + + match: TSQueryMatch + capture_idx: u32 + for ts_query_cursor_next_capture(cursor, &match, &capture_idx) { + if capture_idx >= u32(match.capture_count) { + continue + } + cap := match.captures[capture_idx] + name_len: u32 + name_c := ts_query_capture_name_for_id(gc.query, cap.index, &name_len) + if name_c == nil { + continue + } + name_full := string(name_c) + name := name_full + if len(name_full) > int(name_len) { + name = name_full[:int(name_len)] + } + append(&captures, Capture{ + start = ts_node_start_byte(cap.node), + end = ts_node_end_byte(cap.node), + name = name, + }) + } + + if len(captures) == 0 { + return code + } + + sb := strings.builder_make() + + last_pos: u32 = 0 + stack: [dynamic]Capture + defer delete(stack) + + for cap in captures { + for len(stack) > 0 { + top := stack[len(stack) - 1] + if top.end <= cap.start { + if top.end > last_pos { + strings.write_string(&sb, escape_html(raw_code[last_pos:top.end])) + } + strings.write_string(&sb, "") + last_pos = top.end + pop(&stack) + } else { + break + } + } + + if cap.start > last_pos { + strings.write_string(&sb, escape_html(raw_code[last_pos:cap.start])) + last_pos = cap.start + } + + css_class := capture_name_to_css(cap.name) + strings.write_string(&sb, fmt.tprintf("", css_class)) + append(&stack, cap) + } + + for len(stack) > 0 { + top := pop(&stack) + if top.end > last_pos { + strings.write_string(&sb, escape_html(raw_code[last_pos:top.end])) + } + strings.write_string(&sb, "") + last_pos = top.end + } + + if int(last_pos) < len(raw_code) { + strings.write_string(&sb, escape_html(raw_code[last_pos:])) + } + + return strings.to_string(sb) +} + +highlight_code :: proc(html: string, file_path: string) -> string { + PREFIX :: `
' {
+			code_start += 1
+		} else {
+			pos = lang_end
+			continue
+		}
+
+		end_rel := strings.index(html[code_start:], CODE_END)
+		if end_rel < 0 {
+			break
+		}
+		end_idx := code_start + end_rel
+
+		code := html[code_start:end_idx]
+		highlighted := highlight_block(code, lang, file_path)
+		append(&parts, fmt.tprintf(`
%s
`, lang, highlighted))
+
+		pos = end_idx + len(CODE_END)
+	}
+
+	if pos < len(html) {
+		append(&parts, html[pos:])
+	}
+
+	if len(parts) == 0 {
+		return html
+	}
+	return strings.join(parts[:], "")
+}
diff --git a/main.odin b/main.odin
index 1e4beba..310be46 100644
--- a/main.odin
+++ b/main.odin
@@ -5,6 +5,7 @@ import "core:os"
 
 main :: proc() {
 	console_logger := log.create_console_logger()
+	context.logger = console_logger
 	defer log.destroy_console_logger(console_logger)
 
 	site: Site
diff --git a/tree_sitter.odin b/tree_sitter.odin
new file mode 100644
index 0000000..33ea476
--- /dev/null
+++ b/tree_sitter.odin
@@ -0,0 +1,115 @@
+package main
+
+import "core:c"
+
+GRAPHS_PATH: string = "/home/spencer/.config/helix/runtime/grammars"
+QUERIES_PATH: string = "/nix/store/n9da8d007ygbgsx983jr3ar3wb1fsh6q-helix-25.07.1/lib/runtime/queries"
+
+TSLanguage :: distinct rawptr
+TSParser :: distinct rawptr
+TSTree :: distinct rawptr
+TSQuery :: distinct rawptr
+TSQueryCursor :: distinct rawptr
+
+TSPoint :: struct {
+	row:    u32,
+	column: u32,
+}
+
+TSNode :: struct {
+	ctx:   [4]u32,
+	id:    rawptr,
+	tree:  rawptr,
+}
+
+TSQueryCapture :: struct {
+	node:  TSNode,
+	index: u32,
+	_:     u32,
+}
+
+TSQueryMatch :: struct {
+	id:            u32,
+	pattern_index: u16,
+	capture_count: u16,
+	captures:      [^]TSQueryCapture,
+}
+
+TSQueryError :: enum c.int {
+	None     = 0,
+	Syntax,
+	NodeType,
+	Field,
+	Capture,
+	Structure,
+	Language,
+}
+
+RTLD_LAZY :: c.int(1)
+
+foreign import lib "system:tree-sitter"
+foreign import libdl "system:dl"
+
+foreign lib {
+	ts_parser_new :: proc() -> TSParser ---
+	ts_parser_delete :: proc(self: TSParser) ---
+	ts_parser_set_language :: proc(self: TSParser, language: TSLanguage) -> bool ---
+	ts_parser_parse_string :: proc(
+		self: TSParser,
+		old_tree: TSTree,
+		string: cstring,
+		length: u32,
+	) -> TSTree ---
+}
+
+foreign lib {
+	ts_tree_root_node :: proc(self: TSTree) -> TSNode ---
+	ts_tree_delete :: proc(self: TSTree) ---
+}
+
+foreign lib {
+	ts_node_start_byte :: proc(self: TSNode) -> u32 ---
+	ts_node_end_byte :: proc(self: TSNode) -> u32 ---
+	ts_node_has_error :: proc(self: TSNode) -> bool ---
+	ts_node_is_error :: proc(self: TSNode) -> bool ---
+	ts_node_child_count :: proc(self: TSNode) -> u32 ---
+	ts_node_child :: proc(self: TSNode, child_index: u32) -> TSNode ---
+	ts_node_start_point :: proc(self: TSNode) -> TSPoint ---
+}
+
+foreign lib {
+	ts_query_new :: proc(
+		language: TSLanguage,
+		source: cstring,
+		source_len: u32,
+		error_offset: ^u32,
+		error_type: ^TSQueryError,
+	) -> TSQuery ---
+	ts_query_delete :: proc(self: TSQuery) ---
+	ts_query_capture_name_for_id :: proc(
+		self: TSQuery,
+		index: u32,
+		length: ^u32,
+	) -> cstring ---
+}
+
+foreign lib {
+	ts_query_cursor_new :: proc() -> TSQueryCursor ---
+	ts_query_cursor_delete :: proc(self: TSQueryCursor) ---
+	ts_query_cursor_exec :: proc(
+		self: TSQueryCursor,
+		query: TSQuery,
+		node: TSNode,
+	) ---
+	ts_query_cursor_next_capture :: proc(
+		self: TSQueryCursor,
+		match: ^TSQueryMatch,
+		capture_index: ^u32,
+	) -> bool ---
+}
+
+foreign libdl {
+	dlopen :: proc(filename: cstring, flags: c.int) -> rawptr ---
+	dlsym :: proc(handle: rawptr, symbol: cstring) -> rawptr ---
+	dlclose :: proc(handle: rawptr) -> c.int ---
+}