feat: Added server-side syntax-highlighting (via tree-sitter).

TODOS.md

@@ -1,19 +1,22 @@
-- [ ] add content-hash fingerprinting for tailwind cache busting.
+- README.md
-- [ ] Evaluate Tufte CSS — borrow sidenote CSS or replace TailwindCSS entirely
+  - [ ] "Zero is beautiful"
-  - Option B: Steal Tufte's sidenote/margin-note CSS (adapt for dark theme), keep TailwindCSS
+- [ ] Content-hash fingerprinting for CSS and JS cache busting
-  - Option C: Full Tufte CSS — drop TailwindCSS, no build step, semantic HTML, customize for dark theme + Roboto
+- [ ] proper date/time/now object.
-  - Our sidenote HTML pattern already matches Tufte's exactly
 - [ ] Block attributes on code fences (`{ #ex-1 }`) — hello-world.md
 - [x] Emoji shortcodes (`:shrug:` etc.) — 2 instances
   - [ ] Backslash in shrug not visible.
 - [ ] include-code shortcode (`{{< include-code ... >}}`) — i-ported-fd-to-odin
 - [x] Nix build integration — main flake runs thor + tailwindcss instead of Hugo
-- [ ] Fix copy-to-clipboard button x-axis positioning inside code blocks
-- [ ] Content-hash fingerprinting for CSS and JS cache busting
 - [x] OpenGraph meta tags
 - [ ] Search up for `thor.json` files.
 - [ ] OpenGraph meta tags — verify all fields match production site
+- [ ] Table of contents support.
 - [ ] Nav items should be active when the current page is selected.
+- [ ] Theme selector for syntax highlighting.
+  - use http://github.com/helix-editor/helix/tree/master/runtime/themes) as a
+    guide
+- [ ] Search for grammars in multiple places
+- [ ] Download missing grammars.
 - [ ] Review every file in thor
   - [ ] Review alerts.odin
   - [ ] Review content.odin

content.odin

@@ -178,7 +178,7 @@ load_page :: proc(
 		expanded := expand_emoji(body)
 		clean_body, defs := strip_definitions(expanded)
 		html := cm.markdown_to_html_from_string(clean_body, {.Unsafe})
-		page.body_html = inject_alerts(inject_sidenotes(html, defs))
+		page.body_html = highlight_code(inject_alerts(inject_sidenotes(html, defs)), file_path)
 	}
 
 	switch page_type {

flake.nix

@@ -81,6 +81,7 @@
             buildInputs = [
               pkgs.git
               pkgs.cmark
+              pkgs.tree-sitter
             ];
 
             doCheck = true;
@@ -111,6 +112,7 @@
                 odin
                 ols
                 cmark
+                tree-sitter
 
                 # IDE
                 unstable.helix

highlight.odin

@@ -0,0 +1,426 @@
+package main
+
+import "core:fmt"
+import "core:log"
+import "core:os"
+import "core:strings"
+
+Grammar_Cache :: struct {
+	language: TSLanguage,
+	parser:   TSParser,
+	query:    TSQuery,
+}
+
+Get_Language_Proc :: #type proc() -> TSLanguage
+
+grammar_cache: map[string]^Grammar_Cache
+
+load_grammar :: proc(lang: string) -> ^Grammar_Cache {
+	if grammar_cache == nil {
+		grammar_cache = make(map[string]^Grammar_Cache)
+	}
+	if cached, ok := grammar_cache[lang]; ok {
+		return cached
+	}
+
+	// Cache nil by default so failures aren't retried.
+	grammar_cache[lang] = nil
+
+	if GRAPHS_PATH == "" {
+		log.warnf("highlight: no grammars path set, skipping %s", lang)
+		return nil
+	}
+
+	so_path := fmt.tprintf("%s/%s.so", GRAPHS_PATH, lang)
+	so_c := strings.clone_to_cstring(so_path)
+	defer delete(so_c)
+	handle := dlopen(so_c, RTLD_LAZY)
+	if handle == nil {
+		log.warnf("highlight: cannot load grammar %s (%s)", lang, so_path)
+		return nil
+	}
+
+	sym_name := fmt.tprintf("tree_sitter_%s", lang)
+	sym_c := strings.clone_to_cstring(sym_name)
+	defer delete(sym_c)
+	sym := dlsym(handle, sym_c)
+	if sym == nil {
+		log.errorf("highlight: cannot find symbol %s in %s", sym_name, so_path)
+		return nil
+	}
+	get_language := transmute(Get_Language_Proc)(sym)
+	language := get_language()
+
+	parser := ts_parser_new()
+	if parser == nil {
+		log.errorf("highlight: cannot create parser for %s", lang)
+		return nil
+	}
+	if !ts_parser_set_language(parser, language) {
+		log.errorf("highlight: ABI mismatch for %s grammar", lang)
+		ts_parser_delete(parser)
+		return nil
+	}
+
+	if QUERIES_PATH == "" {
+		log.warnf("highlight: no queries path set, skipping %s", lang)
+		ts_parser_delete(parser)
+		return nil
+	}
+
+	query_path := fmt.tprintf("%s/%s/highlights.scm", QUERIES_PATH, lang)
+	query_src, err := os.read_entire_file_from_path(query_path, context.allocator)
+	if err != nil {
+		log.warnf("highlight: cannot load query %s", query_path)
+		ts_parser_delete(parser)
+		return nil
+	}
+	query_str := string(query_src)
+	query_c := strings.clone_to_cstring(query_str)
+	defer delete(query_c)
+
+	err_offset: u32
+	err_type: TSQueryError
+	query := ts_query_new(
+		language,
+		query_c,
+		u32(len(query_src)),
+		&err_offset,
+		&err_type,
+	)
+	if query == nil {
+		tok := extract_query_token(query_src, err_offset)
+		cause := fmt.tprintf("query error at byte %d (type %v)", err_offset, err_type)
+		#partial switch err_type {
+		case .NodeType:
+			if tok != "" {
+				cause = fmt.tprintf("query references unknown node type '%s' (byte %d); the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions", tok, err_offset, lang)
+			} else {
+				cause = fmt.tprintf("query references an unknown node type at byte %d; the grammar (.so) and query (.scm) are likely from different tree-sitter-%s versions", err_offset, lang)
+			}
+		case .Field:
+			cause = fmt.tprintf("query references unknown field '%s' at byte %d", tok, err_offset)
+		case .Capture:
+			cause = fmt.tprintf("query uses an invalid capture '%s' at byte %d", tok, err_offset)
+		case .Syntax:
+			cause = fmt.tprintf("query has a syntax error at byte %d", err_offset)
+		case .Structure:
+			cause = fmt.tprintf("query has an illegal pattern structure at byte %d", err_offset)
+		case .Language:
+			cause = "grammar language is null (broken grammar .so)"
+		}
+		log.errorf("highlight: %s query failed: %s", lang, cause)
+
+		gram_v := helix_version_from_path(so_path)
+		query_v := helix_version_from_path(query_path)
+		gram_note := "(version unknown)"
+		if gram_v != "" do gram_note = fmt.tprintf("helix %s", gram_v)
+		query_note := "(version unknown)"
+		if query_v != "" do query_note = fmt.tprintf("helix %s", query_v)
+		log.errorf("  grammar: %s [%s]", so_path, gram_note)
+		log.errorf("  query:   %s [%s]", query_path, query_note)
+		if gram_v != "" && query_v != "" && gram_v != query_v {
+			log.errorf("  >> helix VERSION MISMATCH: grammar %s vs query %s", gram_v, query_v)
+		}
+
+		ts_parser_delete(parser)
+		return nil
+	}
+
+	gc := new(Grammar_Cache)
+	gc.language = language
+	gc.parser = parser
+	gc.query = query
+	grammar_cache[lang] = gc
+	return gc
+}
+
+extract_query_token :: proc(src: []byte, offset: u32) -> string {
+	end := offset
+	for int(end) < len(src) {
+		c := src[end]
+		is_ident := (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') ||
+			(c >= '0' && c <= '9') || c == '_' || c == '-' || c == '.'
+		if !is_ident do break
+		end += 1
+	}
+	if end <= offset do return ""
+	return string(src[offset:end])
+}
+
+helix_version_from_path :: proc(path: string) -> string {
+	tag := "-helix-"
+	idx := strings.index(path, tag)
+	if idx < 0 do return ""
+	start := idx + len(tag)
+	end := start
+	for end < len(path) {
+		c := path[end]
+		if !((c >= '0' && c <= '9') || c == '.') do break
+		end += 1
+	}
+	if end <= start do return ""
+	return path[start:end]
+}
+
+Capture :: struct {
+	start: u32,
+	end:   u32,
+	name:  string,
+}
+
+find_first_error_line :: proc(root: TSNode) -> int {
+	if ts_node_is_error(root) {
+		return int(ts_node_start_point(root).row) + 1
+	}
+	for i in 0..<ts_node_child_count(root) {
+		child := ts_node_child(root, u32(i))
+		if ts_node_has_error(child) {
+			line := find_first_error_line(child)
+			if line > 0 {
+				return line
+			}
+		}
+	}
+	return 0
+}
+
+capture_name_to_css :: proc(name: string) -> string {
+	sb := strings.builder_make()
+	strings.write_string(&sb, "hl-")
+	for i in 0..<len(name) {
+		if name[i] == '.' {
+			strings.write_byte(&sb, '-')
+		} else {
+			strings.write_byte(&sb, name[i])
+		}
+	}
+	return strings.to_string(sb)
+}
+
+escape_html :: proc(s: string) -> string {
+	parts: [dynamic]string
+	defer delete(parts)
+	start := 0
+	for i in 0..<len(s) {
+		switch s[i] {
+		case '&':
+			if i > start do append(&parts, s[start:i])
+			append(&parts, "&amp;")
+			start = i + 1
+		case '<':
+			if i > start do append(&parts, s[start:i])
+			append(&parts, "&lt;")
+			start = i + 1
+		case '>':
+			if i > start do append(&parts, s[start:i])
+			append(&parts, "&gt;")
+			start = i + 1
+		case '"':
+			if i > start do append(&parts, s[start:i])
+			append(&parts, "&quot;")
+			start = i + 1
+		}
+	}
+	if start < len(s) do append(&parts, s[start:])
+	if len(parts) == 0 do return s
+	return strings.join(parts[:], "")
+}
+
+unescape_html :: proc(s: string) -> string {
+	parts: [dynamic]string
+	defer delete(parts)
+	start := 0
+	for i in 0..<len(s) {
+		if s[i] != '&' do continue
+		semi := strings.index(s[i:], ";")
+		if semi < 0 do break
+		entity := s[i : i + semi + 1]
+		replacement := ""
+		switch entity {
+		case "&amp;":  replacement = "&"
+		case "&lt;":   replacement = "<"
+		case "&gt;":   replacement = ">"
+		case "&quot;": replacement = "\""
+		case "&#39;", "&apos;": replacement = "'"
+		case: continue
+		}
+		if i > start do append(&parts, s[start:i])
+		append(&parts, replacement)
+		start = i + semi + 1
+	}
+	if start < len(s) do append(&parts, s[start:])
+	if len(parts) == 0 do return s
+	return strings.join(parts[:], "")
+}
+
+highlight_block :: proc(code: string, lang: string, file_path: string) -> string {
+	gc := load_grammar(lang)
+	if gc == nil {
+		return code
+	}
+
+	raw_code := unescape_html(code)
+	raw_c := strings.clone_to_cstring(raw_code)
+	defer delete(raw_c)
+
+	tree := ts_parser_parse_string(gc.parser, nil, raw_c, u32(len(raw_code)))
+	if tree == nil {
+		return code
+	}
+	defer ts_tree_delete(tree)
+
+	root := ts_tree_root_node(tree)
+
+	if ts_node_has_error(root) {
+		line := find_first_error_line(root)
+		if line > 0 {
+			log.warnf("highlight: syntax errors in %s code block at line %d (%s)", lang, line, file_path)
+		} else {
+			log.warnf("highlight: syntax errors in %s code block (%s)", lang, file_path)
+		}
+	}
+
+	cursor := ts_query_cursor_new()
+	if cursor == nil {
+		return code
+	}
+	defer ts_query_cursor_delete(cursor)
+
+	ts_query_cursor_exec(cursor, gc.query, root)
+
+	captures: [dynamic]Capture
+	defer delete(captures)
+
+	match: TSQueryMatch
+	capture_idx: u32
+	for ts_query_cursor_next_capture(cursor, &match, &capture_idx) {
+		if capture_idx >= u32(match.capture_count) {
+			continue
+		}
+		cap := match.captures[capture_idx]
+		name_len: u32
+		name_c := ts_query_capture_name_for_id(gc.query, cap.index, &name_len)
+		if name_c == nil {
+			continue
+		}
+		name_full := string(name_c)
+		name := name_full
+		if len(name_full) > int(name_len) {
+			name = name_full[:int(name_len)]
+		}
+		append(&captures, Capture{
+			start = ts_node_start_byte(cap.node),
+			end = ts_node_end_byte(cap.node),
+			name = name,
+		})
+	}
+
+	if len(captures) == 0 {
+		return code
+	}
+
+	sb := strings.builder_make()
+
+	last_pos: u32 = 0
+	stack: [dynamic]Capture
+	defer delete(stack)
+
+	for cap in captures {
+		for len(stack) > 0 {
+			top := stack[len(stack) - 1]
+			if top.end <= cap.start {
+				if top.end > last_pos {
+					strings.write_string(&sb, escape_html(raw_code[last_pos:top.end]))
+				}
+				strings.write_string(&sb, "</span>")
+				last_pos = top.end
+				pop(&stack)
+			} else {
+				break
+			}
+		}
+
+		if cap.start > last_pos {
+			strings.write_string(&sb, escape_html(raw_code[last_pos:cap.start]))
+			last_pos = cap.start
+		}
+
+		css_class := capture_name_to_css(cap.name)
+		strings.write_string(&sb, fmt.tprintf("<span class=\"%s\">", css_class))
+		append(&stack, cap)
+	}
+
+	for len(stack) > 0 {
+		top := pop(&stack)
+		if top.end > last_pos {
+			strings.write_string(&sb, escape_html(raw_code[last_pos:top.end]))
+		}
+		strings.write_string(&sb, "</span>")
+		last_pos = top.end
+	}
+
+	if int(last_pos) < len(raw_code) {
+		strings.write_string(&sb, escape_html(raw_code[last_pos:]))
+	}
+
+	return strings.to_string(sb)
+}
+
+highlight_code :: proc(html: string, file_path: string) -> string {
+	PREFIX :: `<pre><code class="language-`
+	CODE_END :: `</code></pre>`
+
+	parts: [dynamic]string
+	defer delete(parts)
+	pos := 0
+
+	for {
+		rel := strings.index(html[pos:], PREFIX)
+		if rel < 0 {
+			break
+		}
+		idx := pos + rel
+
+		if idx > pos {
+			append(&parts, html[pos:idx])
+		}
+
+		lang_start := idx + len(PREFIX)
+		lang_end_rel := strings.index(html[lang_start:], `"`)
+		if lang_end_rel < 0 {
+			break
+		}
+		lang_end := lang_start + lang_end_rel
+		lang := html[lang_start:lang_end]
+
+		code_start := lang_end + 1
+		if code_start < len(html) && html[code_start] == '>' {
+			code_start += 1
+		} else {
+			pos = lang_end
+			continue
+		}
+
+		end_rel := strings.index(html[code_start:], CODE_END)
+		if end_rel < 0 {
+			break
+		}
+		end_idx := code_start + end_rel
+
+		code := html[code_start:end_idx]
+		highlighted := highlight_block(code, lang, file_path)
+		append(&parts, fmt.tprintf(`<pre><code class="language-%s">%s</code></pre>`, lang, highlighted))
+
+		pos = end_idx + len(CODE_END)
+	}
+
+	if pos < len(html) {
+		append(&parts, html[pos:])
+	}
+
+	if len(parts) == 0 {
+		return html
+	}
+	return strings.join(parts[:], "")
+}

main.odin

@@ -5,6 +5,7 @@ import "core:os"
 
 main :: proc() {
 	console_logger := log.create_console_logger()
+	context.logger = console_logger
 	defer log.destroy_console_logger(console_logger)
 
 	site: Site

tree_sitter.odin

@@ -0,0 +1,115 @@
+package main
+
+import "core:c"
+
+GRAPHS_PATH: string = "/home/spencer/.config/helix/runtime/grammars"
+QUERIES_PATH: string = "/nix/store/n9da8d007ygbgsx983jr3ar3wb1fsh6q-helix-25.07.1/lib/runtime/queries"
+
+TSLanguage :: distinct rawptr
+TSParser :: distinct rawptr
+TSTree :: distinct rawptr
+TSQuery :: distinct rawptr
+TSQueryCursor :: distinct rawptr
+
+TSPoint :: struct {
+	row:    u32,
+	column: u32,
+}
+
+TSNode :: struct {
+	ctx:   [4]u32,
+	id:    rawptr,
+	tree:  rawptr,
+}
+
+TSQueryCapture :: struct {
+	node:  TSNode,
+	index: u32,
+	_:     u32,
+}
+
+TSQueryMatch :: struct {
+	id:            u32,
+	pattern_index: u16,
+	capture_count: u16,
+	captures:      [^]TSQueryCapture,
+}
+
+TSQueryError :: enum c.int {
+	None     = 0,
+	Syntax,
+	NodeType,
+	Field,
+	Capture,
+	Structure,
+	Language,
+}
+
+RTLD_LAZY :: c.int(1)
+
+foreign import lib "system:tree-sitter"
+foreign import libdl "system:dl"
+
+foreign lib {
+	ts_parser_new :: proc() -> TSParser ---
+	ts_parser_delete :: proc(self: TSParser) ---
+	ts_parser_set_language :: proc(self: TSParser, language: TSLanguage) -> bool ---
+	ts_parser_parse_string :: proc(
+		self: TSParser,
+		old_tree: TSTree,
+		string: cstring,
+		length: u32,
+	) -> TSTree ---
+}
+
+foreign lib {
+	ts_tree_root_node :: proc(self: TSTree) -> TSNode ---
+	ts_tree_delete :: proc(self: TSTree) ---
+}
+
+foreign lib {
+	ts_node_start_byte :: proc(self: TSNode) -> u32 ---
+	ts_node_end_byte :: proc(self: TSNode) -> u32 ---
+	ts_node_has_error :: proc(self: TSNode) -> bool ---
+	ts_node_is_error :: proc(self: TSNode) -> bool ---
+	ts_node_child_count :: proc(self: TSNode) -> u32 ---
+	ts_node_child :: proc(self: TSNode, child_index: u32) -> TSNode ---
+	ts_node_start_point :: proc(self: TSNode) -> TSPoint ---
+}
+
+foreign lib {
+	ts_query_new :: proc(
+		language: TSLanguage,
+		source: cstring,
+		source_len: u32,
+		error_offset: ^u32,
+		error_type: ^TSQueryError,
+	) -> TSQuery ---
+	ts_query_delete :: proc(self: TSQuery) ---
+	ts_query_capture_name_for_id :: proc(
+		self: TSQuery,
+		index: u32,
+		length: ^u32,
+	) -> cstring ---
+}
+
+foreign lib {
+	ts_query_cursor_new :: proc() -> TSQueryCursor ---
+	ts_query_cursor_delete :: proc(self: TSQueryCursor) ---
+	ts_query_cursor_exec :: proc(
+		self: TSQueryCursor,
+		query: TSQuery,
+		node: TSNode,
+	) ---
+	ts_query_cursor_next_capture :: proc(
+		self: TSQueryCursor,
+		match: ^TSQueryMatch,
+		capture_index: ^u32,
+	) -> bool ---
+}
+
+foreign libdl {
+	dlopen :: proc(filename: cstring, flags: c.int) -> rawptr ---
+	dlsym :: proc(handle: rawptr, symbol: cstring) -> rawptr ---
+	dlclose :: proc(handle: rawptr) -> c.int ---
+}