From 9b732c4879262cbb2af9429846319845bfa45d41 Mon Sep 17 00:00:00 2001 From: Spencer Brower <6729162+sbrow@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:25:47 -0400 Subject: [PATCH] feat: Added `-minify` flag that does simple html/css minification. --- TODOS.md | 2 + content.odin | 58 +++++++++++ flake.nix | 103 +++++++++++++------ highlight.odin | 125 ++++++++++++++--------- minify.odin | 253 +++++++++++++++++++++++++++++++++++++++++++++++ render.odin | 12 +++ site.odin | 15 +++ tree_sitter.odin | 14 +++ 8 files changed, 506 insertions(+), 76 deletions(-) create mode 100644 minify.odin diff --git a/TODOS.md b/TODOS.md index 35c93c9..884d32c 100644 --- a/TODOS.md +++ b/TODOS.md @@ -14,6 +14,7 @@ - [ ] ensure sidenote numbers render in display order and not in declaration order. - [x] Search up for `thor.json` files. - [ ] OpenGraph meta tags — verify all fields match production site +- [ ] set opengraph tags / description automatically if unset. (Like hugo does) - [ ] Table of contents support. - [ ] Nav items should be active when the current page is selected. - [ ] Theme selector for syntax highlighting. @@ -32,6 +33,7 @@ - [ ] event based - [ ] Free cmark HTML output (`body_html`) — cmark allocates via C malloc, not the arena, so it leaks per iteration in watch mode - [ ] commands +- [ ] Add spall and find ways to reduce run time. (Getting close to 1/2sec here.) - [ ] Review every file in thor - [ ] Review alerts.odin - [ ] Review content.odin diff --git a/content.odin b/content.odin index 462495f..6de0e3d 100644 --- a/content.odin +++ b/content.odin @@ -255,3 +255,61 @@ copy_static_recursive :: proc(current: string, rel_prefix: string, output_dir: s } } +// copy_assets_dir recursively copies files from assets_dir to output_dir. +// .css files are minified when .Minify is enabled; all other files are copied +// verbatim with a warning suggesting they belong in static/. +// Silently skips if assets_dir doesn't exist. +copy_assets_dir :: proc(assets_dir: string, output_dir: string, features: bit_set[Feature]) { + if !os.exists(assets_dir) { + return + } + copy_assets_recursive(assets_dir, "", output_dir, features) +} + +copy_assets_recursive :: proc( + current: string, + rel_prefix: string, + output_dir: string, + features: bit_set[Feature], +) { + entries, err := os.read_all_directory_by_path(current, context.allocator) + if err != nil { + log.warnf("thor: cannot read %s: %v", current, err) + return + } + defer os.file_info_slice_delete(entries, context.allocator) + + for entry in entries { + rel := rel_prefix == "" ? entry.name : fmt.tprintf("%s/%s", rel_prefix, entry.name) + switch entry.type { + case .Regular: + dest := fmt.tprintf("%s/%s", output_dir, rel) + if idx := strings.last_index(dest, "/"); idx >= 0 { + if err := os.make_directory_all(dest[:idx]); err != nil && err != .Exist { + log.warnf("thor: cannot create %s: %v", dest[:idx], err) + continue + } + } + if .Minify in features && strings.has_suffix(entry.name, ".css") { + data, read_err := os.read_entire_file_from_path(entry.fullpath, context.allocator) + if read_err != nil { + log.warnf("thor: cannot read %s: %v", entry.fullpath, read_err) + continue + } + minified := minify_css(string(data)) + write_file(dest, minified) + } else { + if !strings.has_suffix(entry.name, ".css") { + log.warnf("thor: %s is not a CSS file; consider moving it to static/", entry.fullpath) + } + if err := os.copy_file(dest, entry.fullpath); err != nil { + log.warnf("thor: cannot copy %s: %v", entry.fullpath, err) + } + } + case .Directory: + copy_assets_recursive(entry.fullpath, rel, output_dir, features) + case .Undetermined, .Symlink, .Named_Pipe, .Socket, .Block_Device, .Character_Device: + } + } +} + diff --git a/flake.nix b/flake.nix index 7cb486a..bde799f 100644 --- a/flake.nix +++ b/flake.nix @@ -12,12 +12,13 @@ }; outputs = - inputs@{ self - , flake-parts - , nixpkgs - , nixpkgs-unstable - # , process-compose-flake - , treefmt-nix + inputs@{ + self, + flake-parts, + nixpkgs, + nixpkgs-unstable, + # , process-compose-flake + treefmt-nix, }: flake-parts.lib.mkFlake { inherit inputs; } { imports = [ @@ -27,7 +28,42 @@ systems = [ "x86_64-linux" ]; perSystem = - { pkgs, system, inputs', ... }: { + { + pkgs, + system, + inputs', + ... + }: + let + mkGrammarStaticLib = name: src: pkgs.stdenv.mkDerivation { + inherit name src; + dontConfigure = true; + + buildPhase = '' + runHook preBuild + if [ -f src/scanner.cc ]; then + $CXX -fPIC -c src/scanner.cc -o scanner.o + elif [ -f src/scanner.c ]; then + $CC -fPIC -c src/scanner.c -o scanner.o + fi + $CC -fPIC -c src/parser.c -o parser.o + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + mkdir -p $out/lib + ar rcs $out/lib/lib${name}.a *.o + runHook postInstall + ''; + }; + + html-grammar = mkGrammarStaticLib "tree-sitter-html" + pkgs.tree-sitter-grammars.tree-sitter-html.src; + css-grammar = mkGrammarStaticLib "tree-sitter-css" + pkgs.tree-sitter-grammars.tree-sitter-css.src; + in + { _module.args.pkgs = import nixpkgs { inherit system; config.allowUnfree = true; @@ -48,22 +84,20 @@ ".env.local" ]; - # Format nix files programs.nixpkgs-fmt.enable = true; programs.deadnix.enable = true; # Format js, json, and yaml files programs.prettier.enable = true; - settings.formatter.prettier = - { - excludes = [ - "public/**" - "resources/js/modernizr.js" - "storage/app/caniuse.json" - "*.md" - ]; - }; + settings.formatter.prettier = { + excludes = [ + "public/**" + "resources/js/modernizr.js" + "storage/app/caniuse.json" + "*.md" + ]; + }; }; #process-compose.default.settings.processes = { }; @@ -82,8 +116,12 @@ pkgs.git pkgs.cmark pkgs.tree-sitter + html-grammar + css-grammar ]; + LIBRARY_PATH = "${html-grammar}/lib:${css-grammar}/lib"; + doCheck = true; checkPhase = '' runHook preCheck @@ -94,8 +132,6 @@ buildPhase = '' runHook preBuild odin build . -o:speed -out:${pname}-keep - echo "Listing filles..." - ls . runHook postBuild ''; @@ -106,20 +142,23 @@ ''; }; - devShells.default = pkgs.mkShell - { - buildInputs = with pkgs; [ - odin - ols - cmark - tree-sitter + devShells.default = pkgs.mkShell { + buildInputs = with pkgs; [ + odin + ols + cmark + tree-sitter - # IDE - unstable.helix - typescript-language-server - vscode-langservers-extracted - ]; - }; + # IDE + unstable.helix + typescript-language-server + vscode-langservers-extracted + ]; + + shellHook = '' + export LIBRARY_PATH="${html-grammar}/lib:${css-grammar}/lib:$LIBRARY_PATH" + ''; + }; }; }; } diff --git a/highlight.odin b/highlight.odin index 63d7859..4bc8591 100644 --- a/highlight.odin +++ b/highlight.odin @@ -6,16 +6,29 @@ import "core:os" import "core:strings" Grammar_Cache :: struct { - language: TSLanguage, - parser: TSParser, - query: TSQuery, + language: TSLanguage, + parser: TSParser, + query: TSQuery, + query_failed: bool, } Get_Language_Proc :: #type proc() -> TSLanguage grammar_cache: map[string]^Grammar_Cache -load_grammar :: proc(lang: string) -> ^Grammar_Cache { +builtin_language :: proc(lang: string) -> (language: TSLanguage, ok: bool) { + switch lang { + case "html": + language = tree_sitter_html() + ok = true + case "css": + language = tree_sitter_css() + ok = true + } + return +} + +ensure_parser :: proc(lang: string) -> ^Grammar_Cache { if grammar_cache == nil { grammar_cache = make(map[string]^Grammar_Cache) } @@ -23,33 +36,38 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache { return cached } - // Cache nil by default so failures aren't retried. grammar_cache[lang] = nil - if GRAPHS_PATH == "" { - log.warnf("highlight: no grammars path set, skipping %s", lang) - return nil - } + language: TSLanguage - so_path := fmt.tprintf("%s/%s.so", GRAPHS_PATH, lang) - so_c := strings.clone_to_cstring(so_path) - defer delete(so_c) - handle := dlopen(so_c, RTLD_LAZY) - if handle == nil { - log.warnf("highlight: cannot load grammar %s (%s)", lang, so_path) - return nil - } + if builtin, ok := builtin_language(lang); ok { + language = builtin + } else { + if GRAPHS_PATH == "" { + log.warnf("highlight: no grammars path set, skipping %s", lang) + return nil + } - sym_name := fmt.tprintf("tree_sitter_%s", lang) - sym_c := strings.clone_to_cstring(sym_name) - defer delete(sym_c) - sym := dlsym(handle, sym_c) - if sym == nil { - log.errorf("highlight: cannot find symbol %s in %s", sym_name, so_path) - return nil + so_path := fmt.tprintf("%s/%s.so", GRAPHS_PATH, lang) + so_c := strings.clone_to_cstring(so_path) + defer delete(so_c) + handle := dlopen(so_c, RTLD_LAZY) + if handle == nil { + log.warnf("highlight: cannot load grammar %s (%s)", lang, so_path) + return nil + } + + sym_name := fmt.tprintf("tree_sitter_%s", lang) + sym_c := strings.clone_to_cstring(sym_name) + defer delete(sym_c) + sym := dlsym(handle, sym_c) + if sym == nil { + log.errorf("highlight: cannot find symbol %s in %s", sym_name, so_path) + return nil + } + get_language := transmute(Get_Language_Proc)(sym) + language = get_language() } - get_language := transmute(Get_Language_Proc)(sym) - language := get_language() parser := ts_parser_new() if parser == nil { @@ -62,9 +80,28 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache { return nil } + gc := new(Grammar_Cache) + gc.language = language + gc.parser = parser + grammar_cache[lang] = gc + return gc +} + +load_grammar :: proc(lang: string) -> ^Grammar_Cache { + gc := ensure_parser(lang) + if gc == nil { + return nil + } + if gc.query != nil { + return gc + } + if gc.query_failed { + return nil + } + if QUERIES_PATH == "" { log.warnf("highlight: no queries path set, skipping %s", lang) - ts_parser_delete(parser) + gc.query_failed = true return nil } @@ -72,7 +109,7 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache { query_src, err := os.read_entire_file_from_path(query_path, context.allocator) if err != nil { log.warnf("highlight: cannot load query %s", query_path) - ts_parser_delete(parser) + gc.query_failed = true return nil } query_str := string(query_src) @@ -82,7 +119,7 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache { err_offset: u32 err_type: TSQueryError query := ts_query_new( - language, + gc.language, query_c, u32(len(query_src)), &err_offset, @@ -111,27 +148,27 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache { } log.errorf("highlight: %s query failed: %s", lang, cause) - gram_v := helix_version_from_path(so_path) - query_v := helix_version_from_path(query_path) - gram_note := "(version unknown)" - if gram_v != "" do gram_note = fmt.tprintf("helix %s", gram_v) - query_note := "(version unknown)" - if query_v != "" do query_note = fmt.tprintf("helix %s", query_v) - log.errorf(" grammar: %s [%s]", so_path, gram_note) - log.errorf(" query: %s [%s]", query_path, query_note) - if gram_v != "" && query_v != "" && gram_v != query_v { - log.errorf(" >> helix VERSION MISMATCH: grammar %s vs query %s", gram_v, query_v) + _, is_builtin := builtin_language(lang) + if !is_builtin { + so_path := fmt.tprintf("%s/%s.so", GRAPHS_PATH, lang) + gram_v := helix_version_from_path(so_path) + query_v := helix_version_from_path(query_path) + gram_note := "(version unknown)" + if gram_v != "" do gram_note = fmt.tprintf("helix %s", gram_v) + query_note := "(version unknown)" + if query_v != "" do query_note = fmt.tprintf("helix %s", query_v) + log.errorf(" grammar: %s [%s]", so_path, gram_note) + log.errorf(" query: %s [%s]", query_path, query_note) + if gram_v != "" && query_v != "" && gram_v != query_v { + log.errorf(" >> helix VERSION MISMATCH: grammar %s vs query %s", gram_v, query_v) + } } - ts_parser_delete(parser) + gc.query_failed = true return nil } - gc := new(Grammar_Cache) - gc.language = language - gc.parser = parser gc.query = query - grammar_cache[lang] = gc return gc } diff --git a/minify.odin b/minify.odin new file mode 100644 index 0000000..a68fd9f --- /dev/null +++ b/minify.odin @@ -0,0 +1,253 @@ +package main + +import "core:log" +import "core:strings" + +PRESERVE_TAGS :: [?]string{"pre", "code", "textarea"} + +Range :: struct { + start: u32, + end: u32, +} + +minify_html :: proc(source: string) -> string { + gc := ensure_parser("html") + if gc == nil { + return source + } + + source_c := strings.clone_to_cstring(source) + defer delete(source_c) + + tree := ts_parser_parse_string(gc.parser, nil, source_c, u32(len(source))) + if tree == nil { + return source + } + defer ts_tree_delete(tree) + + root := ts_tree_root_node(tree) + + if ts_node_has_error(root) { + log.warnf("minify: HTML parse errors, skipping minification") + return source + } + + comments: [dynamic]Range + defer delete(comments) + preserves: [dynamic]Range + defer delete(preserves) + + collect_html_ranges(root, source, &comments, &preserves) + + sb := strings.builder_make() + + ci := 0 + pi := 0 + i := 0 + last_written: u8 = 0 + + for i < len(source) { + if pi < len(preserves) && u32(i) >= preserves[pi].start { + p := preserves[pi] + segment := source[i:p.end] + strings.write_string(&sb, segment) + if len(segment) > 0 { + last_written = segment[len(segment)-1] + } + i = int(p.end) + pi += 1 + continue + } + + if ci < len(comments) && u32(i) >= comments[ci].start { + i = int(comments[ci].end) + ci += 1 + continue + } + + c := source[i] + if c == ' ' || c == '\t' || c == '\n' || c == '\r' { + j := i + 1 + for j < len(source) { + c2 := source[j] + if c2 != ' ' && c2 != '\t' && c2 != '\n' && c2 != '\r' { + break + } + j += 1 + } + next: u8 = 0 + if j < len(source) { + next = source[j] + } + if last_written != '>' || next != '<' { + strings.write_byte(&sb, ' ') + last_written = ' ' + } + i = j + } else { + strings.write_byte(&sb, c) + last_written = c + i += 1 + } + } + + return strings.to_string(sb) +} + +collect_html_ranges :: proc( + node: TSNode, + source: string, + comments: ^[dynamic]Range, + preserves: ^[dynamic]Range, +) { + child_count := ts_node_named_child_count(node) + for i in 0.. string { + child_count := ts_node_named_child_count(element) + for i in 0.. bool { + for t in PRESERVE_TAGS { + if tag == t do return true + } + return false +} + +CSS_DELIMS :: [?]u8{'{', '}', ':', ';', ','} + +is_css_delim :: proc(c: u8) -> bool { + for d in CSS_DELIMS { + if c == d do return true + } + return false +} + +minify_css :: proc(source: string) -> string { + gc := ensure_parser("css") + if gc == nil { + return source + } + + source_c := strings.clone_to_cstring(source) + defer delete(source_c) + + tree := ts_parser_parse_string(gc.parser, nil, source_c, u32(len(source))) + if tree == nil { + return source + } + defer ts_tree_delete(tree) + + root := ts_tree_root_node(tree) + + if ts_node_has_error(root) { + log.warnf("minify: CSS parse errors, skipping minification") + return source + } + + comments: [dynamic]Range + defer delete(comments) + + collect_css_comments(root, &comments) + + sb := strings.builder_make() + + ci := 0 + i := 0 + last_written: u8 = 0 + + for i < len(source) { + if ci < len(comments) && u32(i) >= comments[ci].start { + i = int(comments[ci].end) + ci += 1 + continue + } + + c := source[i] + if c == ' ' || c == '\t' || c == '\n' || c == '\r' { + j := i + 1 + for j < len(source) { + c2 := source[j] + if c2 != ' ' && c2 != '\t' && c2 != '\n' && c2 != '\r' { + break + } + j += 1 + } + + next: u8 = 0 + if j < len(source) { + next = source[j] + } + + if !is_css_delim(last_written) && !is_css_delim(next) { + strings.write_byte(&sb, ' ') + last_written = ' ' + } + i = j + } else { + strings.write_byte(&sb, c) + last_written = c + i += 1 + } + } + + return strings.to_string(sb) +} + +collect_css_comments :: proc(node: TSNode, comments: ^[dynamic]Range) { + child_count := ts_node_named_child_count(node) + for i in 0.. mem.Allocator { diff --git a/tree_sitter.odin b/tree_sitter.odin index 33ea476..12f10fa 100644 --- a/tree_sitter.odin +++ b/tree_sitter.odin @@ -49,6 +49,8 @@ RTLD_LAZY :: c.int(1) foreign import lib "system:tree-sitter" foreign import libdl "system:dl" +foreign import html_grammar "system:tree-sitter-html" +foreign import css_grammar "system:tree-sitter-css" foreign lib { ts_parser_new :: proc() -> TSParser --- @@ -74,7 +76,11 @@ foreign lib { ts_node_is_error :: proc(self: TSNode) -> bool --- ts_node_child_count :: proc(self: TSNode) -> u32 --- ts_node_child :: proc(self: TSNode, child_index: u32) -> TSNode --- + ts_node_named_child_count :: proc(self: TSNode) -> u32 --- + ts_node_named_child :: proc(self: TSNode, child_index: u32) -> TSNode --- ts_node_start_point :: proc(self: TSNode) -> TSPoint --- + ts_node_type :: proc(self: TSNode) -> cstring --- + ts_node_parent :: proc(self: TSNode) -> TSNode --- } foreign lib { @@ -113,3 +119,11 @@ foreign libdl { dlsym :: proc(handle: rawptr, symbol: cstring) -> rawptr --- dlclose :: proc(handle: rawptr) -> c.int --- } + +foreign html_grammar { + tree_sitter_html :: proc() -> TSLanguage --- +} + +foreign css_grammar { + tree_sitter_css :: proc() -> TSLanguage --- +}