feat: Added -minify flag that does simple html/css minification.

TODOS.md

@@ -14,6 +14,7 @@
 - [ ] ensure sidenote numbers render in display order and not in declaration order.
 - [x] Search up for `thor.json` files.
 - [ ] OpenGraph meta tags — verify all fields match production site
+- [ ] set opengraph tags / description automatically if unset. (Like hugo does)
 - [ ] Table of contents support.
 - [ ] Nav items should be active when the current page is selected.
 - [ ] Theme selector for syntax highlighting.
@@ -32,6 +33,7 @@
   - [ ] event based
 - [ ] Free cmark HTML output (`body_html`) — cmark allocates via C malloc, not the arena, so it leaks per iteration in watch mode
 - [ ] commands
+- [ ] Add spall and find ways to reduce run time. (Getting close to 1/2sec here.)
 - [ ] Review every file in thor
   - [ ] Review alerts.odin
   - [ ] Review content.odin

content.odin

@@ -255,3 +255,61 @@ copy_static_recursive :: proc(current: string, rel_prefix: string, output_dir: s
 	}
 }
 
+// copy_assets_dir recursively copies files from assets_dir to output_dir.
+// .css files are minified when .Minify is enabled; all other files are copied
+// verbatim with a warning suggesting they belong in static/.
+// Silently skips if assets_dir doesn't exist.
+copy_assets_dir :: proc(assets_dir: string, output_dir: string, features: bit_set[Feature]) {
+	if !os.exists(assets_dir) {
+		return
+	}
+	copy_assets_recursive(assets_dir, "", output_dir, features)
+}
+
+copy_assets_recursive :: proc(
+	current: string,
+	rel_prefix: string,
+	output_dir: string,
+	features: bit_set[Feature],
+) {
+	entries, err := os.read_all_directory_by_path(current, context.allocator)
+	if err != nil {
+		log.warnf("thor: cannot read %s: %v", current, err)
+		return
+	}
+	defer os.file_info_slice_delete(entries, context.allocator)
+
+	for entry in entries {
+		rel := rel_prefix == "" ? entry.name : fmt.tprintf("%s/%s", rel_prefix, entry.name)
+		switch entry.type {
+		case .Regular:
+			dest := fmt.tprintf("%s/%s", output_dir, rel)
+			if idx := strings.last_index(dest, "/"); idx >= 0 {
+				if err := os.make_directory_all(dest[:idx]); err != nil && err != .Exist {
+					log.warnf("thor: cannot create %s: %v", dest[:idx], err)
+					continue
+				}
+			}
+			if .Minify in features && strings.has_suffix(entry.name, ".css") {
+				data, read_err := os.read_entire_file_from_path(entry.fullpath, context.allocator)
+				if read_err != nil {
+					log.warnf("thor: cannot read %s: %v", entry.fullpath, read_err)
+					continue
+				}
+				minified := minify_css(string(data))
+				write_file(dest, minified)
+			} else {
+				if !strings.has_suffix(entry.name, ".css") {
+					log.warnf("thor: %s is not a CSS file; consider moving it to static/", entry.fullpath)
+				}
+				if err := os.copy_file(dest, entry.fullpath); err != nil {
+					log.warnf("thor: cannot copy %s: %v", entry.fullpath, err)
+				}
+			}
+		case .Directory:
+			copy_assets_recursive(entry.fullpath, rel, output_dir, features)
+		case .Undetermined, .Symlink, .Named_Pipe, .Socket, .Block_Device, .Character_Device:
+		}
+	}
+}
+

flake.nix

@@ -12,12 +12,13 @@
   };
 
   outputs =
-    inputs@{ self
+    inputs@{
-    , flake-parts
+      self,
-    , nixpkgs
+      flake-parts,
-    , nixpkgs-unstable
+      nixpkgs,
+      nixpkgs-unstable,
       # , process-compose-flake
-    , treefmt-nix
+      treefmt-nix,
     }:
     flake-parts.lib.mkFlake { inherit inputs; } {
       imports = [
@@ -27,7 +28,42 @@
       systems = [ "x86_64-linux" ];
 
       perSystem =
-        { pkgs, system, inputs', ... }: {
+        {
+          pkgs,
+          system,
+          inputs',
+          ...
+        }:
+        let
+          mkGrammarStaticLib = name: src: pkgs.stdenv.mkDerivation {
+            inherit name src;
+            dontConfigure = true;
+
+            buildPhase = ''
+              runHook preBuild
+              if [ -f src/scanner.cc ]; then
+                $CXX -fPIC -c src/scanner.cc -o scanner.o
+              elif [ -f src/scanner.c ]; then
+                $CC -fPIC -c src/scanner.c -o scanner.o
+              fi
+              $CC -fPIC -c src/parser.c -o parser.o
+              runHook postBuild
+            '';
+
+            installPhase = ''
+              runHook preInstall
+              mkdir -p $out/lib
+              ar rcs $out/lib/lib${name}.a *.o
+              runHook postInstall
+            '';
+          };
+
+          html-grammar = mkGrammarStaticLib "tree-sitter-html"
+            pkgs.tree-sitter-grammars.tree-sitter-html.src;
+          css-grammar = mkGrammarStaticLib "tree-sitter-css"
+            pkgs.tree-sitter-grammars.tree-sitter-css.src;
+        in
+        {
           _module.args.pkgs = import nixpkgs {
             inherit system;
             config.allowUnfree = true;
@@ -48,15 +84,13 @@
               ".env.local"
             ];
 
-
             # Format nix files
             programs.nixpkgs-fmt.enable = true;
             programs.deadnix.enable = true;
 
             # Format js, json, and yaml files
             programs.prettier.enable = true;
-            settings.formatter.prettier =
+            settings.formatter.prettier = {
-              {
               excludes = [
                 "public/**"
                 "resources/js/modernizr.js"
@@ -82,8 +116,12 @@
               pkgs.git
               pkgs.cmark
               pkgs.tree-sitter
+              html-grammar
+              css-grammar
             ];
 
+            LIBRARY_PATH = "${html-grammar}/lib:${css-grammar}/lib";
+
             doCheck = true;
             checkPhase = ''
               runHook preCheck
@@ -94,8 +132,6 @@
             buildPhase = ''
               runHook preBuild
               odin build . -o:speed -out:${pname}-keep
-              echo "Listing filles..."
-              ls .
               runHook postBuild
             '';
 
@@ -106,8 +142,7 @@
             '';
           };
 
-          devShells.default = pkgs.mkShell
+          devShells.default = pkgs.mkShell {
-            {
             buildInputs = with pkgs; [
               odin
               ols
@@ -119,6 +154,10 @@
               typescript-language-server
               vscode-langservers-extracted
             ];
+
+            shellHook = ''
+              export LIBRARY_PATH="${html-grammar}/lib:${css-grammar}/lib:$LIBRARY_PATH"
+            '';
           };
         };
     };

highlight.odin

@@ -9,13 +9,26 @@ Grammar_Cache :: struct {
 	language:     TSLanguage,
 	parser:       TSParser,
 	query:        TSQuery,
+	query_failed: bool,
 }
 
 Get_Language_Proc :: #type proc() -> TSLanguage
 
 grammar_cache: map[string]^Grammar_Cache
 
-load_grammar :: proc(lang: string) -> ^Grammar_Cache {
+builtin_language :: proc(lang: string) -> (language: TSLanguage, ok: bool) {
+	switch lang {
+	case "html":
+		language = tree_sitter_html()
+		ok = true
+	case "css":
+		language = tree_sitter_css()
+		ok = true
+	}
+	return
+}
+
+ensure_parser :: proc(lang: string) -> ^Grammar_Cache {
 	if grammar_cache == nil {
 		grammar_cache = make(map[string]^Grammar_Cache)
 	}
@@ -23,9 +36,13 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {
 		return cached
 	}
 
-	// Cache nil by default so failures aren't retried.
 	grammar_cache[lang] = nil
 
+	language: TSLanguage
+
+	if builtin, ok := builtin_language(lang); ok {
+		language = builtin
+	} else {
 		if GRAPHS_PATH == "" {
 			log.warnf("highlight: no grammars path set, skipping %s", lang)
 			return nil
@@ -49,7 +66,8 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {
 			return nil
 		}
 		get_language := transmute(Get_Language_Proc)(sym)
-	language := get_language()
+		language = get_language()
+	}
 
 	parser := ts_parser_new()
 	if parser == nil {
@@ -62,9 +80,28 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {
 		return nil
 	}
 
+	gc := new(Grammar_Cache)
+	gc.language = language
+	gc.parser = parser
+	grammar_cache[lang] = gc
+	return gc
+}
+
+load_grammar :: proc(lang: string) -> ^Grammar_Cache {
+	gc := ensure_parser(lang)
+	if gc == nil {
+		return nil
+	}
+	if gc.query != nil {
+		return gc
+	}
+	if gc.query_failed {
+		return nil
+	}
+
 	if QUERIES_PATH == "" {
 		log.warnf("highlight: no queries path set, skipping %s", lang)
-		ts_parser_delete(parser)
+		gc.query_failed = true
 		return nil
 	}
 
@@ -72,7 +109,7 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {
 	query_src, err := os.read_entire_file_from_path(query_path, context.allocator)
 	if err != nil {
 		log.warnf("highlight: cannot load query %s", query_path)
-		ts_parser_delete(parser)
+		gc.query_failed = true
 		return nil
 	}
 	query_str := string(query_src)
@@ -82,7 +119,7 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {
 	err_offset: u32
 	err_type: TSQueryError
 	query := ts_query_new(
-		language,
+		gc.language,
 		query_c,
 		u32(len(query_src)),
 		&err_offset,
@@ -111,6 +148,9 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {
 		}
 		log.errorf("highlight: %s query failed: %s", lang, cause)
 
+		_, is_builtin := builtin_language(lang)
+		if !is_builtin {
+			so_path := fmt.tprintf("%s/%s.so", GRAPHS_PATH, lang)
 			gram_v := helix_version_from_path(so_path)
 			query_v := helix_version_from_path(query_path)
 			gram_note := "(version unknown)"
@@ -122,16 +162,13 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {
 			if gram_v != "" && query_v != "" && gram_v != query_v {
 				log.errorf("  >> helix VERSION MISMATCH: grammar %s vs query %s", gram_v, query_v)
 			}
+		}
 
-		ts_parser_delete(parser)
+		gc.query_failed = true
 		return nil
 	}
 
-	gc := new(Grammar_Cache)
-	gc.language = language
-	gc.parser = parser
 	gc.query = query
-	grammar_cache[lang] = gc
 	return gc
 }
 

minify.odin

@@ -0,0 +1,253 @@
+package main
+
+import "core:log"
+import "core:strings"
+
+PRESERVE_TAGS :: [?]string{"pre", "code", "textarea"}
+
+Range :: struct {
+	start: u32,
+	end:   u32,
+}
+
+minify_html :: proc(source: string) -> string {
+	gc := ensure_parser("html")
+	if gc == nil {
+		return source
+	}
+
+	source_c := strings.clone_to_cstring(source)
+	defer delete(source_c)
+
+	tree := ts_parser_parse_string(gc.parser, nil, source_c, u32(len(source)))
+	if tree == nil {
+		return source
+	}
+	defer ts_tree_delete(tree)
+
+	root := ts_tree_root_node(tree)
+
+	if ts_node_has_error(root) {
+		log.warnf("minify: HTML parse errors, skipping minification")
+		return source
+	}
+
+	comments: [dynamic]Range
+	defer delete(comments)
+	preserves: [dynamic]Range
+	defer delete(preserves)
+
+	collect_html_ranges(root, source, &comments, &preserves)
+
+	sb := strings.builder_make()
+
+	ci := 0
+	pi := 0
+	i := 0
+	last_written: u8 = 0
+
+	for i < len(source) {
+		if pi < len(preserves) && u32(i) >= preserves[pi].start {
+			p := preserves[pi]
+			segment := source[i:p.end]
+			strings.write_string(&sb, segment)
+			if len(segment) > 0 {
+				last_written = segment[len(segment)-1]
+			}
+			i = int(p.end)
+			pi += 1
+			continue
+		}
+
+		if ci < len(comments) && u32(i) >= comments[ci].start {
+			i = int(comments[ci].end)
+			ci += 1
+			continue
+		}
+
+		c := source[i]
+		if c == ' ' || c == '\t' || c == '\n' || c == '\r' {
+			j := i + 1
+			for j < len(source) {
+				c2 := source[j]
+				if c2 != ' ' && c2 != '\t' && c2 != '\n' && c2 != '\r' {
+					break
+				}
+				j += 1
+			}
+			next: u8 = 0
+			if j < len(source) {
+				next = source[j]
+			}
+			if last_written != '>' || next != '<' {
+				strings.write_byte(&sb, ' ')
+				last_written = ' '
+			}
+			i = j
+		} else {
+			strings.write_byte(&sb, c)
+			last_written = c
+			i += 1
+		}
+	}
+
+	return strings.to_string(sb)
+}
+
+collect_html_ranges :: proc(
+	node: TSNode,
+	source: string,
+	comments: ^[dynamic]Range,
+	preserves: ^[dynamic]Range,
+) {
+	child_count := ts_node_named_child_count(node)
+	for i in 0..<child_count {
+		child := ts_node_named_child(node, u32(i))
+		type_str := string(ts_node_type(child))
+
+		if type_str == "comment" {
+			append(comments, Range{
+				start = ts_node_start_byte(child),
+				end = ts_node_end_byte(child),
+			})
+		} else if type_str == "script_element" || type_str == "style_element" {
+			append(preserves, Range{
+				start = ts_node_start_byte(child),
+				end = ts_node_end_byte(child),
+			})
+		} else if type_str == "element" {
+			tag := html_tag_name(child, source)
+			if is_preserve_tag(tag) {
+				append(preserves, Range{
+					start = ts_node_start_byte(child),
+					end = ts_node_end_byte(child),
+				})
+			} else {
+				collect_html_ranges(child, source, comments, preserves)
+			}
+		} else {
+			collect_html_ranges(child, source, comments, preserves)
+		}
+	}
+}
+
+html_tag_name :: proc(element: TSNode, source: string) -> string {
+	child_count := ts_node_named_child_count(element)
+	for i in 0..<child_count {
+		child := ts_node_named_child(element, u32(i))
+		if string(ts_node_type(child)) == "start_tag" {
+			tag_child_count := ts_node_named_child_count(child)
+			for j in 0..<tag_child_count {
+				tag_child := ts_node_named_child(child, u32(j))
+				if string(ts_node_type(tag_child)) == "tag_name" {
+					start := ts_node_start_byte(tag_child)
+					end := ts_node_end_byte(tag_child)
+					return source[start:end]
+				}
+			}
+		}
+	}
+	return ""
+}
+
+is_preserve_tag :: proc(tag: string) -> bool {
+	for t in PRESERVE_TAGS {
+		if tag == t do return true
+	}
+	return false
+}
+
+CSS_DELIMS :: [?]u8{'{', '}', ':', ';', ','}
+
+is_css_delim :: proc(c: u8) -> bool {
+	for d in CSS_DELIMS {
+		if c == d do return true
+	}
+	return false
+}
+
+minify_css :: proc(source: string) -> string {
+	gc := ensure_parser("css")
+	if gc == nil {
+		return source
+	}
+
+	source_c := strings.clone_to_cstring(source)
+	defer delete(source_c)
+
+	tree := ts_parser_parse_string(gc.parser, nil, source_c, u32(len(source)))
+	if tree == nil {
+		return source
+	}
+	defer ts_tree_delete(tree)
+
+	root := ts_tree_root_node(tree)
+
+	if ts_node_has_error(root) {
+		log.warnf("minify: CSS parse errors, skipping minification")
+		return source
+	}
+
+	comments: [dynamic]Range
+	defer delete(comments)
+
+	collect_css_comments(root, &comments)
+
+	sb := strings.builder_make()
+
+	ci := 0
+	i := 0
+	last_written: u8 = 0
+
+	for i < len(source) {
+		if ci < len(comments) && u32(i) >= comments[ci].start {
+			i = int(comments[ci].end)
+			ci += 1
+			continue
+		}
+
+		c := source[i]
+		if c == ' ' || c == '\t' || c == '\n' || c == '\r' {
+			j := i + 1
+			for j < len(source) {
+				c2 := source[j]
+				if c2 != ' ' && c2 != '\t' && c2 != '\n' && c2 != '\r' {
+					break
+				}
+				j += 1
+			}
+
+			next: u8 = 0
+			if j < len(source) {
+				next = source[j]
+			}
+
+			if !is_css_delim(last_written) && !is_css_delim(next) {
+				strings.write_byte(&sb, ' ')
+				last_written = ' '
+			}
+			i = j
+		} else {
+			strings.write_byte(&sb, c)
+			last_written = c
+			i += 1
+		}
+	}
+
+	return strings.to_string(sb)
+}
+
+collect_css_comments :: proc(node: TSNode, comments: ^[dynamic]Range) {
+	child_count := ts_node_named_child_count(node)
+	for i in 0..<child_count {
+		child := ts_node_named_child(node, u32(i))
+		if string(ts_node_type(child)) == "comment" {
+			append(comments, Range{
+				start = ts_node_start_byte(child),
+				end = ts_node_end_byte(child),
+			})
+		} else {
+			collect_css_comments(child, comments)
+		}
+	}
+}

render.odin

@@ -104,17 +104,26 @@ render_site :: proc(pages: []Page, config: Site) {
 			continue
 		}
 		html := render_page_html(page, config)
+		if .Minify in config.features {
+			html = minify_html(html)
+		}
 		write_page(config.output_dir, page.permalink, html)
 	}
 
 	// Render home page
 	if has_home {
 		home_html := render_home_html(home, pages, config)
+		if .Minify in config.features {
+			home_html = minify_html(home_html)
+		}
 		write_file(fmt.tprintf("%s/index.html", config.output_dir), home_html)
 	}
 
 	// Render posts list page
 	posts_html := render_posts_html(pages, config)
+	if .Minify in config.features {
+		posts_html = minify_html(posts_html)
+	}
 	write_page(config.output_dir, "/posts/", posts_html)
 
 	// Generate RSS feed
@@ -128,6 +137,9 @@ render_site :: proc(pages: []Page, config: Site) {
 	// Copy static directory (favicon, CSS, images, etc.)
 	copy_static_dir(config.static_dir, config.output_dir)
 
+	// Copy and maybe minify assets directory
+	copy_assets_dir(config.assets_dir, config.output_dir, config.features)
+
 	// Generate robots.txt
 	robots := fmt.aprintf("User-agent: *\nAllow: /\nSitemap: %s/sitemap.xml\n", config.base_url)
 	write_file(fmt.tprintf("%s/robots.txt", config.output_dir), robots)

site.odin

@@ -18,6 +18,7 @@ Site :: struct {
 	config_path: string,
 	content_dir: string,
 	static_dir:  string,
+	assets_dir:  string,
 	output_dir:  string,
 	layouts_dir: string,
 	params:      json.Value,
@@ -27,6 +28,7 @@ Site :: struct {
 Feature :: enum {
 	Sections,
 	Drafts,
+	Minify,
 	Watch,
 }
 
@@ -37,6 +39,7 @@ Flags :: struct {
 	base_url:    string `args:"name=base-url"`,
 	content_dir: string `args:"name=content"`,
 	static_dir:  string `args:"name=static"`,
+	assets_dir:  string `args:"name=assets"`,
 	output_dir:  string `args:"name=output"`,
 	layouts_dir: string,
 	author:      string,
@@ -44,6 +47,7 @@ Flags :: struct {
 	sectionate:  bool `args:"name=sections"`,
 	drafts:      bool `args:"name=drafts"`,
 	watch:       bool,
+	minify:      bool `args:"name=minify"`,
 }
 
 init_site :: proc(site: ^Site, args: []string) {
@@ -86,6 +90,9 @@ init_site :: proc(site: ^Site, args: []string) {
 	if site.static_dir == "" {
 		site.static_dir = fmt.tprintf("%s/static", config_dir)
 	}
+	if site.assets_dir == "" {
+		site.assets_dir = fmt.tprintf("%s/assets", config_dir)
+	}
 	if site.output_dir == "" {
 		site.output_dir = fmt.tprintf("%s/public", config_dir)
 	}
@@ -129,6 +136,9 @@ merge_flags :: proc(config: ^Flags, flags: Flags) {
 	if flags.static_dir != "" {
 		config.static_dir = flags.static_dir
 	}
+	if flags.assets_dir != "" {
+		config.assets_dir = flags.assets_dir
+	}
 	if flags.output_dir != "" {
 		config.output_dir = flags.output_dir
 	}
@@ -141,6 +151,9 @@ merge_flags :: proc(config: ^Flags, flags: Flags) {
 	if flags.sectionate {
 		config.sectionate = true
 	}
+	if flags.minify {
+		config.minify = true
+	}
 	config.config_path = flags.config_path
 }
 
@@ -152,6 +165,7 @@ site_apply_flags :: proc(site: ^Site, flags: Flags) {
 	site.config_path = flags.config_path
 	site.content_dir = flags.content_dir
 	site.static_dir = flags.static_dir
+	site.assets_dir = flags.assets_dir
 	site.output_dir = flags.output_dir
 	site.layouts_dir = flags.layouts_dir
 	site.params = flags.params
@@ -159,6 +173,7 @@ site_apply_flags :: proc(site: ^Site, flags: Flags) {
 	if flags.sectionate {site.features += {.Sections}}
 	if flags.drafts {site.features += {.Drafts}}
 	if flags.watch {site.features += {.Watch}}
+	if flags.minify {site.features += {.Minify}}
 }
 
 site_allocator :: proc(site: ^Site) -> mem.Allocator {

tree_sitter.odin

@@ -49,6 +49,8 @@ RTLD_LAZY :: c.int(1)
 
 foreign import lib "system:tree-sitter"
 foreign import libdl "system:dl"
+foreign import html_grammar "system:tree-sitter-html"
+foreign import css_grammar "system:tree-sitter-css"
 
 foreign lib {
 	ts_parser_new :: proc() -> TSParser ---
@@ -74,7 +76,11 @@ foreign lib {
 	ts_node_is_error :: proc(self: TSNode) -> bool ---
 	ts_node_child_count :: proc(self: TSNode) -> u32 ---
 	ts_node_child :: proc(self: TSNode, child_index: u32) -> TSNode ---
+	ts_node_named_child_count :: proc(self: TSNode) -> u32 ---
+	ts_node_named_child :: proc(self: TSNode, child_index: u32) -> TSNode ---
 	ts_node_start_point :: proc(self: TSNode) -> TSPoint ---
+	ts_node_type :: proc(self: TSNode) -> cstring ---
+	ts_node_parent :: proc(self: TSNode) -> TSNode ---
 }
 
 foreign lib {
@@ -113,3 +119,11 @@ foreign libdl {
 	dlsym :: proc(handle: rawptr, symbol: cstring) -> rawptr ---
 	dlclose :: proc(handle: rawptr) -> c.int ---
 }
+
+foreign html_grammar {
+	tree_sitter_html :: proc() -> TSLanguage ---
+}
+
+foreign css_grammar {
+	tree_sitter_css :: proc() -> TSLanguage ---
+}