feat: Added -minify flag that does simple html/css minification.

This commit is contained in:
Spencer Brower
2026-07-14 17:25:47 -04:00
parent 00b73c04ba
commit 9b732c4879
8 changed files with 506 additions and 76 deletions
+2
View File
@@ -14,6 +14,7 @@
- [ ] ensure sidenote numbers render in display order and not in declaration order.
- [x] Search up for `thor.json` files.
- [ ] OpenGraph meta tags — verify all fields match production site
- [ ] set opengraph tags / description automatically if unset. (Like hugo does)
- [ ] Table of contents support.
- [ ] Nav items should be active when the current page is selected.
- [ ] Theme selector for syntax highlighting.
@@ -32,6 +33,7 @@
- [ ] event based
- [ ] Free cmark HTML output (`body_html`) — cmark allocates via C malloc, not the arena, so it leaks per iteration in watch mode
- [ ] commands
- [ ] Add spall and find ways to reduce run time. (Getting close to 1/2sec here.)
- [ ] Review every file in thor
- [ ] Review alerts.odin
- [ ] Review content.odin
+58
View File
@@ -255,3 +255,61 @@ copy_static_recursive :: proc(current: string, rel_prefix: string, output_dir: s
}
}
// copy_assets_dir recursively copies files from assets_dir to output_dir.
// .css files are minified when .Minify is enabled; all other files are copied
// verbatim with a warning suggesting they belong in static/.
// Silently skips if assets_dir doesn't exist.
copy_assets_dir :: proc(assets_dir: string, output_dir: string, features: bit_set[Feature]) {
if !os.exists(assets_dir) {
return
}
copy_assets_recursive(assets_dir, "", output_dir, features)
}
copy_assets_recursive :: proc(
current: string,
rel_prefix: string,
output_dir: string,
features: bit_set[Feature],
) {
entries, err := os.read_all_directory_by_path(current, context.allocator)
if err != nil {
log.warnf("thor: cannot read %s: %v", current, err)
return
}
defer os.file_info_slice_delete(entries, context.allocator)
for entry in entries {
rel := rel_prefix == "" ? entry.name : fmt.tprintf("%s/%s", rel_prefix, entry.name)
switch entry.type {
case .Regular:
dest := fmt.tprintf("%s/%s", output_dir, rel)
if idx := strings.last_index(dest, "/"); idx >= 0 {
if err := os.make_directory_all(dest[:idx]); err != nil && err != .Exist {
log.warnf("thor: cannot create %s: %v", dest[:idx], err)
continue
}
}
if .Minify in features && strings.has_suffix(entry.name, ".css") {
data, read_err := os.read_entire_file_from_path(entry.fullpath, context.allocator)
if read_err != nil {
log.warnf("thor: cannot read %s: %v", entry.fullpath, read_err)
continue
}
minified := minify_css(string(data))
write_file(dest, minified)
} else {
if !strings.has_suffix(entry.name, ".css") {
log.warnf("thor: %s is not a CSS file; consider moving it to static/", entry.fullpath)
}
if err := os.copy_file(dest, entry.fullpath); err != nil {
log.warnf("thor: cannot copy %s: %v", entry.fullpath, err)
}
}
case .Directory:
copy_assets_recursive(entry.fullpath, rel, output_dir, features)
case .Undetermined, .Symlink, .Named_Pipe, .Socket, .Block_Device, .Character_Device:
}
}
}
+71 -32
View File
@@ -12,12 +12,13 @@
};
outputs =
inputs@{ self
, flake-parts
, nixpkgs
, nixpkgs-unstable
# , process-compose-flake
, treefmt-nix
inputs@{
self,
flake-parts,
nixpkgs,
nixpkgs-unstable,
# , process-compose-flake
treefmt-nix,
}:
flake-parts.lib.mkFlake { inherit inputs; } {
imports = [
@@ -27,7 +28,42 @@
systems = [ "x86_64-linux" ];
perSystem =
{ pkgs, system, inputs', ... }: {
{
pkgs,
system,
inputs',
...
}:
let
mkGrammarStaticLib = name: src: pkgs.stdenv.mkDerivation {
inherit name src;
dontConfigure = true;
buildPhase = ''
runHook preBuild
if [ -f src/scanner.cc ]; then
$CXX -fPIC -c src/scanner.cc -o scanner.o
elif [ -f src/scanner.c ]; then
$CC -fPIC -c src/scanner.c -o scanner.o
fi
$CC -fPIC -c src/parser.c -o parser.o
runHook postBuild
'';
installPhase = ''
runHook preInstall
mkdir -p $out/lib
ar rcs $out/lib/lib${name}.a *.o
runHook postInstall
'';
};
html-grammar = mkGrammarStaticLib "tree-sitter-html"
pkgs.tree-sitter-grammars.tree-sitter-html.src;
css-grammar = mkGrammarStaticLib "tree-sitter-css"
pkgs.tree-sitter-grammars.tree-sitter-css.src;
in
{
_module.args.pkgs = import nixpkgs {
inherit system;
config.allowUnfree = true;
@@ -48,22 +84,20 @@
".env.local"
];
# Format nix files
programs.nixpkgs-fmt.enable = true;
programs.deadnix.enable = true;
# Format js, json, and yaml files
programs.prettier.enable = true;
settings.formatter.prettier =
{
excludes = [
"public/**"
"resources/js/modernizr.js"
"storage/app/caniuse.json"
"*.md"
];
};
settings.formatter.prettier = {
excludes = [
"public/**"
"resources/js/modernizr.js"
"storage/app/caniuse.json"
"*.md"
];
};
};
#process-compose.default.settings.processes = { };
@@ -82,8 +116,12 @@
pkgs.git
pkgs.cmark
pkgs.tree-sitter
html-grammar
css-grammar
];
LIBRARY_PATH = "${html-grammar}/lib:${css-grammar}/lib";
doCheck = true;
checkPhase = ''
runHook preCheck
@@ -94,8 +132,6 @@
buildPhase = ''
runHook preBuild
odin build . -o:speed -out:${pname}-keep
echo "Listing filles..."
ls .
runHook postBuild
'';
@@ -106,20 +142,23 @@
'';
};
devShells.default = pkgs.mkShell
{
buildInputs = with pkgs; [
odin
ols
cmark
tree-sitter
devShells.default = pkgs.mkShell {
buildInputs = with pkgs; [
odin
ols
cmark
tree-sitter
# IDE
unstable.helix
typescript-language-server
vscode-langservers-extracted
];
};
# IDE
unstable.helix
typescript-language-server
vscode-langservers-extracted
];
shellHook = ''
export LIBRARY_PATH="${html-grammar}/lib:${css-grammar}/lib:$LIBRARY_PATH"
'';
};
};
};
}
+81 -44
View File
@@ -6,16 +6,29 @@ import "core:os"
import "core:strings"
Grammar_Cache :: struct {
language: TSLanguage,
parser: TSParser,
query: TSQuery,
language: TSLanguage,
parser: TSParser,
query: TSQuery,
query_failed: bool,
}
Get_Language_Proc :: #type proc() -> TSLanguage
grammar_cache: map[string]^Grammar_Cache
load_grammar :: proc(lang: string) -> ^Grammar_Cache {
builtin_language :: proc(lang: string) -> (language: TSLanguage, ok: bool) {
switch lang {
case "html":
language = tree_sitter_html()
ok = true
case "css":
language = tree_sitter_css()
ok = true
}
return
}
ensure_parser :: proc(lang: string) -> ^Grammar_Cache {
if grammar_cache == nil {
grammar_cache = make(map[string]^Grammar_Cache)
}
@@ -23,33 +36,38 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {
return cached
}
// Cache nil by default so failures aren't retried.
grammar_cache[lang] = nil
if GRAPHS_PATH == "" {
log.warnf("highlight: no grammars path set, skipping %s", lang)
return nil
}
language: TSLanguage
so_path := fmt.tprintf("%s/%s.so", GRAPHS_PATH, lang)
so_c := strings.clone_to_cstring(so_path)
defer delete(so_c)
handle := dlopen(so_c, RTLD_LAZY)
if handle == nil {
log.warnf("highlight: cannot load grammar %s (%s)", lang, so_path)
return nil
}
if builtin, ok := builtin_language(lang); ok {
language = builtin
} else {
if GRAPHS_PATH == "" {
log.warnf("highlight: no grammars path set, skipping %s", lang)
return nil
}
sym_name := fmt.tprintf("tree_sitter_%s", lang)
sym_c := strings.clone_to_cstring(sym_name)
defer delete(sym_c)
sym := dlsym(handle, sym_c)
if sym == nil {
log.errorf("highlight: cannot find symbol %s in %s", sym_name, so_path)
return nil
so_path := fmt.tprintf("%s/%s.so", GRAPHS_PATH, lang)
so_c := strings.clone_to_cstring(so_path)
defer delete(so_c)
handle := dlopen(so_c, RTLD_LAZY)
if handle == nil {
log.warnf("highlight: cannot load grammar %s (%s)", lang, so_path)
return nil
}
sym_name := fmt.tprintf("tree_sitter_%s", lang)
sym_c := strings.clone_to_cstring(sym_name)
defer delete(sym_c)
sym := dlsym(handle, sym_c)
if sym == nil {
log.errorf("highlight: cannot find symbol %s in %s", sym_name, so_path)
return nil
}
get_language := transmute(Get_Language_Proc)(sym)
language = get_language()
}
get_language := transmute(Get_Language_Proc)(sym)
language := get_language()
parser := ts_parser_new()
if parser == nil {
@@ -62,9 +80,28 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {
return nil
}
gc := new(Grammar_Cache)
gc.language = language
gc.parser = parser
grammar_cache[lang] = gc
return gc
}
load_grammar :: proc(lang: string) -> ^Grammar_Cache {
gc := ensure_parser(lang)
if gc == nil {
return nil
}
if gc.query != nil {
return gc
}
if gc.query_failed {
return nil
}
if QUERIES_PATH == "" {
log.warnf("highlight: no queries path set, skipping %s", lang)
ts_parser_delete(parser)
gc.query_failed = true
return nil
}
@@ -72,7 +109,7 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {
query_src, err := os.read_entire_file_from_path(query_path, context.allocator)
if err != nil {
log.warnf("highlight: cannot load query %s", query_path)
ts_parser_delete(parser)
gc.query_failed = true
return nil
}
query_str := string(query_src)
@@ -82,7 +119,7 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {
err_offset: u32
err_type: TSQueryError
query := ts_query_new(
language,
gc.language,
query_c,
u32(len(query_src)),
&err_offset,
@@ -111,27 +148,27 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache {
}
log.errorf("highlight: %s query failed: %s", lang, cause)
gram_v := helix_version_from_path(so_path)
query_v := helix_version_from_path(query_path)
gram_note := "(version unknown)"
if gram_v != "" do gram_note = fmt.tprintf("helix %s", gram_v)
query_note := "(version unknown)"
if query_v != "" do query_note = fmt.tprintf("helix %s", query_v)
log.errorf(" grammar: %s [%s]", so_path, gram_note)
log.errorf(" query: %s [%s]", query_path, query_note)
if gram_v != "" && query_v != "" && gram_v != query_v {
log.errorf(" >> helix VERSION MISMATCH: grammar %s vs query %s", gram_v, query_v)
_, is_builtin := builtin_language(lang)
if !is_builtin {
so_path := fmt.tprintf("%s/%s.so", GRAPHS_PATH, lang)
gram_v := helix_version_from_path(so_path)
query_v := helix_version_from_path(query_path)
gram_note := "(version unknown)"
if gram_v != "" do gram_note = fmt.tprintf("helix %s", gram_v)
query_note := "(version unknown)"
if query_v != "" do query_note = fmt.tprintf("helix %s", query_v)
log.errorf(" grammar: %s [%s]", so_path, gram_note)
log.errorf(" query: %s [%s]", query_path, query_note)
if gram_v != "" && query_v != "" && gram_v != query_v {
log.errorf(" >> helix VERSION MISMATCH: grammar %s vs query %s", gram_v, query_v)
}
}
ts_parser_delete(parser)
gc.query_failed = true
return nil
}
gc := new(Grammar_Cache)
gc.language = language
gc.parser = parser
gc.query = query
grammar_cache[lang] = gc
return gc
}
+253
View File
@@ -0,0 +1,253 @@
package main
import "core:log"
import "core:strings"
PRESERVE_TAGS :: [?]string{"pre", "code", "textarea"}
Range :: struct {
start: u32,
end: u32,
}
minify_html :: proc(source: string) -> string {
gc := ensure_parser("html")
if gc == nil {
return source
}
source_c := strings.clone_to_cstring(source)
defer delete(source_c)
tree := ts_parser_parse_string(gc.parser, nil, source_c, u32(len(source)))
if tree == nil {
return source
}
defer ts_tree_delete(tree)
root := ts_tree_root_node(tree)
if ts_node_has_error(root) {
log.warnf("minify: HTML parse errors, skipping minification")
return source
}
comments: [dynamic]Range
defer delete(comments)
preserves: [dynamic]Range
defer delete(preserves)
collect_html_ranges(root, source, &comments, &preserves)
sb := strings.builder_make()
ci := 0
pi := 0
i := 0
last_written: u8 = 0
for i < len(source) {
if pi < len(preserves) && u32(i) >= preserves[pi].start {
p := preserves[pi]
segment := source[i:p.end]
strings.write_string(&sb, segment)
if len(segment) > 0 {
last_written = segment[len(segment)-1]
}
i = int(p.end)
pi += 1
continue
}
if ci < len(comments) && u32(i) >= comments[ci].start {
i = int(comments[ci].end)
ci += 1
continue
}
c := source[i]
if c == ' ' || c == '\t' || c == '\n' || c == '\r' {
j := i + 1
for j < len(source) {
c2 := source[j]
if c2 != ' ' && c2 != '\t' && c2 != '\n' && c2 != '\r' {
break
}
j += 1
}
next: u8 = 0
if j < len(source) {
next = source[j]
}
if last_written != '>' || next != '<' {
strings.write_byte(&sb, ' ')
last_written = ' '
}
i = j
} else {
strings.write_byte(&sb, c)
last_written = c
i += 1
}
}
return strings.to_string(sb)
}
collect_html_ranges :: proc(
node: TSNode,
source: string,
comments: ^[dynamic]Range,
preserves: ^[dynamic]Range,
) {
child_count := ts_node_named_child_count(node)
for i in 0..<child_count {
child := ts_node_named_child(node, u32(i))
type_str := string(ts_node_type(child))
if type_str == "comment" {
append(comments, Range{
start = ts_node_start_byte(child),
end = ts_node_end_byte(child),
})
} else if type_str == "script_element" || type_str == "style_element" {
append(preserves, Range{
start = ts_node_start_byte(child),
end = ts_node_end_byte(child),
})
} else if type_str == "element" {
tag := html_tag_name(child, source)
if is_preserve_tag(tag) {
append(preserves, Range{
start = ts_node_start_byte(child),
end = ts_node_end_byte(child),
})
} else {
collect_html_ranges(child, source, comments, preserves)
}
} else {
collect_html_ranges(child, source, comments, preserves)
}
}
}
html_tag_name :: proc(element: TSNode, source: string) -> string {
child_count := ts_node_named_child_count(element)
for i in 0..<child_count {
child := ts_node_named_child(element, u32(i))
if string(ts_node_type(child)) == "start_tag" {
tag_child_count := ts_node_named_child_count(child)
for j in 0..<tag_child_count {
tag_child := ts_node_named_child(child, u32(j))
if string(ts_node_type(tag_child)) == "tag_name" {
start := ts_node_start_byte(tag_child)
end := ts_node_end_byte(tag_child)
return source[start:end]
}
}
}
}
return ""
}
is_preserve_tag :: proc(tag: string) -> bool {
for t in PRESERVE_TAGS {
if tag == t do return true
}
return false
}
CSS_DELIMS :: [?]u8{'{', '}', ':', ';', ','}
is_css_delim :: proc(c: u8) -> bool {
for d in CSS_DELIMS {
if c == d do return true
}
return false
}
minify_css :: proc(source: string) -> string {
gc := ensure_parser("css")
if gc == nil {
return source
}
source_c := strings.clone_to_cstring(source)
defer delete(source_c)
tree := ts_parser_parse_string(gc.parser, nil, source_c, u32(len(source)))
if tree == nil {
return source
}
defer ts_tree_delete(tree)
root := ts_tree_root_node(tree)
if ts_node_has_error(root) {
log.warnf("minify: CSS parse errors, skipping minification")
return source
}
comments: [dynamic]Range
defer delete(comments)
collect_css_comments(root, &comments)
sb := strings.builder_make()
ci := 0
i := 0
last_written: u8 = 0
for i < len(source) {
if ci < len(comments) && u32(i) >= comments[ci].start {
i = int(comments[ci].end)
ci += 1
continue
}
c := source[i]
if c == ' ' || c == '\t' || c == '\n' || c == '\r' {
j := i + 1
for j < len(source) {
c2 := source[j]
if c2 != ' ' && c2 != '\t' && c2 != '\n' && c2 != '\r' {
break
}
j += 1
}
next: u8 = 0
if j < len(source) {
next = source[j]
}
if !is_css_delim(last_written) && !is_css_delim(next) {
strings.write_byte(&sb, ' ')
last_written = ' '
}
i = j
} else {
strings.write_byte(&sb, c)
last_written = c
i += 1
}
}
return strings.to_string(sb)
}
collect_css_comments :: proc(node: TSNode, comments: ^[dynamic]Range) {
child_count := ts_node_named_child_count(node)
for i in 0..<child_count {
child := ts_node_named_child(node, u32(i))
if string(ts_node_type(child)) == "comment" {
append(comments, Range{
start = ts_node_start_byte(child),
end = ts_node_end_byte(child),
})
} else {
collect_css_comments(child, comments)
}
}
}
+12
View File
@@ -104,17 +104,26 @@ render_site :: proc(pages: []Page, config: Site) {
continue
}
html := render_page_html(page, config)
if .Minify in config.features {
html = minify_html(html)
}
write_page(config.output_dir, page.permalink, html)
}
// Render home page
if has_home {
home_html := render_home_html(home, pages, config)
if .Minify in config.features {
home_html = minify_html(home_html)
}
write_file(fmt.tprintf("%s/index.html", config.output_dir), home_html)
}
// Render posts list page
posts_html := render_posts_html(pages, config)
if .Minify in config.features {
posts_html = minify_html(posts_html)
}
write_page(config.output_dir, "/posts/", posts_html)
// Generate RSS feed
@@ -128,6 +137,9 @@ render_site :: proc(pages: []Page, config: Site) {
// Copy static directory (favicon, CSS, images, etc.)
copy_static_dir(config.static_dir, config.output_dir)
// Copy and maybe minify assets directory
copy_assets_dir(config.assets_dir, config.output_dir, config.features)
// Generate robots.txt
robots := fmt.aprintf("User-agent: *\nAllow: /\nSitemap: %s/sitemap.xml\n", config.base_url)
write_file(fmt.tprintf("%s/robots.txt", config.output_dir), robots)
+15
View File
@@ -18,6 +18,7 @@ Site :: struct {
config_path: string,
content_dir: string,
static_dir: string,
assets_dir: string,
output_dir: string,
layouts_dir: string,
params: json.Value,
@@ -27,6 +28,7 @@ Site :: struct {
Feature :: enum {
Sections,
Drafts,
Minify,
Watch,
}
@@ -37,6 +39,7 @@ Flags :: struct {
base_url: string `args:"name=base-url"`,
content_dir: string `args:"name=content"`,
static_dir: string `args:"name=static"`,
assets_dir: string `args:"name=assets"`,
output_dir: string `args:"name=output"`,
layouts_dir: string,
author: string,
@@ -44,6 +47,7 @@ Flags :: struct {
sectionate: bool `args:"name=sections"`,
drafts: bool `args:"name=drafts"`,
watch: bool,
minify: bool `args:"name=minify"`,
}
init_site :: proc(site: ^Site, args: []string) {
@@ -86,6 +90,9 @@ init_site :: proc(site: ^Site, args: []string) {
if site.static_dir == "" {
site.static_dir = fmt.tprintf("%s/static", config_dir)
}
if site.assets_dir == "" {
site.assets_dir = fmt.tprintf("%s/assets", config_dir)
}
if site.output_dir == "" {
site.output_dir = fmt.tprintf("%s/public", config_dir)
}
@@ -129,6 +136,9 @@ merge_flags :: proc(config: ^Flags, flags: Flags) {
if flags.static_dir != "" {
config.static_dir = flags.static_dir
}
if flags.assets_dir != "" {
config.assets_dir = flags.assets_dir
}
if flags.output_dir != "" {
config.output_dir = flags.output_dir
}
@@ -141,6 +151,9 @@ merge_flags :: proc(config: ^Flags, flags: Flags) {
if flags.sectionate {
config.sectionate = true
}
if flags.minify {
config.minify = true
}
config.config_path = flags.config_path
}
@@ -152,6 +165,7 @@ site_apply_flags :: proc(site: ^Site, flags: Flags) {
site.config_path = flags.config_path
site.content_dir = flags.content_dir
site.static_dir = flags.static_dir
site.assets_dir = flags.assets_dir
site.output_dir = flags.output_dir
site.layouts_dir = flags.layouts_dir
site.params = flags.params
@@ -159,6 +173,7 @@ site_apply_flags :: proc(site: ^Site, flags: Flags) {
if flags.sectionate {site.features += {.Sections}}
if flags.drafts {site.features += {.Drafts}}
if flags.watch {site.features += {.Watch}}
if flags.minify {site.features += {.Minify}}
}
site_allocator :: proc(site: ^Site) -> mem.Allocator {
+14
View File
@@ -49,6 +49,8 @@ RTLD_LAZY :: c.int(1)
foreign import lib "system:tree-sitter"
foreign import libdl "system:dl"
foreign import html_grammar "system:tree-sitter-html"
foreign import css_grammar "system:tree-sitter-css"
foreign lib {
ts_parser_new :: proc() -> TSParser ---
@@ -74,7 +76,11 @@ foreign lib {
ts_node_is_error :: proc(self: TSNode) -> bool ---
ts_node_child_count :: proc(self: TSNode) -> u32 ---
ts_node_child :: proc(self: TSNode, child_index: u32) -> TSNode ---
ts_node_named_child_count :: proc(self: TSNode) -> u32 ---
ts_node_named_child :: proc(self: TSNode, child_index: u32) -> TSNode ---
ts_node_start_point :: proc(self: TSNode) -> TSPoint ---
ts_node_type :: proc(self: TSNode) -> cstring ---
ts_node_parent :: proc(self: TSNode) -> TSNode ---
}
foreign lib {
@@ -113,3 +119,11 @@ foreign libdl {
dlsym :: proc(handle: rawptr, symbol: cstring) -> rawptr ---
dlclose :: proc(handle: rawptr) -> c.int ---
}
foreign html_grammar {
tree_sitter_html :: proc() -> TSLanguage ---
}
foreign css_grammar {
tree_sitter_css :: proc() -> TSLanguage ---
}