feat(tools): adaptive stdin/stdout I/O for the tool pipe

Infer per command whether a tool uses file paths or streams: if the command
omits {{in}}, thor feeds the source on the tool's stdin; if it omits {{out}},
thor captures stdout and writes it to the output file. All four
in/out x stdin/stdout combinations end with a real file at out_path.

This makes the docs' recommended stdin/stdout form actually work and removes
the silent "ships unprocessed CSS" failure mode. Adds tests for the streaming
forms and a TODO recording the future fingerprint/contents-Resource migration.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-09-08 13:58:50 -04:00
committed by Spencer Brower
co-authored by Claude Opus 4.8
parent 122627c51c
commit b4f5e06ec8
3 changed files with 116 additions and 8 deletions
+11
View File
@@ -20,6 +20,14 @@
- [ ] instead of `warnings: [dynamic]Error` we should use `warnings: [dynamic; 8]Error`
- when limit reached, the template fails and stops rendering.
- [ ] Brainstorm plugin/tool/(postccs/less/tailwind/sass) support.
- [ ] `fingerprint` pipe + contents-based composition. The `tool` pipe currently
carries a URL string and each stage publishes a file to the output dir, so
`| tool sass | fingerprint` would leave a dead unhashed intermediate. When
fingerprint lands, move the pipe value from a URL string to a contents/
Resource (bytes + logical name) where only the terminal stage publishes;
a Resource must stringify to its URL so existing bare-`tool` templates keep
working. stdin/stdout tools compose best there (bytes stay in memory), which
is why stdin/stdout is the documented preferred form over `{{in}}`/`{{out}}`.
- [ ] Reconsider `"menus": {}` opt-out: it currently discards all page frontmatter
menus (with only a warning). Decide whether empty config should instead just
suppress the auto-generated menu while still honoring frontmatter.
@@ -33,6 +41,9 @@
- [ ] warn if stylesheet url starts with /assets (and is therefore invalid).
- [ ] warn when heading levels are skipped in a toc.
- [ ] extract `[TOC]` from `.md` body (or at least from the start.)
- [ ] add `--production` flag
- set `minify` to true
- warn when `base_url` unset
## General
+51 -8
View File
@@ -14,9 +14,12 @@ import "core:sync"
// {{out}} absolute path the tool should write its result to
// {{#minify}} section, non-empty when the site is building with minify on
//
// The tool does its own file I/O through {{in}}/{{out}}; neither mustache nor
// the host shuffles bytes. The pipe takes an asset path (e.g. "/css/main.css"),
// runs the tool, and returns the root-relative URL of the written output.
// {{in}}/{{out}} are optional per tool. If a command omits {{in}}, thor feeds
// the source file on the tool's stdin; if it omits {{out}}, thor captures the
// tool's stdout. Stdin/stdout is the preferred form (it will compose best with
// the future `fingerprint` pipe); {{in}}/{{out}} is the fallback for tools that
// can't stream. The pipe takes an asset path (e.g. "/css/main.css"), runs the
// tool, and returns the root-relative URL of the written output.
// Tool_Registry is the host-supplied configuration for the `tool` pipe. The
// host (thor) owns one per build and threads a pointer to it through render, so
@@ -127,7 +130,18 @@ run_tool :: proc(
return cached, nil
}
// Ensure the output's parent dir exists so the tool can write {{out}}.
// How the tool speaks I/O is inferred from whether the command references
// the paths thor substituted:
// {{in}} present -> the tool opens the source file itself
// {{in}} absent -> thor feeds the source file on stdin
// {{out}} present -> the tool writes the output file itself
// {{out}} absent -> thor captures stdout and writes it to out_path
// Either way the run ends with a real file at out_path.
uses_in := strings.contains(cmd_str, in_path)
uses_out := strings.contains(cmd_str, out_path)
// Ensure the output's parent dir exists (for the tool's {{out}} write or
// thor's stdout write).
if idx := strings.last_index(out_path, "/"); idx >= 0 {
if merr := os.make_directory_all(out_path[:idx]); merr != nil && merr != .Exist {
return "", Error_Body {
@@ -138,10 +152,28 @@ run_tool :: proc(
}
}
state, _, stderr, xerr := os.process_exec(
os.Process_Desc{command = argv},
context.temp_allocator,
)
desc := os.Process_Desc {
command = argv,
}
// Feed the source on stdin unless the tool takes it as a {{in}} path.
stdin_file: ^os.File
if !uses_in {
f, oerr := os.open(in_path)
if oerr != nil {
return "", Error_Body {
msg = fmt.tprintf("tool '%s': cannot open %s: %v", name, in_path, oerr),
pos = pos,
kind = .Data,
}
}
stdin_file = f
desc.stdin = f
}
defer if stdin_file != nil {
os.close(stdin_file)
}
state, stdout, stderr, xerr := os.process_exec(desc, context.temp_allocator)
if xerr != nil {
return "", Error_Body {
msg = fmt.tprintf("tool '%s': failed to run '%s': %v", name, argv[0], xerr),
@@ -163,6 +195,17 @@ run_tool :: proc(
}
}
// If the tool streamed to stdout, persist it as the output file.
if !uses_out {
if werr := os.write_entire_file(out_path, stdout); werr != nil {
return "", Error_Body {
msg = fmt.tprintf("tool '%s': cannot write %s: %v", name, out_path, werr),
pos = pos,
kind = .Data,
}
}
}
// Memoize for the rest of this build. `key` and `out_url` are already on the
// temp allocator — the same build-scoped lifetime as the manifest — so they
// are stored directly, no clone needed.
+54
View File
@@ -140,6 +140,60 @@ test_tool_runs :: proc(t: ^testing.T) {
}
}
// tool_io_case runs `cmd` over a source file containing `content` and returns
// the resulting output-file contents. Exercises the in/out ↔ stdin/stdout 2×2.
tool_io_case :: proc(name, cmd, content: string) -> (out: string, ok: bool) {
in_dir, out_dir := tool_test_dirs(name)
_ = os.write_entire_file_from_string(fmt.tprintf("%s/a.css", in_dir), content)
commands := make(map[string]string, context.temp_allocator)
commands[name] = cmd
reg := Tool_Registry {
commands = commands,
input_dir = in_dir,
output_dir = out_dir,
}
_, err := run_tool(&reg, name, "/a.css", 0)
if err != nil {
return "", false
}
data, rerr := os.read_entire_file_from_path(
fmt.tprintf("%s/a.css", out_dir),
context.temp_allocator,
)
return string(data), rerr == nil
}
@(test)
test_tool_stdin_stdout :: proc(t: ^testing.T) {
when ODIN_OS != .Windows {
// `cat` with no args: reads stdin (thor feeds the source), writes stdout
// (thor captures and persists it).
out, ok := tool_io_case("stream", "cat", "body{a:1}")
testing.expect(t, ok, "cat (stdin→stdout) should produce output")
testing.expect_value(t, out, "body{a:1}")
}
}
@(test)
test_tool_in_path_stdout :: proc(t: ^testing.T) {
when ODIN_OS != .Windows {
// `cat {{in}}`: tool opens the file by path, streams to stdout.
out, ok := tool_io_case("in_stream", "cat {{in}}", "body{b:2}")
testing.expect(t, ok, "cat {{in}} (file→stdout) should produce output")
testing.expect_value(t, out, "body{b:2}")
}
}
@(test)
test_tool_stdin_out_path :: proc(t: ^testing.T) {
when ODIN_OS != .Windows {
// `tee {{out}}`: thor feeds stdin, tool writes the file itself.
out, ok := tool_io_case("stream_out", "tee {{out}}", "body{c:3}")
testing.expect(t, ok, "tee {{out}} (stdin→file) should produce output")
testing.expect_value(t, out, "body{c:3}")
}
}
@(test)
test_tool_minify_section :: proc(t: ^testing.T) {
// The command template is itself mustache: {{#minify}} should expand only