spencer/thor
1
0
Fork 0
mirror of https://github.com/sbrow/thor.git synced 2026-10-25 18:21:54 -04:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: spencer/thor:main
Branches Tags
spencer/thor:main
..
compare: spencer/thor:5088f31a58fea2a7deeb13f66c2c6b9aa6d245d0
Branches Tags
spencer/thor:main
1 Commits
main .. 5088f31a58
Author SHA1 Message Date
Spencer Brower 5088f31a58 feat: Added a default stylesheet for extensions. 2026-08-04 18:18:08 -04:00
99 changed files with 1403 additions and 5743 deletions
Expand all files Collapse all files
.github/workflows/pages.yml
-78
View File
@@ -1,78 +0,0 @@
# Custom Pages build: compiles Thor from source via the Nix flake, uses it to
# generate thor/site/, and publishes the result. Replaces the default Jekyll build.
name: Deploy site to GitHub Pages
on:
push:
branches: [main]
# Only rebuild when something that affects the output changes.
paths:
- "site/**"
- "**/*.odin"
- "defaults/**"
- "flake.nix"
- "flake.lock"
- ".github/workflows/pages.yml"
workflow_dispatch:
permissions:
contents: read
pages: write
id-token: write
# Allow one concurrent deployment; don't cancel an in-progress publish.
concurrency:
group: pages
cancel-in-progress: false
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Nix
uses: cachix/install-nix-action@v30
with:
extra_nix_config: |
experimental-features = nix-command flakes
# Free cache: snapshots /nix/store into GitHub's Actions cache. The key is
# hashed over the inputs that change Thor's output, so a site-content-only
# change is a full hit (Thor not recompiled), while an Odin source change
# restores the nearest store and rebuilds just the delta.
- name: Cache /nix/store
uses: nix-community/cache-nix-action@v6
with:
primary-key: nix-${{ runner.os }}-${{ hashFiles('flake.lock', '**/*.nix', '**/*.odin') }}
restore-prefixes-first-match: nix-${{ runner.os }}-
gc-max-store-size-linux: 2G
purge: true
purge-prefixes: nix-${{ runner.os }}-
purge-created: 604800 # drop entries older than 7 days
# First run compiles Thor + the tree-sitter grammar libs; later runs
# restore them from the cache above and skip the compile.
- name: Build Thor
run: nix build .#default --print-build-logs
- name: Generate site
working-directory: site
run: ../result/bin/thor -base-url=https://oneidiotdeveloper.com/thor -minify
- name: Upload Pages artifact
uses: actions/upload-pages-artifact@v3
with:
path: site/public
deploy:
needs: build
runs-on: ubuntu-latest
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4
AGENTS.md
+2 -4
View File
@@ -32,7 +32,6 @@ thor/
├── treesitter/ # FFI types + grammar management (standalone package)
├── markdown/ # Content transformation pipeline (imports ../treesitter)
├── mustache/ # Template engine with lambdas + pipe filters + diagnostics
├── diagnostics/ # Shared diagnostic utilities (suggest_correction)
├── content.odin # Page struct, Pending_File, scan_content_files, collect_languages, load_page
├── render.odin # Template rendering, Template_Context, sort_pages, RSS, sitemap
├── menus.odin # Menu_Entry, DEFAULT_WEIGHT, build_menus, collect_auto_menus, merge_page_menus, parse_page_menus, parse_config_menus
@@ -73,7 +72,7 @@ thor/
| Package | Files | Responsibility |
|---|---|---|
| `treesitter/` | `treesitter.odin` | FFI types (`Parser`, `Node`, `Query`, etc.), `@(link_prefix="ts_")` foreign bindings, grammar management (`Grammar_Store` with persistent allocator, `load_language`/`compile_query` building blocks, `ensure_parser`/`load_grammar` lazy loading, `preload_grammar`/`preload_grammars` for parallel loading with `sync.Mutex` cache protection), statically-linked HTML/CSS grammars |
| `markdown/` | `markdown.odin` | `Extension` enum, `DEFAULT_EXTENSIONS`, `EXTENSION_NAMES`, `process(body, ext, file_path, allocator)` — full pipeline (clones cmark output, frees original), `parse_extension_list` (with "did you mean?" hints via `diagnostics.suggest_correction`), `apply_extension_config` (same), `resolve_extension_conflicts` (Footnotes wins over Sidenotes) |
| `markdown/` | `markdown.odin` | `Extension` enum, `DEFAULT_EXTENSIONS`, `process(body, ext, file_path, allocator)` — full pipeline (clones cmark output, frees original), `parse_extension_list`, `apply_extension_config` |
| | `footnotes.odin` | `strip_definitions` (pre-cmark, shared by `.Sidenotes` + `.Footnotes`), `inject_notes` (post-cmark sidenote rendering), `inject_footnotes` (post-cmark standard footnote rendering — numbered `<sup>` links + `<section class="footnotes"><ol>` at bottom). `.Sidenotes` and `.Footnotes` are mutually exclusive; `resolve_extension_conflicts` in `markdown.odin` picks `.Footnotes` if both are set. |
| | `alerts.odin` | `inject_alerts` — GitHub alert blocks (`> [!NOTE]`) → styled blockquotes with semantic class names (`alert-note` etc.) |
| | `emoji.odin` | `expand_emoji` — `:shortcode:` → unicode emoji |
@@ -83,7 +82,6 @@ thor/
| | `deflists.odin` | `convert_deflists` — pre-cmark pass. Scans for definition list patterns (`term\n\n: definition`) and converts to `<dl><dt><dd>` HTML blocks. Terms and definitions rendered through cmark individually for inline markdown. Consecutive pairs grouped into single `<dl>`. |
| | `toc.odin` | `generate_toc(html, allocator)` — page-level feature (not a pipeline extension). Scans `<h1>`-`<h6>` for IDs (after `inject_heading_ids`), builds nested `<ul>` with `<a href="#id">` links. Called from `load_page` when frontmatter `"toc": true`. Depends on `.HeadingIDs` being enabled. |
| `mustache/` | See [Mustache engine](#mustache-engine) below | Template engine |
| `diagnostics/` | `diagnostics.odin` | Shared diagnostic utilities: `suggest_correction` (Levenshtein "did you mean?"), `format_error` (rust-style multi-line error formatter with `caret_start`/`caret_end` params), `line_col`, `line_text`, `count_lines`, `digit_count`, `display_width`, `should_colorize`, `write_gutter`. Used by `mustache` (tag-aware errors via `format_tag_error` wrapper), `markdown` (extension name suggestions), and `render.odin` (parse/render error display). |
| `bench/` | `bench.odin` + `templates/` | Standalone template rendering benchmark. Generates 500 posts + 100 comments, renders with indented partials + inheritance + pipes. `--dump <path>` for output validation, positional arg for iteration count (default 250). |
Icon SVGs live as HTML partials in `layouts/partials/icons/` (home, github, rss, chevron_up, star).
@@ -421,7 +419,7 @@ Spec-compliant implementation at `mustache/`. See `mustache/SPEC.md` for the imp
| `tokenizer.odin` | Tokenizer (template string → `[]Token`), standalone whitespace detection |
| `data.odin` | Reflection-based data model: `base_value` (peels union/any/nested-any layers), `lookup_in` (structs + maps, handles `Type_Info_Any` value kind in maps), `resolve_name`, `is_truthy`, `any_to_string`, `write_value`, `list_info`, `extract_list_element`, `collect_map_keys` |
| `pipes.odin` | Pipes extension: `Pipe_Op` enum (`.Format`, `.Group_By`), `pipe_op_from_string`/`pipe_op_candidates` (reflection-based enum name lookup), `Pipe_Filter` AST (with `op_pos` for diagnostics), `parse_pipeline` (tracks byte offsets via `strings.index`), `apply_pipeline`, `apply_filter` (exhaustive enum switch), `apply_group_by`, `apply_format`. Misspelled pipe op suggestions via `suggest_correction`. Stored on `Node.filters`; render-scoped results in temp allocator. |
| `diagnostic.odin` | Mustache-specific diagnostics: `context_extent` (scans `{{ }}` tags for caret underline), `format_tag_error` (wraps `diags.format_error` + `context_extent`), `format_render_error` (formats `Error` using `Template` source/path). Generic error formatting lives in `diagnostics/` package. |
| `diagnostic.odin` | Rust-style error formatter: `format_error` (multi-line context, ANSI colors via `core:terminal/ansi`, `colorize` param), `format_render_error` (formats `Error`), `line_col`, `line_text`, `context_extent`, `count_lines`, `digit_count`, `should_colorize`. |
| `suggest.odin` | Strict-warning helpers: `validate_key_path` (walks dotted path, crosses maps silently), `suggest_correction` (Levenshtein via `core:strings/levenshtein_distance`), `collect_struct_keys` (via reflection, recurses into `using`), `struct_has_field` (distinguishes missing field from nil value — needed for `Maybe(bool)`), `collect_partial_names`, `collect_block_names`. |
| `spec_test.odin` | JSON spec test runner — loads `spec/specs/*.json`, runs each test case. Uses `log.nil_logger()` to suppress expected warnings. |
| `pipes_test.odin` | Pipe filter tests (`group_by` + `format`) |
ASSET_PIPELINE_PLAN.md
-135
View File
@@ -1,135 +0,0 @@
# Asset Pipeline Plan
Design for letting thor users bring their own CSS toolchain (Sass, PostCSS,
Lightning CSS, Tailwind, …).
## Thesis
Let thor users bring their own CSS toolchain by **orchestrating the tool they
already installed** — never bundling or reimplementing one. This is expressed
through thor's existing mustache pipe grammar with two new ops: `tool` and
`fingerprint`.
The earlier research design (content-addressed registry + deferred token
resolution + a separate post-render path for "whole-output scanners") is
**discarded**. It was overbuilt. Every tool — Tailwind included — scans
*source*, which exists at render time, so every transform runs **eagerly,
synchronously, and memoized**. No placeholder tokens, no post-render
string-replace pass, no second integration surface.
## The two ops
```html
<link href="{{ "css/main.scss" | tool sass | fingerprint | rel_url }}">
```
```json5
// thor.json
{
"tools": {
"sass": "sass $in $out",
"postcss": "postcss $in -o $out",
"lightningcss": "lightningcss --minify $in -o $out",
"tailwind": "tailwindcss -i $in -o $out"
}
}
```
## Core model
The value flowing through the pipe is **bytes held by thor**, with a logical
name attached (`css/main.css`). Each stage transforms the bytes; the **final
stage publishes** one file to the output dir and returns its URL.
- A literal (`"css/main.scss"`) or context key names the **source** file on disk.
- `| tool <name>` runs a subprocess, replacing the carried bytes with its output.
- `| fingerprint` writes the content-addressed file and returns its hashed URL.
- The chain composes with the existing `rel_url` for base_url subpaths.
**Memoization:** results are cached per build, keyed by (op-chain, source). The
first page to reference an asset runs the tool; every other page hits the cache.
This is what stops N pages from shelling out N times.
## `| tool <name>`
Looks up `<name>` in the `thor.json` `tools` table and runs its command
template. `$in` and `$out` are **optional per-tool I/O adapters**:
| Template has | Behavior |
|---|---|
| `$in` | thor writes current bytes to a scratch path, substitutes it |
| no `$in` | thor pipes current bytes to the tool's **stdin** |
| `$out` | thor substitutes a scratch path, reads it back after |
| no `$out` | thor captures the tool's **stdout** |
Defaulting to stdin/stdout avoids scratch-file overhead when the tool supports
it. Tools that can't stream (Lightning CSS wants a path; Tailwind v3 needs an
input file) just get `$in`/`$out` in their template. Either way the bytes are
thor's, so chaining works regardless of each tool's preference:
`| tool sass | tool postcss` composes even if sass streams and postcss wants
files.
- **`$out` is always thor-controlled scratch** — the tool never picks the
published filename.
- **Extension derivation:** thor owns the output name; `main.scss` through
`tool sass` becomes `main.css`.
- **Errors:** thor gates on exit code and surfaces the tool's stderr through one
code path. If the tool is missing from PATH, fail with its stderr — never
bundle, never assume a runtime.
- **A bare `tool` (no `fingerprint`) still publishes** — to the logical unhashed
path (`output/css/main.css`), returning `/css/main.css`.
## `| fingerprint`
Content-addresses whatever bytes it currently holds. It has **two outputs**:
- **Side effect:** writes `css/main.<hash>.css` to the output dir.
- **Return value:** the hashed URL string `/css/main.<hash>.css` (feeds
`rel_url`).
Properties:
- **Works standalone**, no `tool` needed: `{{ "css/main.css" | fingerprint }}`
is plain cache-busting of a hand-written file — probably the most common use.
- **Position decides what's hashed** — it hashes current bytes, so it belongs
**last** in the chain. No terminal-only special-casing; the rule is just "put
it last."
- **Hash format:** `name.<hash>.ext` (CDN-friendly), 8 hex chars of SHA-256.
## Tailwind is not special
It scans your **source** (`layouts/`, `content/`, partials) — which exists at
render time — so it runs synchronously like sass. Its `thor.json` entry just
needs content globs pointing at the source tree (or v4 auto-detection). The
**one** asymmetry is watch invalidation: sass re-runs when `main.scss` changes;
Tailwind must re-run when *any* scanned source changes. That's a cache-key
breadth difference, not an architectural one. The cost of scanning source vs.
output is losing dynamically-assembled class tokens (`col-{{n}}`) — a Tailwind
anti-pattern whose sanctioned fix is a safelist, not warping the build.
## Interaction: `copy_assets_dir`
Today thor copies everything under `assets/` to the output dir. Any source
consumed by a pipe must be **excluded from that plain copy** — otherwise you'd
ship the raw `main.scss` (or a dead unhashed `main.css` alongside the hashed
one). The pipe owns the write for assets it touches.
## Implementation steps (separable)
1. **`| fingerprint`** — no subprocess, no config. Hash + write + return URL,
plus the `copy_assets_dir` exclusion. Smallest blast radius, independently
useful, ships first.
2. **`| tool <name>`** — `thor.json` tools table, the `$in`/`$out`/stdin/stdout
adapter, subprocess + exit-code/stderr handling, extension derivation,
memoization. Composes with step 1.
## Open questions for implementation
- **Watch invalidation:** simplest correct v1 recomputes the memo each build
(cheap for the common single-asset case). A cross-build skip keyed on source
content/mtime is a later optimization — and Tailwind's broad content-glob key
is where it matters most.
- **Source-map sidecars:** sass/lightning emit `.map` files next to `$out`;
disable in the template or ignore the sidecar. Minor.
- **Error policy default:** hard-fail the build vs. warn-and-skip on tool
failure/absence. Lean fail; possibly configurable later.
AUDITING_BINARY_SIZE.md
-96
View File
@@ -1,96 +0,0 @@
# Visualizing & Managing `thor` Binary Size
How to inspect what makes up the compiled `thor` binary, and how to keep that
inspection useful once we bundle layout/asset files into the binary.
## Quick inspection (tools already available)
The binary is a not-stripped ELF, so it ships symbols we can break down.
```bash
# Section totals (text / data / bss)
size ./thor
# Top symbols by size
nm --print-size --size-sort --radix=d ./thor \
| awk '{sz=$2+0; sub(/^[0-9a-fA-F]+ +[0-9]+ +[A-Za-z] +/,""); printf "%8d %s\n", sz, $0}' \
| sort -rn | head -25
# Aggregate symbol sizes by Odin module namespace (foo::bar -> "foo")
nm --print-size --size-sort --radix=d ./thor \
| awk '{sz=$2+0; rest=$0; sub(/^[0-9a-fA-F]+ +[0-9]+ +[A-Za-z] +/,"",rest);
ns=rest; if (rest ~ /::/){sub(/::.*/,"",ns)}
else if (rest ~ /^ts_/){ns="treesitter (ts_*)"} else {ns="<other/runtime>"};
s[ns]+=sz} END{for (k in s) printf "%9d %s\n", s[k], k}' \
| sort -rn | head -25
```
As of this writing the top contributors were: tree-sitter tables (~118 KB),
`main` (~96 KB), runtime/other (~93 KB), `mustache` (~67 KB), then `fmt` /
`encoding_json` from the stdlib.
## Better visualization: `bloaty`
`bloaty` (Google's binary-size profiler) is the right tool. Not installed, but
runnable via Nix:
```bash
bloaty ./thor # by section
bloaty ./thor -d symbols -n 30 # by symbol, hierarchical
bloaty ./thor -d sections,symbols # section then symbols within each
# via nix (no install):
nix run nixpkgs#bloaty -- ./thor -d symbols -n 30
```
For a stripped-size estimate:
```bash
readelf -S ./thor | grep -E 'debug|symtab|strtab' # strippable overhead
strip -s ./thor -o thor.stripped && size thor.stripped
```
Odin also supports smaller builds: `odin build . -o:size -no-bounds-check`.
## IMPORTANT: keeping bloaty useful once we bundle layouts into the binary
Today the VFS (`vfs.odin`) reads layout/asset files lazily from disk at runtime
(`VFS_Entry.fs_path` + `os.read_entire_file_from_path`); `VFS_Entry.data` is
never populated at build time. So embedded content is **not** in `./thor` yet.
When we switch to embedding via Odin `#load(...)`, the file bytes land in the
binary's `.rodata` section. Whether bloaty can show *per-file* sizes depends on
how we emit the embed:
- **Section total — always visible.** `.rodata` grows by ~the total bundled
size. `bloaty ./thor -d sections` (or `size`) shows the aggregate, and we can
diff before/after. This always works.
- **Per-file breakdown — only if each file is a NAMED symbol.**
- A generated `map[string]VFS_Entry` literal full of inline `#load(...)`
expressions makes each blob an **anonymous** `.rodata` constant. Bloaty
lumps them together — no per-file breakdown.
- Assigning each file to its own **package-level variable** makes each a named
global symbol, so `bloaty ./thor -d symbols` (and the `nm` recipe above)
show each file individually:
```odin
layout_base_html := #load("defaults/layouts/base.html")
layout_single_html := #load("defaults/layouts/single.html")
// ...then reference these in the VFS map instead of inlining #load.
```
### Decision to bake into the bundling codegen
When we build the codegen that replaces the runtime `mount_dir` disk walk, have
it emit **one named package-level variable per embedded file** (not an anonymous
map literal). That single choice is what makes bloaty / `nm` per-file analysis
possible — cheap to do up front, painful to retrofit.
Additional options that help regardless of symbolization:
1. Build with Odin `-debug` (DWARF) and use `bloaty ./thor -d compileunits` to
attribute embedded bytes to the generated source file.
2. Since the generator already knows every file and its size, emit a size
manifest at bundle time (`path -> bytes`, sorted / treemap). This is the most
reliable per-file view and can't drift from reality; bloaty then just
confirms the `.rodata` total matches.
CODE_REVIEW.md
-38
View File
@@ -1,38 +0,0 @@
## Code Review
A human should manually review every file in the project. AI cannot complete
these tasks.
- [ ] Review every file in thor
- [ ] Review assets.odin
- [ ] Review content.odin
- [ ] Review defaults.odin
- [ ] Review feed.odin
- [ ] Review frontmatter.odin
- [ ] Review main.odin
- [ ] Review minify.odin
- [ ] Review `markdown/`
- [x] Review alerts.odin
- [x] Review alerts_test.odin
- [x] Review emoji.odin
- [x] Review emoji_test.odin
- [ ] Review footnotes.odin
- [ ] Review footnotes_test.odin
- [ ] Review deflists.odin
- [ ] Review deflists_test.odin
- [ ] Review highlight.odin
- [ ] Review markdown.odin
- [x] Review sectionate.odin
- [x] Review sectionate_test.odin
- [ ] Review suggest.odin
- [ ] Review suggest_test.odin
- [ ] Review `diagnostics/`
- [ ] Review diagnostics.odin
- [ ] Review diagnostics_test.odin
- [x] Review opengraph.odin
- [ ] Review render.odin
- [x] Review site.odin
- [ ] Review treesitter/treesitter.odin
- [ ] Review vfs.odin
- [ ] Review procs
- [ ] markdown.transform_alert
DIAGNOSTIC_TESTS.yaml
-379
View File
@@ -1,379 +0,0 @@
# This file contains tests for all known mustache syntax errors and their
# desired error messages. Each one is converted into an Odin test.
name: Missing 2nd closing brace
enable: false # TODO: Re-enable when passing
should: error
input: |
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
expected: |
Missing closing brace
--> <test>:1:1
1 | <menu>
2 | <a class="goto-top opacity-0" href="#">{{> icons/chevron_up}</a>
| ^^ opened ^ never closed
3 | {{#params.social}}
4 | <a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
5 | {{/params.social}}
|
---
name: Missing 2nd opening brace
enable: false # TODO: Re-enable when passing
should: error
input: |
<menu>
<a class="goto-top opacity-0" href="#">{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
expected: |
Missing opening brace
--> <test>:1:1
1 | <menu>
2 | <a class="goto-top opacity-0" href="#">{> icons/chevron_up}}</a>
| ^ opened ^^ closed
3 | {{#params.social}}
4 | <a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
5 | {{/params.social}}
|
---
# TODO: make it work
name: Invalid timezone
enable: false
should: warn
input: |
{
"date": {
"timezone": "America/New_Yorkside"
}
}
expected: "Did you mean 'America/New_York'? pointing to line+col no in the config"
actual: |
[WARN ] --- [148:init_site()] unable to load timezone 'America/New_Yorkside'
---
name: Mismatched section tags
enable: false # TODO: Re-enable when passing
should: error
input: |
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.ocial}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{now | format "2006" }} {{params.author.name}}</p>
</footer>
expected: |
Mismatched section: 'ocial' opened, 'social' closed
--> <test>:4:5
|
4 | {{#params.ocial}}
| ^^^^^ opened — did you mean 'social'?
5 | <a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
6 | {{/params.social}}
| ^^^^^^ expected {{/params.ocial}}, got {{/params.social}}
|
---
name: Missing closing section
enable: false # TODO: Re-enable when passing
should: error
input: |
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
<!-- {{/params.social}} -->
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{now | format "2006" }} {{params.author.name}}</p>
</footer>
expected: |
Missing closing tag '{{/params.social}}'
--> <test>:4:5
|
4 | {{#params.social}}
| ^^^^^^^^^^^^^^^^^ opened
5 | <a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
6 | <!-- {{/params.social}} -->
| ^^^^^^^^^^^^^^^^^^ closing tag is commented out
|
---
name: empty tag
enable: true
should: error
input: |
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{}} {{params.author.name}}</p>
</footer>
expected: |
Empty tag
--> <test>:11:28
|
9 | href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
10 | </p>
11 | <p><small>&copy;</small> {{}} {{params.author.name}}</p>
| ^^^^ tag must contain a key
12 | </footer>
|
---
name: empty raw tag
enable: true
should: error
input: |
Hello, {{&}}
expected: |
Empty tag
--> <test>:1:8
|
1 | Hello, {{&}}
| ^^^^^ tag must contain a key
|
---
name: Missing pipe arguement
enable: true
should: error
input: |
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{now | group_by }} {{params.author.name}}</p>
</footer>
expected: |
group_by expects 1 argument, got 0
--> <test>:11:36
|
9 | href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
10 | </p>
11 | <p><small>&copy;</small> {{now | group_by }} {{params.author.name}}</p>
| ^^^^^^^^ missing argument
12 | </footer>
|
---
name: Double dot access
enable: false # TODO: Re-enable when passing
should: error
input: |
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{now | format "2006" }} {{params..name}}</p>
</footer>
expected: |
Invalid key path: consecutive dots
--> <test>:11:60
|
9 | href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
10 | </p>
11 | <p><small>&copy;</small> {{now | format "2006" }} {{params..name}}</p>
| ^^ missing path name
12 | </footer>
|
---
name: Too many opening braces
enable: false # TODO: Re-enable when passing
should: error
input: |
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{{{now | format "2006" }}} {{params.author.name}}</p>
</footer>
expected: |
Too many opening braces
--> <test>:11:28
|
9 | href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
10 | </p>
11 | <p><small>&copy;</small> {{{{now | format "2006" }}} {{params.author.name}}</p>
| ^^^^ expected {{{, got {{{{
12 | </footer>
|
---
name: Too many closing braces
enable: false # TODO: Re-enable when passing
should: error
input: |
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{now | format "2006" }}}} {{params.author.name}}</p>
</footer>
expected: |
Too many closing braces
--> <test>:11:50
|
9 | href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
10 | </p>
11 | <p><small>&copy;</small> {{now | format "2006" }}}} {{params.author.name}}</p>
| ^^^^ expected }}, got }}}
12 | </footer>
|
---
# TODO: Implement
name: Frontmatter missing closing quote on key
enable: false
should: warn
input: |
{
"title: Docs"
}
[TOC]
## Introduction
expected: 'TODO:'
actual: |
[ERROR] --- [42:parse_frontmatter()] failed to parse frontmatter JSON: Expected_Colon_After_Key
---
# TODO: Implement
name: Fronmatter missing opening quote on value
should: warn
enable: false
input: |
{
"title": Docs"
}
[TOC]
## Introduction
expected: 'detailed diagnostic pointing to the point of failure and suggesting the addition of a quote'
actual: Nothing. The page compiles with no warnings or errors
---
name: No error when missing index page
enable: false # TODO: Re-enable when passing
should: error
expected: 'Your site has no index page! Please create an index.md or index.html file in ./content/'
---
# FIXME: Doesn't work properly
name: format pipe with no date format set
enable: false # TODO: Re-enable when passing
should: warn
input: '{{ now | format }}'
expected: |
no date format configured — set date.format in thor.json or specify a format, e.g. {{key | format "2 Jan 2006"}}
--> <test>:1:10
|
1 | {{now | format}}
| ^^^^^^ defaulting to "2 Jan 2006"
---
# TODO: called First/last, but only tests first
name: first/last with zero argument
enable: true
should: error
input: |
<ul>{{#items | first 0}}<li>{{.}}</li>{{/items}}</ul>
expected: |
argument must not be zero
--> <test>:1:16
|
1 | <ul>{{#items | first 0}}<li>{{.}}</li>{{/items}}</ul>
| ^^^^^^^
|
---
# TODO: called First/last, but only tests first
name: first/last with negative argument
enable: true
should: warn
input: |
<ul>{{#items | first -3}}<li>{{.}}</li>{{/items}}</ul>
expected: |
first -3 is equivalent to last 3
--> <test>:1:16
|
1 | <ul>{{#items | first -3}}<li>{{.}}</li>{{/items}}</ul>
| ^^^^^^^^ consider using the clearer form
|
---
# TODO: called First/last, but only tests first
name: first/last on string with no argument
# enable: true
input: '{{word | first}}'
should: warn
expected: |
defaulting to n=1 for string
--> <test>:1:1
|
1 | {{word | first}}
| ^^^^^ consider specifying n explicitly
|
---
name: "first succeeds on params.author.name"
enable: true
input: '{{ params.author.name | first 3 }}'
should: ok
expected: "foo"
---
# TODO: called First/last, but only tests first
# TODO: Include | last 3 }} in the suggestion.
name: "first/last in partial"
enable: true
input: "{{> footer }}"
should: warn
expected: |
first -3 is equivalent to last 3
--> fixtures/layouts/partials/footer.html:11:78
|
9 | href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
10 | </p>
11 | <p><small>&copy;</small> {{! now | format "2006" }} {{params.author.name | first -3 }}</p>
| ^^^^^^^^ consider using the clearer form
12 | </footer>
|
---
name: "Pipes are ignored in comments"
enable: true
should: ok
input: 'Hello, {{! now | format "2006" }}'
expected: 'Hello, '
DIAGNOSTIC_TODOS.yaml
+231
View File
@@ -0,0 +1,231 @@
name: Missing 2nd closing brace
input: |
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
expected: Somthing that points to the opening and lack of close.
actual: |
[ERROR] --- [383:load_partials()] unexpected {{/params.social}}
--> /home/spencer/github.com/sbrow.github.io/layouts/partials/footer.html:6:5
|
4 | {{#params.social}}
5 | <a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
6 | {{/params.social}}
| ^^^^^^^^^^^^^^^^^^
7 | </menu>
8 | <p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
---
name: Missing 2nd opening brace
input: |
<menu>
<a class="goto-top opacity-0" href="#">{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
expected: Somthing that shows the whole block, and points out the missing opening brace
actual: Nothing. The page compiles with no warnings or errors
---
name: Invalid timezone
input: |
{
"date": {
"timezone": "America/New_Yorkside"
}
}
expected: "Did you mean 'America/New_York'? pointing to line+col no in the config"
actual: |
[WARN ] --- [148:init_site()] unable to load timezone 'America/New_Yorkside'
---
name: Mismatched section tags
input: |
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.ocial}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{now | format "2006" }} {{params.author.name}}</p>
</footer>
expected: Mostly the same, but with rust style markers at the opening and close
actual: |
[ERROR] --- [383:load_partials()] expected {{/params.ocial}}, got {{/params.social}}
--> /home/spencer/github.com/sbrow.github.io/layouts/partials/footer.html:6:5
|
4 | {{#params.ocial}}
5 | <a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
6 | {{/params.social}}
| ^^^^^^^^^^^^^^^^^^
7 | </menu>
8 | <p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
---
name: Missing closing section
input: |
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
<!-- {{/params.social}} -->
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{now | format "2006" }} {{params.author.name}}</p>
</footer>
expected: "Missing closing tag '{{/params.social}}' ...opened <here> ... expected close <here>"
actual: Nothing. The page compiles with no warnings or errors
---
name: empty tag
input: |
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{}} {{params.author.name}}</p>
</footer>
expected: "Found an empty tag at <line:col>"
actual: |
[WARN ] --- [920:warn_unknown_key()] unknown key ''
--> /home/spencer/github.com/sbrow.github.io/layouts/partials/footer.html:1:1
|
1 | <footer>
| ^
2 | <menu>
3 | <a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
|
---
name: Missing pipe arguement
input: |
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{now | group_by }} {{params.author.name}}</p>
</footer>
expected: "Missing filter argument <pointing to exact location>"
actual: |
[ERROR] --- [152:render_template()] group_by expects 1 argument, got 0
--> /home/spencer/github.com/sbrow.github.io/layouts/partials/footer.html:1:1
|
1 | <footer>
| ^
2 | <menu>
3 | <a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
|
---
name: Double dot access
input: |
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{now | format "2006" }} {{params..name}}</p>
</footer>
expected: Error "Invalid access key"
actual: Nothing. The page compiles with no warnings or errors
---
name: Too many opening braces
input: |
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{{{now | format "2006" }}} {{params.author.name}}</p>
</footer>
expected: "Too many opening braces <point to location>"
actual: |
[WARN ] --- [920:warn_unknown_key()] unknown key '{now'
--> /home/spencer/github.com/sbrow.github.io/layouts/partials/footer.html:1:1
|
1 | <footer>
| ^ did you mean 'now'?
2 | <menu>
3 | <a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
|
---
name: Too many closing braces
input: |
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{now | format "2006" }}}} {{params.author.name}}</p>
</footer>
expected: "Too many closing braces <Point to location>"
actual: Nothing. The page compiles with no warnings or errors
---
name: Frontmatter missing closing quote on key
input: |
{
"title: Docs"
}
[TOC]
## Introduction
expected: 'TODO:'
actual: |
[ERROR] --- [42:parse_frontmatter()] failed to parse frontmatter JSON: Expected_Colon_After_Key
---
name: Fronmatter missing opening quote on value
input: |
{
"title": Docs"
}
[TOC]
## Introduction
expected: 'detailed diagnostic pointing to the point of failure and suggesting the addition of a quote'
actual: Nothing. The page compiles with no warnings or errors
---
name: No error when missing index page.
expected: 'Your site has no index page! Please create an index.md or index.html file in ./content/'
actual: Nothing. The site compiles with no warnings or errors
---
name: format pipe with no date format set
expected: |
Show where the error was triggered. Should also show the format that will be used
actual: |
[ERROR] --- [333:apply_format()] format pipe used but no date format configured (set date.format in thor.json) Default will be used
GRAMMAR_CACHE_THREADING.md
-171
View File
@@ -1,171 +0,0 @@
# Grammar cache thread-safety & allocator lifetime
Status: **resolved.** All three issues below are fixed. The cache was redesigned
so the library caches only the immutable, shareable objects and callers own the
per-thread ones. The rest of this document is kept as the rationale.
## Resolution (what shipped)
The `Grammar_Cache` (which mixed shareable `language`/`query` with per-thread
`parser`/`cursor`) was split:
- **`Grammar`** holds only the immutable `language` + `query`. It lives in a
process-lifetime **`registry`** (`map[string]^Grammar`) guarded by one mutex.
- **`grammar(lang)`** is the thread-safe, self-initializing, lazy loader — fixes
**#1a** (all map access under the lock) and lets tests run with no setup.
- The registry allocator is pinned with `os.heap_allocator()` — fixes **#2**; the
cache no longer captures a transient `context.allocator`.
- Parsers/cursors are **caller-owned**: `open_parser(g)` mints one, the caller
`defer`s `parser_delete` / `query_cursor_delete`. No parser is shared, so
**#1b** is gone. `open_parser` is the seam where a future pool / thread-local
strategy plugs in without touching call sites.
- Deleted: `ensure_parser`, `load_grammar`, `preload_grammar`, `grammar_worker`,
`cache_mutex`; `preload_grammars` is now a serial registry warm-up. The minify
tests dropped their `parse_mu` serialization and preload scaffolding entirely.
The **later** decision — pool vs per-thread parser reuse for genuine parallel
parsing — is still open, but intentionally so: it only matters once we parse in
parallel, and it now changes only `open_parser`, not the rest of the library.
The original investigation follows.
## How we got here
While adding a feature to minify inline `<style>` blocks (route the CSS body
through `minify_css`), the new unit tests exercised `minify_html` / `minify_css`
for the first time. Those are the first tests to touch the tree-sitter parser
cache, and under the multi-threaded test runner (default 16 threads) they failed
**intermittently** with a rotating cast of symptoms:
- `panic: unable to insert into a map`
- tree-sitter internal aborts: `ts_subtree_retain: Assertion 'self.ptr->ref_count > 0' failed`, `_array__erase: Assertion 'index < *size' failed`
- `malloc(): unaligned tcache chunk detected`
- assertions passing on the wrong data (e.g. minified output missing `<style>`)
The flakiness comes from **three distinct problems**, two thread-safety and one
allocator-lifetime. They are orthogonal — fixing one does not fix the others.
## The three issues
### #1a — concurrent cache **writes** (data race)
`ensure_parser` (`treesitter.odin`) reads and writes `grammar_store.cache`
with **no lock**. `cache_mutex` exists but is only taken on the parallel preload
path (`grammar_worker`), never in the lazy `ensure_parser` / `load_grammar`
paths. Two threads first-touching `"html"`/`"css"` at once corrupt the map.
### #1b — concurrent parser **use** (data race)
There is exactly **one cached `Grammar_Cache` (and one `parser`) per language**.
When two threads both call `minify_html`, they fetch the *same* `gc.parser` and
call `parser_parse_string` on it simultaneously. A tree-sitter `TSParser`
mutates its internal subtree pool during a parse, so concurrent parses corrupt
it — that's the `ts_subtree_retain` / `_array__erase` aborts.
This is **not fixed by preloading/"loading html+css statically."** Loading gives
you *one shared* parser; the race is about *using* that one parser from two
threads. It is fixed only by serializing the parse or giving each thread its own
parser.
### #2 — allocator lifetime (not a race)
`init_persistent` captures `context.allocator` into the process-lifetime
`grammar_store`. In `main` that happens to be the heap (it runs before the site
arena is installed at `main.odin`), so production is fine. But whatever
allocator is live at that instant becomes the cache's owner forever:
- Under the **test runner**, `context.allocator` is a **per-test tracking
allocator** that is torn down when that test ends → the global cache is left
pointing at dead memory (→ `unable to insert into a map`, garbage reads).
- A **lazy** first call during rendering would capture the **site arena**
(`context.allocator` after `main.odin` swaps it in), which is destroyed every
watch-mode rebuild → dangling parsers across builds.
This would break even single-threaded; threading just decides *which* test wins
the init race.
## tree-sitter's threading model (the root fact)
tree-sitter objects have specific, documented threading rules:
| Object | Shareable across threads? | Rule |
|---|---|---|
| `TSLanguage` (`tree_sitter_html()` etc.) | Yes | Immutable; share freely. |
| `TSParser` (`gc.parser`) | **No** | One per thread; never parse on one parser from two threads. |
| `TSTree` | Only via `ts_tree_copy` | Copy before using on another thread. |
| `TSQuery` (`gc.query`) | Yes | Immutable after compile. |
| `TSQueryCursor` (`gc.cursor`) | **No** | One per thread, like the parser. |
This explains the existing design:
- The preload fan-out (`preload_grammar` / `grammar_worker`) creates a parser
**per worker** and only publishes the finished `Grammar_Cache` under
`cache_mutex` — because parsers can't be shared during construction/use.
- `ensure_parser` + the shared cached parser is fine in production **only
because rendering is single-threaded**.
A cached `Grammar_Cache` mixes shareable (`language`, `query`) and per-thread
(`parser`, `cursor`) objects in one shared struct, which makes it
single-threaded-use by construction — a fact that was previously undocumented.
## Where this actually bites
**Only in tests.** Confirmed by grep:
- Production parses on one thread: every real caller — `minify.odin` (html/css)
and `markdown/highlight.odin` — runs inside `main`'s single-threaded render
loop. No concurrent `ensure_parser` / `parser_parse_string`.
- The one threaded path in production, `preload_grammars`, is already safe (own
parser per worker + `cache_mutex` on the write) and only handles non-builtin
grammars.
- Only `minify_test.odin` touches tree-sitter from tests, and the test runner
runs those on many threads.
So production is correct **by convention** (single-threaded-from-heap usage),
not **by construction** — nothing stops a future parallel render from breaking
it, and until now nothing documented the constraint.
## Chosen direction
**For now:** keep the single global grammar cache and make it safe by
**serializing the non-thread-safe behavior with a mutex** — only one thread uses
a parser at a time. This is simple, matches how production already behaves, and
unblocks the tests.
**Later (TODO):** decide whether true multi-threaded parsing is worth it for
performance. If yes, give each thread its own parser + cursor (as the preload
workers already do) rather than sharing one behind a lock.
## Implementation options (deferred — pick when we resume)
All assume the allocator fix (#2) below, which is independent and unambiguous.
- **#2 allocator (do regardless):** pin the heap allocator explicitly —
`grammar_store.allocator = runtime.heap_allocator()` (or `os.heap_allocator()`)
instead of `context.allocator`. Makes the cache persistent from any context;
behavior-identical in production. Optionally make `init_persistent` idempotent
and/or an `@(init)` so callers need no ordering.
Then, for the races:
- **Option A — library mutex (sustainable):** take `cache_mutex` in
`ensure_parser` (fixes #1a) and add a locking `parse` helper that wraps
`parser_parse_string` (fixes #1b), wired into the three call sites
(`minify.odin` ×2, `markdown/highlight.odin` ×1). Tests then need no
scaffolding. Serializes all parsing.
- **Option B — test-side only (minimal):** leave the library single-threaded by
contract; make the tests not violate it. Either fold the minify tests into one
`@(test)` (no concurrency at all → no locks), or wrap the `minify_*` calls in a
test-local mutex. Production stays lock-free.
Recommendation captured during the discussion: the allocator one-liner is
worth doing immediately; for the races, Option A is the "correct" long-term
answer but only necessary if/when we want concurrent parsing — otherwise
Option B is enough.
## Pointers
- `treesitter/treesitter.odin`: doc comments on `Grammar_Cache`, `cache_mutex`,
`init_persistent`, `ensure_parser` reference this file.
- Original feature that surfaced this: minify inline `<style>` bodies via
`minify_css` (reverted; to be re-landed alongside whichever fix we choose).
HUGO.md
+51 -8
View File
@@ -1,11 +1,54 @@
# Hugo Notes
# Hugo vs Thor: Why Thor Doesn't Need Shortcode Context Isolation
from the [Hugo docs](https://gohugo.io/quick-reference/glossary/#default-sort-order)
## Hugo's Shortcode Isolation
default sort order
Hugo strictly separates shortcode context from layout template context. Shortcodes run during markdown rendering, before layouts. They get a limited context (`.Page`, `.Site`, `.Params`), not the full layout rendering state.
: The default sort order for page collections, used when no other criteria are set, follows this priority:
1. weight (ascending)
2. date (descending)
3. linkTitle falling back to title (ascending)
4. logical path (ascending)
### Why Hugo does this
1. **Circular dependencies** — Hugo shortcodes output into `.Content`. Layouts read `.Content` and compute derived properties (`.TableOfContents`, `.WordCount`, `.ReadingTime`). If shortcodes could access these computed properties, you'd get circular dependencies (shortcode output → computed property → shortcode reads it).
2. **Multiple layouts/themes** — Hugo supports user-selectable themes and multiple layouts per section type. A shortcode must produce identical output regardless of which layout renders it. Isolating context guarantees portability.
3. **Shared mutable page object** — Hugo builds a rich `Page` object that both shortcodes and layouts access. Context isolation prevents shortcodes from mutating layout-visible state.
4. **Security** — Shortcodes live in content files (potentially user-authored). Full template access would let content authors inject arbitrary logic, access sensitive config, or break site structure.
5. **Caching** — Hugo caches shortcode output independently. Predictable context = predictable cache behavior.
## Why this doesn't apply to thor
- **No computed page properties** — The content Mustache pass runs on raw markdown (before cmark). The layout Mustache pass runs on finished HTML (after cmark). There's no shared `.Content` object or derived properties. The two passes are at different pipeline stages with no shared mutable state.
- **Single layout set** — Thor has one set of templates (`base.html` + page templates). No theme switching, no multiple layout variants per section. Portability across layouts isn't a concern.
- **Explicit data passing** — Each `mustache.render` call receives its own data struct. Content pass gets frontmatter + params. Layout pass gets page data + site config. No shared mutable object between them.
- **Single author** — Content is authored by the site owner. No untrusted user-generated content.
- **No caching** — Thor rebuilds from scratch every time. No cache invalidation concerns.
## Thor's approach
Thor runs Mustache on content **before** cmark as a pre-processing step, then runs Mustache on layout templates **after** cmark as a post-processing step. These are two independent `mustache.render` calls:
```
content (markdown)
→ mustache.render(content, content_data, partials) ← content pass
→ cmark markdown_to_html
→ inject_sidenotes / inject_alerts / highlight_code
→ mustache.render_in_layout(template, page_data, layout, partials) ← layout pass
→ final HTML
```
The content pass can use Mustache variables, partials (`{{> ./file}}`), sections, and conditionals freely. The layout pass wraps the rendered content in the page chrome. They share nothing except the data we explicitly choose to pass.
## When this WOULD matter for thor
If thor ever adds:
- Computed page properties (table of contents, reading time, word count)
- Multiple selectable themes/layouts
- User-generated content / multi-author support
- Partial template caching
...then context isolation between the content and layout passes would become relevant. Until then, it's unnecessary complexity.
ROADMAP.md
-131
View File
@@ -1,131 +0,0 @@
## v0.2.0
- [ ] [aliases](https://gohugo.io/methods/page/aliases/#redirects)?
- [ ] consider sites with date based urls.
- [ ] Create a json schema file for `thor.json`.
- [ ] Content-hash fingerprinting for CSS and JS cache busting
- [ ] Mount content in VFS
- [ ] `-watch` flag
- [x] basic poll loop
- [ ] filesystem poll loop
- [ ] event based
- [ ] if no `html` tag detected in output, re-render output with base template
(or whatever template is next in the chain). (Hugo seems to do this)
- [ ] Markdown
- [ ] Block attributes extension — hello-world.md
- [ ] add tables extension
- [ ] enable template level rendering of TOCs
- [ ] Write css for toc sidebar and figure out where to put it.
- [ ] Add [hugo style configuration](https://gohugo.io/configuration/markup/#table-of-contents)
- [ ] implicit page titles (Set when missing?)
- guess from h1 tag or first h2 tag?
- [ ] Syntax highlighting
- [ ] grammars
- [ ] load grammars dynamically
- [ ] Search in multiple places
- [ ] Download missing grammars.
- [ ] Durable highlight paths: read `GRAPHS_PATH`/`QUERIES_PATH` from env vars set by the flake instead of hardcoded nix store hashes, so they survive `nix flake update` and let the grammar/query version-mismatch detector fire automatically.
- [ ] Theme selector
- use http://github.com/helix-editor/helix/tree/master/runtime/themes) as a
guide
- [ ] Unit tests for highlighting helpers: `capture_name_to_css`, `escape_html`, `unescape_html`, `extract_query_token`, `helix_version_from_path`.
- [ ] `<pre><code>` blocks need to set background to theme background,
regardless of prefers-dark. (or use a different theme)
- [ ] Avoid `json.Value` / `json.Object` where possible.
- [ ] come up with scrapers / scrape sources to harvest site data
- we'll use this to help us sculpt defaults.
- [ ] merge `render_{section,home_html,page_html}` procs.
- [ ] try to combine render_page_html and render_home_html?
- [ ] make `parse` an overload of `parse_text/parse_inline` and `parse_file`, or something.
- [ ] refactor `mustache.load_partials` to accept an iterator so we don't copy `main.load_partials`
### Menus
- [ ] Nav items should be active when the current page is selected.
- [ ] configure opt-out of automatic sections being added to menu.
- [ ] nested menus (i.e. `parent` support)
### Strict Mode
- [ ] opt-in "strict_keys" mode. in this mode, key lookups may not view parent objects.
- [ ] warn/error when unknown key used in mustache.
- [ ] enforce heading structure.
- [ ] Either frontmatter.title set, or 1 h1 tag at top, not both
- [ ] No skipping.
### Dates
- [ ] display an error when no part of the date appears in the output.
- [ ] Handle 0 and whitespace padding i.e. "_2" -> " 2"
- [ ] Do we *need* mustache.Date_Components, or can we use core:time/datetime.DateTime?
## Beyond
- [ ] consider logging the number of times an error occurred. (diagnostics)
- [ ] Need to be careful about diagnostics across module boundaries.
- we don't necessarily want to warn users about theme designers mistakes. (though perhaps we do)
- [ ] consider reporting duplicate weights outside of menus (diagnostic)
- [ ] Do mustache's whitespace rules actually suit us, or should we make our own?
- [ ] create a default `head.html`?
- [ ] improve home link customization.
- [ ] currently an accessibility issue.
- [ ] get rid of the global variables in the `treesitter` package.
- [ ] site-wide Integrity hash
- Allows users to verify their output didn't change after upgrading to a new version
- [ ] Add `-production` flag
- sets `-minify`
- [ ] manually pass `site_allocator` to `load_page`
- [ ] Import/export packages. Hugo, jekyll, WordPress, etc.
- [ ] shortcodes?
- include-code shortcode (`{{< include-code ... >}}`) — i-ported-fd-to-odin
- [ ] follow symlinks in `scan_content`?
- [ ] Add overloads for every md extension? i.e. accept ^strings.Builder.
- [ ] content link checker?
- Checks all links on each page to make sure they are valid.
- [ ] Peruse [GitHub's](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#alerts)
docs for any juicy nuggets we may have missed.
- [ ] Avoid using `render_inline_md` if possible.
- [ ] don't use bullshit "sub-tokens", add filters and pipes as proper tokens.
- [ ] Create a Location struct that somewhat matches Odin's [Source_Code_Location](https://pkg.odin-lang.org/base/runtime/#Source_Code_Location)?
- Note that odin's version doesn't contain the stack trace.
- [ ] Link checker / linter that warns if links don't resolve. ("please use {{whatever | rel_url }}").
### Memory Management
- [ ] Leaks in highlighter code.
- [ ] Not sure whether to use temp allocator or site_allocator in opengraph.odin.
- [ ] Not sure whether to use temp allocator or site_allocator in `site_load_content`.
- [ ] Might not need to allocate in `strip_html_tags`
- [ ] Fix `apply_filter`'s `format` case (`mustache/pipes.odin`) boxing `apply_format`'s
`string` result into `any` via bare `return`, which materializes a hidden
header temp in `apply_filter`'s own stack frame. Dangling once the frame
returns; caused the `-o:speed` segfault in `write_value`. Fix: box explicitly
with `any{new_clone(formatted, context.temp_allocator), typeid_of(string)}`.
- [ ] Same pattern in `apply_group_by` (`mustache/pipes.odin`): `return groups, nil`
boxes a freshly-built `[dynamic]Group` as bare `any` — same latent
stack-temp UB, hasn't crashed yet but should get the same treatment.
### Performance
- [ ] See if we can disable bounds checks in `write_indented` and elsewhere.
- [ ] Instead of loading the site fresh each time in watch mode, create a
`reload_site` proc, that just updates changed resources.
- [ ] Only publish referenced assets.
- [ ] Split `load_page` into frontmatter-parse + body-process phases so draft pages can skip the markdown pipeline entirely
- [ ] Use spall to find ways to reduce run time.
- [ ] too many `write_string` calls in `highlight_block`
- [ ] return `src: cstring` from `load_query`.
- [ ] Improve `unescape_html` with simd.
- [ ] generate summary before syntax highlighting.
- [ ] generate summary before markdown to html conversion.
- [ ] mount_recursive is pretty significant
- [ ] thread pool for grammar loading is unbounded.
- [ ] load grammars async.
- [ ] during `load_page`:
- pass each code block to the treesitter queue
- continue working on the page,
- `await` the highlighted code.
- [ ] can markdown extensions run in parallel?
- [ ] enforce MAX_SLUG_LENGTH
- [ ] enforce MAX_CONTEXT_DEPTH
- [ ] ensure struct fields are ordered correctly
TODOS.md
+251 -118
View File
@@ -1,134 +1,162 @@
# TODOs
## High priority
## Priority Unclear (Evaluate Later)
- [ ] Decide the parallel-parsing strategy for the grammar cache (pool vs
thread-local parsers) — deferred until the parallel usage shape is clear.
- Resolved already: the registry (`grammar()`) is thread-safe and self-
initializing, its allocator is pinned to the heap, and parsers/cursors are now
caller-owned (`open_parser`) instead of cached — so the library holds no
single-thread-use state and the multi-threaded tests pass without locks.
- Remaining choice only matters once we actually parse in parallel: reuse
parsers via a pool or per-thread storage, plugged in behind `open_parser`
without touching call sites. A `TSParser`/`TSQueryCursor` is one-per-thread.
- Full write-up: `GRAMMAR_CACHE_THREADING.md`.
- [x] Restore parallel grammar preload (serialized in `edb2e64`); `preload_grammars`
now fans `build_grammar` across a bounded `core:thread.Pool`.
- [ ] Add tests: pool fan-out correctness + `-sanitize:thread` for race-freedom.
- [ ] Concurrent-read path: an `RW_Mutex` on `grammar()` only helps a future
multi-threaded render, not the load side. Revisit if render goes parallel.
- [ ] instead of `warnings: [dynamic]Error` we should use `warnings: [dynamic; 8]Error`
- when limit reached, the template fails and stops rendering.
- [ ] Brainstorm plugin/tool/(postccs/less/tailwind/sass) support.
- [ ] `fingerprint` pipe + contents-based composition. The `tool` pipe currently
carries a URL string and each stage publishes a file to the output dir, so
`| tool sass | fingerprint` would leave a dead unhashed intermediate. When
fingerprint lands, move the pipe value from a URL string to a contents/
Resource (bytes + logical name) where only the terminal stage publishes;
a Resource must stringify to its URL so existing bare-`tool` templates keep
working. stdin/stdout tools compose best there (bytes stay in memory), which
is why stdin/stdout is the documented preferred form over `{{in}}`/`{{out}}`.
- [ ] Reconsider `"menus": {}` opt-out: it currently discards all page frontmatter
menus (with only a warning). Decide whether empty config should instead just
suppress the auto-generated menu while still honoring frontmatter.
- [ ] asset css files not automatically included in site.
- should we at least warn about them?
- [ ] Review the whole `copy_assets_dir` pipeline. It copies every file under
`assets/` to the output dir blindly, whether or not anything references it,
and now also skips dests that already exist (a blunt collision guard so the
raw copy doesn't clobber `tool`-pipe output written earlier in the build).
Reconsider tracking which assets are actually used/published instead.
- [ ] warn if stylesheet url starts with /assets (and is therefore invalid).
- [ ] warn when heading levels are skipped in a toc.
- [ ] extract `[TOC]` from `.md` body (or at least from the start.)
- [ ] add `--production` flag
- set `minify` to true
- warn when `base_url` unset
## General
Remember to polish existing features before moving on to new ones.
- [ ] Bake the default `layouts` into the binary at compile time via
`#load_directory` so the binary is self-contained and doesn't read
`DEFAULTS_PATH` from disk at runtime.
- `#load_directory` is NOT recursive and its `name` is the base filename
only, so `defaults/layouts` and `defaults/layouts/partials` must be loaded
separately and mounted into the VFS with `data` set (not `fs_path`).
- Removes the runtime dependency on `THOR_DEFAULTS_PATH` / `#directory` and
the need to ship `defaults/` alongside the binary (see `flake.nix`).
- [ ] `build` command alias of default
- [ ] `new site` command to set up new project
- [ ] Some render errors blank the entire page instead of scoping to the failing tag.
- [ ] We must remove all mention of `posts` from the odin code.
At present, "posts" are a user-level construct defined as pages in a
particular collection.
- [ ] ? Clean up the default layouts (no idea what this means)
- [ ] `group_by` currently requires a computed `year` field on the page.
- We should replace this with `{{ pages | group_by (date | "2006") }}` or similar
- Polish existing features before moving on to new ones.
- [ ] Do mustache's whitespace rules actually suit us, or should we make our own?
- [x] `{{>title}}` default partial? `{{site.title}} | {{ page.title }}`
- [ ] implicit titles (Set when missing?)
- [ ] create a default `head.html`.
- [ ] html comments are rendered, except in minify mode.
- [ ] does it make sense for partials to be inside layouts?
- current: `layouts/partials`
- alt1: `layouts`, `partials`
- alt2: `templates/layouts`, `templates/partials`
- [ ] add `sort_by` pipe
- [ ] pipes
- [x] first/last
- [ ] write test for error when n == 0
- [x] warn when using negative numbers
- [x] warn when using default arg on string
- [x] add the above warnings to DIAGNOSTIC_TESTS.md
- [ ] Instead of "clearer form", be exact: "last 5"
- [ ] write tests for `first 0` & `last 0`
- [ ] update tests to assert diagnostics when appropriate
- do we want to do this or just use DIAGNOSTIC_TESTS.md as the tests?
- [ ] also update to use file logger so they don't clog up the output
- [ ] narrow highlight in diagnostic to relevent part of line.
- [ ] page.url and page.permalink are confusing names. Switch to url + page_url
- There is confusion over how to do this and whethor or not `| rel_url` should be needed to render these.
Requires **careful thought**.
## Documentation
- [ ] talk about the context stack (and its limit).
- [ ] highlight the differences in the way menus are handled.
- [ ] explain that html/css/js comments are rendered, except in minify mode.
## Investigate
- [ ] running ./thor/thor still logs the debug message: using config /home/spencer/github.com/sbrow.github.io/thor.json
- wrong cwd?
## Diagnostics
All diagnostics (errors) should show
- *What* went wrong
- *where* (in the file)
- *where* (in the stack trace)
- *how* you can fix it (if applicable)
- [ ] try to make file paths clickable links.
- [ ] make sure the frontmatter parser has good diagnostics.
- [ ] add all known diagnostics to [DIAGNOSTIC TODOS](./DIAGNOSTIC_TESTS.yaml).
- [ ] fix the broken diagnostics in [DIAGNOSTIC TODOS](./DIAGNOSTIC_TESTS.yaml)
- [ ] show *stack trace* for template partials?
- [ ] show where the typo occurred (config file path for `apply_extension_config`, full `-ext:value` for `parse_extension_list`)
- [ ] **all** diagnostics in [DIAGNOSTIC TODOS](./DIAGNOSTIC_TESTS.yaml) must be tested.
- [ ] [aliases](https://gohugo.io/methods/page/aliases/#redirects)?
- [ ] Improve diagnostics
- [x] keep track of every error and don't report them more than once.
- [ ] show parsed arg when -extension is unrecognized
- [ ] also do typo detection?
- [ ] `*` make sure the frontmatter parser has good diagnostics.
- [ ] fix the diagnostics in [DIAGNOSTIC TODOS](./DIAGNOSTIC_TODOS.yaml)
- [ ] only report format errors once.
- [ ] only report missing partial errors once.
- [ ] only report missing partiall errors once.
- [ ] All Diagnostics should show:
- [ ] *What* went wrong
- [ ] *where* (in the file)
- [ ] *where* (in the stack trace)
- [ ] *how* you can fix it (if applicable)
- [ ] Create a Location struct that somewhat matches Odin's [Source_Code_Location](https://pkg.odin-lang.org/base/runtime/#Source_Code_Location)?
- Note that odin's version doesn't contain the stack trace.
- [ ] show "stack traces" in template error diagnostics
- [ ] better diagnostics for syntax errors in treesitter.
- [ ] Ensure diagnostics for MAX_CONTEXT_DEPTH are good.
- [ ] improve matching weights message.
- [ ] Test menu diagnostics
- [ ] Honestly, Test **all** diagnostics
- [ ] Need to be careful about diagnostics across module boundaries.
- we don't necessarily want to warn users about theme designers mistakes. (though perhaps we do)
- [ ] consider reporting duplicate weights outside of menus
- [ ] Extend `tag_error` to all render-time errors, not just pipe errors.
Currently only pipe errors (4 sites in `render_nodes`) get stamped with
the correct template source/path. Other render errors still use the
content template's source/path, which can point at the wrong file.
- [ ] try to make file paths clickable links.
- [ ] centralize diagnostics to one place.
- [ ] consider logging the number of times an error occurred.
- [ ] Load grammars dynamically
- [x] starred must be a param.
- [ ] Documentation
- [ ] talk about the context stack (and its limit).
- [ ] highlight the differences in the way menus are handled.
- [ ] consider sites with date based urls.
- [x] Don't show annoying log output in tests.
- [ ] improve home link customization.
- [ ] currently an accessibility issue.
- [x] support JSON5 in in frontmatter
- [ ] Create a json schema file for `thor.json`.
- [x] cleanup `#partial switch`es.
- [ ] improve json diagnostics.
- i.e. "Missing quotes around string", etc.
- [ ] don't use bullshit "sub-tokens", add filters and pipes as proper tokens.
- [ ] Ideas is now in the wrong spot. Date is wrong, and it is showing date
when it shouldn't be.
## Analytics
## Performance
Debug log stats:
- [ ] See if we can disable bounds checks in `write_indented` and elsewhere.
- [ ] Instead of loading the site fresh each time in watch mode, create a
`reload_site` proc, that just updates changed resources.
- [ ] Only publish referenced assets.
- [ ] Split `load_page` into frontmatter-parse + body-process phases so draft pages can skip the markdown pipeline entirely
- [ ] Use spall to find ways to reduce run time.
- [ ] too many `write_string` calls in `highlight_block`
- [ ] return `src: cstring` from `load_query`.
- [ ] Improve `unescape_html` with simd.
- [ ] generate summary before syntax highlighting.
- [ ] generate summary before markdown to html conversion.
- [ ] mount_recursive is pretty significant
- [ ] thread pool for grammar loading is unbounded.
- [ ] load grammars async.
- [ ] during `load_page`:
- pass each code block to the treesitter queue
- continue working on the page,
- `await` the highlighted code.
- [ ] can markdown extensions run in parallel?
- [ ] enforce MAX_SLUG_LENGTH
- [ ] enforce MAX_CONTEXT_DEPTH
- [ ] ensure struct fields are ordered correctly
## Remove Privileged content
- [ ] `group_by` currently requires a computed `year` field on the page.
- We should replace this with `{{ pages | group_by (date | "2006") }}` or similar
## Memory Management
- [ ] Leaks in highlighter code.
- [ ] Not sure whether to use temp allocator or site_allocator in opengraph.odin.
- [ ] Not sure whether to use temp allocator or site_allocator in `site_load_content`.
- [ ] Might not need to allocate in `strip_html_tags`
- [ ] Fix `apply_filter`'s `format` case (`mustache/pipes.odin`) boxing `apply_format`'s
`string` result into `any` via bare `return`, which materializes a hidden
header temp in `apply_filter`'s own stack frame. Dangling once the frame
returns; caused the `-o:speed` segfault in `write_value`. Fix: box explicitly
with `any{new_clone(formatted, context.temp_allocator), typeid_of(string)}`.
- [ ] Same pattern in `apply_group_by` (`mustache/pipes.odin`): `return groups, nil`
boxes a freshly-built `[dynamic]Group` as bare `any` — same latent
stack-temp UB, hasn't crashed yet but should get the same treatment.
## Markdown
- [ ] Add overloads for every extension - accept ^strings.Builder.
- [x] Add conventional (Hugo style) footnotes option.
- [x] Add opt-in deflist support.
- [x] Decide if lambdas actually provide any value.
- [ ] add tables extension
- [x] Table of contents support.
- [ ] enable template level rendering of TOCs
- [ ] Write css for toc sidebar and figure out where to put it.
- [ ] Add [hugo style configuration](https://gohugo.io/configuration/markup/#table-of-contents)
- [ ] Link checker?
- Checks all links on each page to make sure they are valid.
- [ ] Peruse [GitHub's](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#alerts)
docs for any juicy nuggets we may have missed.
- [ ] Avoid using `render_inline_md` if possible.
## Dates
- [ ] display an error when no part of the date appears in the output.
- [ ] Handle 0 and whitespace padding i.e. "_2" -> " 2"
- [ ] Do we *need* mustache.Date_Components, or can we use core:time/datetime.DateTime?
## General
- [ ] Menus
- [ ] configure opt-out of automatic sections being added to menu.
- [ ] nested menus (i.e. `parent` support)
- [ ] get rid of the global variables in the `treesitter` package.
- [ ] enforce heading structure.
- [ ] Either frontmatter.title set, or 1 h1 tag at top, not both
- [ ] No skipping.
- [ ] Consider using `or_else` when applying default values to structs. i.e.
```odin
package main
X :: struct {
foo: string
}
main :: proc () {
x: X
x.foo = x.foo or_else "bar"
}
```
- [ ] Integrity hash
- Allows users to verify their output didn't change after upgrading to a new version
- [ ] Content-hash fingerprinting for CSS and JS cache busting
- [ ] come up with scrapers / scrape sources to harvest site data
- we'll use this to help us sculpt defaults.
- [ ] merge `render_{section,home_html,page_html}` procs.
- [ ] try to combine render_page_html and render_home_html?
- [ ] Debug log stats. (analytics)
- [ ] final Context_Stack cap
- [ ] highest PIPE args used
- [ ] longest slug length + name that generated it
@@ -136,8 +164,113 @@ Debug log stats:
- [ ] number of blocks
- [ ] enabled features / extensions
- [ ] etc
- [ ] Avoid `json.Value` / `json.Object` where possible.
- [ ] make `parse` an overload of `parse_text/parse_inline` and `parse_file`, or something.
- [x] Add page params
- [ ] We must remove all mention of `posts` from the odin code.
At present, "posts" are a user-level construct defined as pages in a
particular collection.
- [ ] running ./thor/thor still logs the debug message: using config /home/spencer/github.com/sbrow.github.io/thor.json
- wrong cwd?
- [ ] Clean up the default layouts
- [ ] Menus
- [x] Detailed frontmatter menu form ("menu": {"main": {"weight": 5}})
- [ ] Menu active state (pre-compute is_active based on page.permalink prefix match)
- [x] Page.weight field for general-purpose page ordering (menus, lists, related posts)
- [ ] if no `html` tag detected in output, re-render output with base template
(or whatever template is next in the chain)
- [ ] Add `-production` flag
- sets `-minify`
- [ ] Mustache diagnostics
- [ ] Partial invocation stack in diagnostics: when an error fires inside a partial, show "invoked from" chain through `{{> name}}` calls. Currently warnings inside partials point at the partial (correct file) but don't show the invocation site.
- [x] Error message doesn't show position of faulty pipe name correctly.
- [ ] `render_template` (`render.odin`) blanks the *entire page* to `""` on any
mustache render error and only `log.errorf`s it — a single bad tag/pipe
anywhere on the page silently kills the whole output with no visible
signal outside the terminal log. Should at least be scoped to the
failing tag/section, or surfaced somewhere the person building the
site will actually see it.
```bash
[ERROR] --- [138:render_template()] unknown pipe op 'formats'
--> /home/spencer/github.com/sbrow.github.io/layouts/home.html:1:1
|
1 | {{<base}}
| ^^^^^^^^^
2 | {{$content}}
3 | <main>
```
- [ ] Block attributes on code fences (`{ #ex-1 }`) — hello-world.md
- [ ] include-code shortcode (`{{< include-code ... >}}`) — i-ported-fd-to-odin
- [ ] follow symlinks in `scan_content`?
- [ ] ensure sidenote numbers render in display order and not in declaration order.
- [ ] Nav items should be active when the current page is selected.
- [ ] Theme selector for syntax highlighting.
- use http://github.com/helix-editor/helix/tree/master/runtime/themes) as a
guide
- [ ] grammars
- [ ] Search in multiple places
- [ ] Download missing grammars.
- [ ] Durable highlight paths: read `GRAPHS_PATH`/`QUERIES_PATH` from env vars set by the flake instead of hardcoded nix store hashes, so they survive `nix flake update` and let the grammar/query version-mismatch detector fire automatically.
- [ ] CI
- [ ] Syntax highlighting in production: CI (`nix build` on ubuntu-latest) has no grammar `.so`s and a machine-specific `QUERIES_PATH` nix store hash, so the deployed site renders unhighlighted. Provide grammars + queries as nix build inputs and pass paths to thor at runtime (env vars/flags).
- [ ] Unit tests for highlighting helpers: `capture_name_to_css`, `escape_html`, `unescape_html`, `extract_query_token`, `helix_version_from_path`.
- [ ] `<pre><code>` blocks need to set background to theme background,
regardless of prefers-dark. (or use a different theme)
- [ ] `-watch` flag
- [x] basic poll loop
- [ ] filesystem poll loop
- [ ] event based
- [x] Free cmark HTML output (`body_html`) — cmark allocates via C malloc, not the arena, so it leaks per iteration in watch mode
- [ ] manually pass `site_allocator` to `load_page`
- [ ] Mount content in VFS
- [ ] commands
- [ ] `build` alias of default
- [ ] `new site` set up new project
- [ ] warn/error when unknown key used in mustache.
- [ ] Import/export packages. Hugo, jekyll, WordPress, etc.
- [ ] opt-in "strict_keys" mode. in this mode, key lookups may not view parent objects.
## sbrow.github.io
- [ ] The home file was accidentally deleted in commit 9a9f731
- [ ] Ideas is now in the wrong spot. Date is wrong, and it is showing date
when it shouldn't be.
## Notes
from the [Hugo docs](https://gohugo.io/quick-reference/glossary/#default-sort-order)
default sort order
: The default sort order for page collections, used when no other criteria are set, follows this priority:
1. weight (ascending)
2. date (descending)
3. linkTitle falling back to title (ascending)
4. logical path (ascending)
## Code Review
A human should manually review every file in the project. AI cannot complete
these tasks.
- [ ] Review every file in thor
- [ ] Review assets.odin
- [ ] Review content.odin
- [ ] Review defaults.odin
- [ ] Review feed.odin
- [ ] Review frontmatter.odin
- [ ] Review main.odin
- [ ] Review minify.odin
- [ ] Review `markdown/`
- [x] Review alerts.odin
- [x] Review alerts_test.odin
- [x] Review emoji.odin
- [x] Review emoji_test.odin
- [ ] Review footnotes.odin
- [ ] Review footnotes_test.odin
- [ ] Review highlight.odin
- [ ] Review markdown.odin
- [x] Review sectionate.odin
- [x] Review sectionate_test.odin
- [ ] Review suggest.odin
- [ ] Review suggest_test.odin
- [x] Review opengraph.odin
- [ ] Review render.odin
- [x] Review site.odin
- [ ] Review treesitter/treesitter.odin
- [ ] Review vfs.odin
- [ ] Review procs
- [ ] markdown.transform_alert
WARNING_COUNTER.md
-109
View File
@@ -1,109 +0,0 @@
# Warning Counter Plan
## Goal
When the same warning fires multiple times across a single page render, show it once with a `[×N]` count prefix instead of N identical diagnostic blocks.
```
[×5] first -3 is equivalent to last 3 — consider using the clearer form
--> layouts/home.html:12:1
|
12 | <ul>{{#items | first -3}}<li>{{.}}</li>{{/items}}</ul>
| ^^^^^^^^^^^^^^^^^^^^^ consider using the clearer form
|
```
## Approach
Module-level buffer in `pipes.odin`. Warnings are collected during render, deduplicated + counted + logged after the page completes. No parameter threading through `render_nodes`.
## Changes
### `mustache/pipes.odin`
**New module-level state:**
```odin
_pending_warnings: [dynamic]Error
```
**New procs:**
```odin
reset_warnings :: proc() {
clear(&_pending_warnings)
}
flush_warnings :: proc(colorize: bool = false) {
if len(_pending_warnings) == 0 do return
counts := make(map[string]int, context.temp_allocator)
first: map[string]Error = make(map[string]Error, context.temp_allocator)
defer delete(counts)
defer delete(first)
for w in _pending_warnings {
b := body(w)
if _, seen := first[b.msg]; !seen {
first[b.msg] = w
}
counts[b.msg] += 1
}
for msg, w in first {
c := counts[msg]
b := body(w)
path := b.path != "" ? b.path : "<input>"
prefix := c > 1 ? fmt.tprintf("[×%d] ", c) : ""
diag := diags.format_error(
path, b.source, b.pos, b.msg,
hint = b.hint,
colorize = colorize,
)
log.warnf("%s%s", prefix, diag)
}
clear(&_pending_warnings)
}
```
**`apply_pipeline` change:**
Replace the current warning handling (which logs immediately):
```odin
if warnings != nil {
append(warnings, ferr)
} else {
log.warnf("%s", format_render_error(ferr, tmpl))
}
```
With:
```odin
append(&_pending_warnings, ferr)
```
Drop the `warnings: ^[dynamic]Error` parameter — no longer needed.
### `render.odin`
In `render_template`, wrap the render call:
```odin
mustache.reset_warnings()
result, err := mustache.render(content_tpl, []any{ctx.site, ctx.page, ctx}, partials)
mustache.flush_warnings(colorize = diags.should_colorize())
```
### `mustache/pipes.odin` cleanup
- Remove `warnings` parameter from `apply_pipeline` signature
- Remove `format_render_error` import/call from `apply_pipeline` (formatting moves to `flush_warnings`)
- Remove the `log` import if no longer used directly in `apply_pipeline`
## Scope
~25 lines new code (`reset_warnings` + `flush_warnings`), ~3 lines changed in `apply_pipeline`, ~2 lines added in `render.odin`. Total ~30 lines.
## Limitations
- Module-level state means warnings don't survive across page renders (by design — `reset_warnings` is called per page)
- Not thread-safe (fine for sequential rendering)
- Counting is by exact `msg` string — warnings with different messages but same root cause are counted separately
assets.odin
+1 -24
View File
@@ -5,20 +5,7 @@ import "core:log"
import "core:os"
import "core:strings"
import ts "treesitter"
copy_assets_dir :: proc(vfs: ^VFS, output_dir: string, features: bit_set[Feature]) {
// One CSS parser, reused for every .css asset (single-threaded). Owned here.
css_parser: ts.Parser
if .Minify in features {
if g, ok := ts.grammar("css"); ok {
css_parser = ts.open_parser(g)
}
}
defer if css_parser != nil {
ts.parser_delete(css_parser)
}
for virtual_path, entry in vfs.files {
if !strings.has_prefix(virtual_path, "assets/") {
continue
@@ -27,12 +14,6 @@ copy_assets_dir :: proc(vfs: ^VFS, output_dir: string, features: bit_set[Feature
rel := virtual_path[len("assets/"):]
dest := fmt.tprintf("%s/%s", output_dir, rel)
// Don't clobber a file already written this build (e.g. the output of a
// `tool` pipe, which publishes to the same dest path as its source).
if os.exists(dest) {
continue
}
if idx := strings.last_index(dest, "/"); idx >= 0 {
if err := os.make_directory_all(dest[:idx]); err != nil && err != .Exist {
log.warnf("cannot create %s: %v", dest[:idx], err)
@@ -43,11 +24,7 @@ copy_assets_dir :: proc(vfs: ^VFS, output_dir: string, features: bit_set[Feature
if .Minify in features && strings.has_suffix(rel, ".css") {
data, ok := vfs_get(vfs, virtual_path)
if ok {
if m, mok := minify_css(css_parser, string(data)); mok {
write_file(dest, m)
} else {
write_file(dest, string(data))
}
write_file(dest, minify_css(string(data)))
}
} else if entry.data != nil {
if err := os.write_entire_file(dest, entry.data); err != nil {
content.odin
+1 -54
View File
@@ -71,9 +71,6 @@ site_load_content :: proc(site: ^Site) {
for &page in site.pages {
page.url = fmt.tprintf("%s%s", site.base_url, page.permalink)
if site.base_path != "" {
page.content = rewrite_root_urls(page.content, site.base_path, site_allocator(site))
}
}
build_menus(site)
@@ -292,57 +289,6 @@ load_page :: proc(
return
}
// rewrite_root_urls prefixes root-relative href/src attribute values with
// base_path so links authored in content (e.g. [x](/ideas)) resolve under a
// subpath deployment. Only values beginning with a single '/' are rewritten;
// absolute ("http://…"), protocol-relative ("//cdn…"), fragment ("#x"),
// mailto:, and document-relative ("foo/bar", "./foo") values all fail that
// test and pass through untouched. Assumes cmark's double-quoted output.
rewrite_root_urls :: proc(html, base_path: string, allocator := context.allocator) -> string {
b := strings.builder_make(allocator)
strings.builder_grow(&b, len(html))
i := 0
for i < len(html) {
n := attr_opener_at(html, i) // ` href="` or ` src="`
if n == 0 {
strings.write_byte(&b, html[i])
i += 1
continue
}
strings.write_string(&b, html[i:i + n]) // copy the opener verbatim
i += n
end := strings.index_byte(html[i:], '"')
if end < 0 { // malformed; bail cleanly
strings.write_string(&b, html[i:])
break
}
value := html[i:i + end]
if url_is_root_relative(value) {
strings.write_string(&b, base_path)
}
strings.write_string(&b, value) // closing quote copied next loop
i += end
}
return strings.to_string(b)
}
// url_is_root_relative reports whether value is a site-root path ("/faq/") as
// opposed to protocol-relative ("//cdn") or anything not starting with '/'.
url_is_root_relative :: proc(value: string) -> bool {
return len(value) >= 1 && value[0] == '/' && !strings.has_prefix(value, "//")
}
// attr_opener_at returns the length of a ` href="` / ` src="` attribute opener
// at s[i:], or 0. The leading space anchors the match to a real HTML attribute
// so we don't rewrite the strings inside link text or code blocks.
attr_opener_at :: proc(s: string, i: int) -> int {
if strings.has_prefix(s[i:], " href=\"") do return 7
if strings.has_prefix(s[i:], " src=\"") do return 6
return 0
}
is_content_file :: proc(name: string) -> bool {
return strings.has_suffix(name, ".md") || strings.has_suffix(name, ".html")
}
@@ -354,3 +300,4 @@ strip_extension :: proc(name: string) -> string {
}
return name[:dot]
}
defaults.odin
+1 -14
View File
@@ -2,17 +2,4 @@ package main
import "core:os"
// Path to the built-in default layouts, resolved at runtime.
//
// The default points at the `defaults` directory next to the source tree (via
// `#directory`), which works for local `odin run`/`odin build` from the repo.
// Packaged builds install `defaults/` to a fixed location and override this
// with `-define:THOR_DEFAULTS_PATH=<path>` (see flake.nix).
//
// TODO: bake these into the binary at compile time via `#load_directory` so the
// binary is self-contained and this runtime path goes away entirely. See
// TODOS.md.
DEFAULTS_PATH :: #config(
THOR_DEFAULTS_PATH,
#directory + os.Path_Separator_String + "defaults",
)
DEFAULTS_PATH :: #directory + os.Path_Separator_String + "defaults"
defaults/layouts/home.html
+1 -1
View File
@@ -5,7 +5,7 @@
{{&content}}
</header>
<ul>
{{#pages}} <li><a href="{{permalink | rel_url}}">{{&title}}</a><span>{{#date}}<time
{{#pages}} <li><a href="{{permalink}}">{{&title}}</a><span>{{#date}}<time
datetime="{{date}}">{{date | format}}</time>{{/date}}</span>
</li>
{{/pages}}
defaults/layouts/partials/nav.html
+2 -2
View File
@@ -1,9 +1,9 @@
<header>
<nav>
<ul>
<li><a href="{{site.base_path}}/">{{> home-link}}</a></li>
<li><a href="/">{{> home-link}}</a></li>
{{#menus.main}}
<li><a href="{{url | rel_url}}">{{name}}</a></li>
<li><a href="{{url}}">{{name}}</a></li>
{{/menus.main}}
</ul>
</nav>
defaults/layouts/partials/styles.html
+1 -72
View File
@@ -1,13 +1,4 @@
{{^params.stylesheets}}
<style>
/* Nav */
header nav ul {
list-style: none;
display: flex;
gap: 1rem;
padding: 0;
}
/* Base layout */
body {
margin-left: auto;
@@ -229,66 +220,4 @@ label.margin-toggle:not(.sidenote-number) {
}
}
</style>
{{/params.stylesheets}}
<style>
/* Syntax highlighting. Build-time <span class="hl-*"> wrappers only appear
inside fenced code blocks. The base class (e.g. hl-keyword) is always
emitted alongside any dotted variant (hl-keyword-control), so styling the
base names covers every capture. Re-theme by overriding the --hl-* vars. */
:root {
--hl-bg: #282c34;
--hl-fg: #abb2bf;
--hl-comment: #5c6370;
--hl-keyword: #c678dd;
--hl-string: #98c379;
--hl-escape: #56b6c2;
--hl-number: #d19a66;
--hl-function: #61afef;
--hl-type: #e5c07b;
--hl-variable: #e06c75;
--hl-attribute: #d19a66;
}
pre:has(> code [class^="hl-"]) {
background: var(--hl-bg);
color: var(--hl-fg);
padding: 0.75rem 1rem;
border-radius: 4px;
}
.hl-comment { color: var(--hl-comment); font-style: italic; }
.hl-keyword,
.hl-conditional,
.hl-repeat,
.hl-exception,
.hl-include,
.hl-storageclass { color: var(--hl-keyword); }
.hl-operator,
.hl-punctuation { color: var(--hl-fg); }
.hl-string,
.hl-character { color: var(--hl-string); }
.hl-escape,
.hl-string-escape,
.hl-string-special { color: var(--hl-escape); }
.hl-number,
.hl-float,
.hl-boolean,
.hl-constant { color: var(--hl-number); }
.hl-function,
.hl-method,
.hl-constructor { color: var(--hl-function); }
.hl-type,
.hl-namespace,
.hl-module { color: var(--hl-type); }
.hl-variable,
.hl-property,
.hl-field,
.hl-parameter,
.hl-tag,
.hl-symbol { color: var(--hl-variable); }
.hl-attribute,
.hl-label,
.hl-preproc { color: var(--hl-attribute); }
</style>
{{#params.stylesheets}}<link rel="stylesheet" href="{{.|rel_url}}">
{{/params.stylesheets}}
{{#params.stylesheets}}<link rel="stylesheet" href="{{.}}">{{/params.stylesheets}}
defaults/layouts/section_index.html
+1 -1
View File
@@ -7,7 +7,7 @@
<section>
<h2>{{key}}</h2>
<ul>
{{#items}} <li><a href="{{permalink | rel_url}}">{{&title}}</a><span>{{#date}}<time
{{#items}} <li><a href="{{permalink}}">{{&title}}</a><span>{{#date}}<time
datetime="{{date}}">{{date | format}}</time>{{/date}}</span>
</li>
{{/items}}
diagnostics/diagnostics.odin
-279
View File
@@ -1,279 +0,0 @@
package diagnostics
import "core:fmt"
import "core:os"
import "core:strings"
import "core:terminal/ansi"
import "core:unicode/utf8"
// suggest_correction returns the closest match from `available` to `missing`
// using Levenshtein distance, or "" if no good match exists. The threshold
// scales with the length of the missing key.
suggest_correction :: proc(available: []string, missing: string) -> string {
if len(available) == 0 || len(missing) == 0 {
return ""
}
threshold := max(2, len(missing) / 3)
best: string
best_dist := threshold + 1
for candidate in available {
if abs(len(candidate) - len(missing)) > threshold {
continue
}
d := strings.levenshtein_distance(missing, candidate)
if d <= threshold && d < best_dist {
best = candidate
best_dist = d
}
}
return best
}
// line_col returns the 1-indexed line and column for a byte offset in source.
line_col :: proc(source: string, pos_in: int) -> (line: int, col: int) {
pos := pos_in
if pos < 0 {
return 1, 1
}
if pos > len(source) {
pos = len(source)
}
line = 1
col = 1
for i := 0; i < pos; i += 1 {
if source[i] == '\n' {
line += 1
col = 1
} else {
col += 1
}
}
return
}
// line_text returns the Nth (1-indexed) line of source, without the trailing
// newline. Returns "" if line is out of range.
line_text :: proc(source: string, line: int) -> string {
if line < 1 {
return ""
}
current := 1
start := 0
for i := 0; i < len(source); i += 1 {
if current == line {
end := i
for end < len(source) && source[end] != '\n' {
end += 1
}
return source[start:end]
}
if source[i] == '\n' {
current += 1
start = i + 1
}
}
if current == line {
return source[start:]
}
return ""
}
// should_colorize returns true if stderr is a TTY and color output is wanted.
should_colorize :: proc() -> bool {
return os.is_tty(os.stderr)
}
// count_lines returns the number of '\n'-separated lines in source.
count_lines :: proc(source: string) -> int {
if len(source) == 0 {
return 1
}
n := 1
for c in source {
if c == '\n' {
n += 1
}
}
if len(source) > 0 && source[len(source) - 1] == '\n' {
n -= 1
}
return n
}
// digit_count returns the number of decimal digits in n (min 1).
digit_count :: proc(n: int) -> int {
if n <= 0 {
return 1
}
c := 0
x := n
for x > 0 {
c += 1
x /= 10
}
return c
}
// display_width returns the number of terminal cells `s` occupies.
display_width :: proc(s: string) -> int {
return utf8.rune_count_in_string(s)
}
// line_start_at returns the byte offset of the start of the line containing pos.
line_start_at :: proc(source: string, pos_in: int) -> int {
pos := pos_in
if pos < 0 {
return 0
}
if pos >= len(source) {
pos = len(source) - 1
}
for pos > 0 && source[pos - 1] != '\n' {
pos -= 1
}
return pos
}
// format_error produces a rust-style multi-line diagnostic string.
//
// <msg>
// --> <path>:<line>:<col>
// |
// N | <source line N-2>
// N | <source line N-1>
// N | <source line N — the error line>
// | ^^^^^^^^^^^ <hint>
// N | <source line N+1>
// N | <source line N+2>
// |
//
// `caret_start`/`caret_end` control the byte range of the caret underline.
// If not provided (or negative), defaults to a single-rune underline at `pos`.
format_error :: proc(
path: string,
source: string,
pos: int,
msg: string,
hint: string = "",
context_before: int = 2,
context_after: int = 2,
colorize: bool = false,
caret_start: int = -1,
caret_end: int = -1,
) -> string {
line, col := line_col(source, pos)
total_lines := count_lines(source)
start_line := line - context_before
if start_line < 1 {
start_line = 1
}
end_line := line + context_after
if end_line > total_lines {
end_line = total_lines
}
width := digit_count(end_line)
if width < 1 {
width = 1
}
cs := caret_start >= 0 ? caret_start : pos
ce := caret_end > cs ? caret_end : cs + 1
ls := line_start_at(source, pos)
caret_start_col := cs - ls + 1
caret_end_col := ce - ls + 1
sb := strings.builder_make(context.temp_allocator)
defer strings.builder_destroy(&sb)
red, faint, reset := "", "", ""
if colorize {
red = ansi.CSI + ansi.FG_RED + ansi.SGR
faint = ansi.CSI + ansi.FAINT + ansi.SGR
reset = ansi.CSI + ansi.RESET + ansi.SGR
}
// Header
strings.write_string(&sb, msg)
strings.write_byte(&sb, '\n')
// Location line
strings.write_string(&sb, faint)
for _ in 0 ..< width {
strings.write_byte(&sb, ' ')
}
strings.write_string(&sb, "--> ")
strings.write_string(&sb, reset)
fmt.sbprintf(&sb, "%s:%d:%d\n", path, line, col)
// Top gutter
write_gutter(&sb, width, faint, reset)
// Context lines
for n in start_line ..= end_line {
num_str := fmt.tprintf("%d", n)
strings.write_string(&sb, faint)
for _ in 0 ..< width - len(num_str) {
strings.write_byte(&sb, ' ')
}
strings.write_string(&sb, num_str)
strings.write_string(&sb, " | ")
strings.write_string(&sb, reset)
strings.write_string(&sb, line_text(source, n))
strings.write_byte(&sb, '\n')
// Caret row after error line
if n == line {
strings.write_string(&sb, faint)
for _ in 0 ..< width + 1 {
strings.write_byte(&sb, ' ')
}
strings.write_string(&sb, "| ")
strings.write_string(&sb, reset)
for _ in 1 ..< caret_start_col {
strings.write_byte(&sb, ' ')
}
if colorize {
strings.write_string(&sb, red)
}
for _ in 0 ..< caret_end_col - caret_start_col {
strings.write_byte(&sb, '^')
}
if colorize {
strings.write_string(&sb, reset)
}
if hint != "" {
strings.write_byte(&sb, ' ')
if colorize {
strings.write_string(&sb, faint)
}
strings.write_string(&sb, hint)
if colorize {
strings.write_string(&sb, reset)
}
}
strings.write_byte(&sb, '\n')
}
}
// Trailing gutter
write_gutter(&sb, width, faint, reset)
return strings.to_string(sb)
}
// write_gutter emits a faint pipe-only gutter line: `<width+1 spaces> |`.
write_gutter :: proc(sb: ^strings.Builder, width: int, faint: string, reset: string) {
strings.write_string(sb, faint)
for _ in 0 ..< width + 1 {
strings.write_byte(sb, ' ')
}
strings.write_string(sb, "|")
strings.write_string(sb, reset)
strings.write_byte(sb, '\n')
}
diagnostics/diagnostics_test.odin
-111
View File
@@ -1,111 +0,0 @@
#+test
package diagnostics
import "core:fmt"
import "core:testing"
@(test)
test_suggest_correction_exact :: proc(t: ^testing.T) {
available := []string{"title", "page.title", "body"}
testing.expect_value(t, suggest_correction(available, "page_titel"), "page.title")
}
@(test)
test_suggest_correction_close :: proc(t: ^testing.T) {
available := []string{"title", "body", "now"}
testing.expect_value(t, suggest_correction(available, "titel"), "title")
}
@(test)
test_suggest_correction_no_match :: proc(t: ^testing.T) {
available := []string{"completely_different", "unrelated"}
testing.expect_value(t, suggest_correction(available, "page_titel"), "")
}
@(test)
test_suggest_correction_empty :: proc(t: ^testing.T) {
testing.expect_value(t, suggest_correction([]string{}, "anything"), "")
testing.expect_value(t, suggest_correction([]string{"a"}, ""), "")
}
// ---------------------------------------------------------------------------
// line_col / line_text / count_lines / digit_count — primitive helpers
// ---------------------------------------------------------------------------
@(test)
test_line_col_basic :: proc(t: ^testing.T) {
src := "abc\ndef\nghi"
cases := [?]struct {
pos: int,
line: int,
col: int,
}{{0, 1, 1}, {2, 1, 3}, {3, 1, 4}, {4, 2, 1}, {6, 2, 3}, {7, 2, 4}, {8, 3, 1}}
for c in cases {
l, col := line_col(src, c.pos)
testing.expect(t, l == c.line, fmt.tprintf("pos %d: line %d, want %d", c.pos, l, c.line))
testing.expect(t, col == c.col, fmt.tprintf("pos %d: col %d, want %d", c.pos, col, c.col))
}
}
@(test)
test_line_col_empty :: proc(t: ^testing.T) {
l, col := line_col("", 0)
testing.expect_value(t, l, 1)
testing.expect_value(t, col, 1)
}
@(test)
test_line_col_negative :: proc(t: ^testing.T) {
l, col := line_col("abc", -1)
testing.expect_value(t, l, 1)
testing.expect_value(t, col, 1)
}
@(test)
test_line_col_past_end :: proc(t: ^testing.T) {
l, col := line_col("abc", 100)
testing.expect_value(t, l, 1)
testing.expect_value(t, col, 4)
}
@(test)
test_line_text_first :: proc(t: ^testing.T) {
src := "first\nsecond\nthird"
testing.expect_value(t, line_text(src, 1), "first")
testing.expect_value(t, line_text(src, 2), "second")
testing.expect_value(t, line_text(src, 3), "third")
}
@(test)
test_line_text_trailing_newline :: proc(t: ^testing.T) {
src := "first\nsecond\n"
testing.expect_value(t, line_text(src, 1), "first")
testing.expect_value(t, line_text(src, 2), "second")
testing.expect_value(t, line_text(src, 3), "")
}
@(test)
test_line_text_out_of_range :: proc(t: ^testing.T) {
testing.expect_value(t, line_text("abc", 5), "")
testing.expect_value(t, line_text("abc", 0), "")
}
@(test)
test_count_lines :: proc(t: ^testing.T) {
testing.expect_value(t, count_lines(""), 1)
testing.expect_value(t, count_lines("abc"), 1)
testing.expect_value(t, count_lines("a\nb"), 2)
testing.expect_value(t, count_lines("a\nb\n"), 2)
testing.expect_value(t, count_lines("a\nb\nc"), 3)
}
@(test)
test_digit_count :: proc(t: ^testing.T) {
testing.expect_value(t, digit_count(0), 1)
testing.expect_value(t, digit_count(1), 1)
testing.expect_value(t, digit_count(9), 1)
testing.expect_value(t, digit_count(10), 2)
testing.expect_value(t, digit_count(99), 2)
testing.expect_value(t, digit_count(100), 3)
testing.expect_value(t, digit_count(-5), 1)
}
feed.odin
+10 -55
View File
@@ -1,6 +1,5 @@
package main
import "base:runtime"
import "core:fmt"
import "core:strings"
import "core:time"
@@ -111,7 +110,8 @@ generate_sitemap :: proc(site: ^Site) -> string {
return strings.to_string(sb)
}
format_rfc822 :: proc(iso: string, allocator := context.temp_allocator) -> string {
// TODO: Leaks
format_rfc822 :: proc(iso: string) -> string {
if len(iso) < 19 {
// TODO: should indicate error somehow
return iso
@@ -124,66 +124,21 @@ format_rfc822 :: proc(iso: string, allocator := context.temp_allocator) -> strin
buf: [8]byte
t := time.to_string_hms(date, buf[:])
// RFC-822 zone: sign followed by two-digit hours and minutes (e.g. +0000,
// +0530, -0800). offset is in minutes.
sign := '+'
off := offset
if off < 0 {
sign = '-'
off = -off
}
return fmt.aprintf(
"%s, %02d %s %d %s %c%02d%02d",
"%s, %02d %s %d %s %3d%2d",
weekday[:3],
time.day(date),
month[:3],
time.year(date),
t,
sign,
off / 60,
off % 60,
allocator = allocator,
offset / 60,
offset % 60,
)
}
// xml_escape escapes the XML metacharacters '&', '<' and '>' in s. Only these
// ASCII characters are touched; all other bytes (including UTF-8 multibyte
// sequences) pass through verbatim, so s must already be valid UTF-8.
//
// The result is allocated in `allocator`, which defaults to
// context.temp_allocator — the returned string is only valid until the next
// temp allocator reset, so callers must not retain it across frames.
xml_escape :: proc(
s: string,
allocator := context.temp_allocator,
) -> (
escaped: string,
err: runtime.Allocator_Error,
) #optional_allocator_error {
sb := strings.builder_make_len_cap(0, len(s), allocator) or_return
// Single pass over the bytes, copying runs of ordinary text in bulk and
// emitting an entity for each metacharacter. The switch is both the
// membership test and the replacement lookup, so no per-match scan.
start := 0
for i := 0; i < len(s); i += 1 {
esc: string
switch s[i] {
case '&':
esc = "&amp;"
case '<':
esc = "&lt;"
case '>':
esc = "&gt;"
case:
continue // ordinary byte: copied as part of the next run
xml_escape :: proc(s: string) -> string {
r, _ := strings.replace_all(s, "&", "&amp;")
r, _ = strings.replace_all(r, "<", "&lt;")
r, _ = strings.replace_all(r, ">", "&gt;")
return r
}
strings.write_string(&sb, s[start:i]) // run before the metacharacter
strings.write_string(&sb, esc)
start = i + 1
}
strings.write_string(&sb, s[start:]) // tail
return strings.to_string(sb), nil
}
feed_test.odin
-166
View File
@@ -1,166 +0,0 @@
#+test
package main
import "core:strings"
import "core:testing"
// ---------------------------------------------------------------------------
// xml_escape
// ---------------------------------------------------------------------------
@(test)
test_xml_escape_passthrough :: proc(t: ^testing.T) {
testing.expect_value(t, xml_escape(""), "")
testing.expect_value(t, xml_escape("hello world"), "hello world")
// UTF-8 multibyte bytes are not metacharacters and pass through untouched.
testing.expect_value(t, xml_escape("héllo — 世界"), "héllo — 世界")
}
@(test)
test_xml_escape_entities :: proc(t: ^testing.T) {
testing.expect_value(t, xml_escape("&"), "&amp;")
testing.expect_value(t, xml_escape("<"), "&lt;")
testing.expect_value(t, xml_escape(">"), "&gt;")
// Adjacent specials, and specials at the very start/end of the string.
testing.expect_value(t, xml_escape("&<>"), "&amp;&lt;&gt;")
testing.expect_value(t, xml_escape("<a>"), "&lt;a&gt;")
}
@(test)
test_xml_escape_mixed :: proc(t: ^testing.T) {
testing.expect_value(t, xml_escape("a<b>c&d"), "a&lt;b&gt;c&amp;d")
testing.expect_value(
t,
xml_escape("<p>Tom & Jerry</p>"),
"&lt;p&gt;Tom &amp; Jerry&lt;/p&gt;",
)
}
// ---------------------------------------------------------------------------
// format_rfc822
// ---------------------------------------------------------------------------
@(test)
test_format_rfc822_short_passthrough :: proc(t: ^testing.T) {
// Inputs shorter than a full ISO-8601 timestamp are returned unchanged
// (and are the input slice itself, so they must not be deleted).
testing.expect_value(t, format_rfc822("2024"), "2024")
testing.expect_value(t, format_rfc822(""), "")
}
@(test)
test_format_rfc822_full_date :: proc(t: ^testing.T) {
// Results are temp-allocated (reset by the test runner), so no delete.
// 2024-01-15 is a Monday; wall-clock time is preserved and the zone is
// rendered as an RFC-822 ±HHMM offset.
testing.expect_value(
t,
format_rfc822("2024-01-15T10:30:00Z"),
"Mon, 15 Jan 2024 10:30:00 +0000",
)
testing.expect_value(
t,
format_rfc822("2024-01-15T10:30:00+05:30"),
"Mon, 15 Jan 2024 10:30:00 +0530",
)
testing.expect_value(
t,
format_rfc822("2024-01-15T10:30:00-08:00"),
"Mon, 15 Jan 2024 10:30:00 -0800",
)
}
// ---------------------------------------------------------------------------
// generate_rss
// ---------------------------------------------------------------------------
@(test)
test_generate_rss_structure :: proc(t: ^testing.T) {
site: Site
site.title = "My & Site"
site.description = "Desc <here>"
site.base_url = "https://example.com"
site.pages = make(#soa[dynamic]Page)
defer delete_soa(site.pages)
// A real content page.
append(
&site.pages,
Page {
section = "posts",
title = "First <Post>",
url = "https://example.com/posts/first/",
content = "<p>Hello & welcome</p>",
},
)
// The home index page (section == "" && _is_index) must be skipped.
append(
&site.pages,
Page {
section = "",
_is_index = true,
title = "HOME_SHOULD_BE_SKIPPED",
url = "https://example.com/",
content = "home",
},
)
// A second content page.
append(
&site.pages,
Page {
section = "posts",
title = "Second Post",
url = "https://example.com/posts/second/",
content = "plain body",
},
)
rss := generate_rss(&site)
defer delete(rss)
// Channel header: site title/description are XML-escaped, link gets a
// trailing slash, and the atom self-link points at index.xml.
testing.expect(t, strings.contains(rss, "<title>My &amp; Site</title>"), rss)
testing.expect(
t,
strings.contains(rss, "<description>Desc &lt;here&gt;</description>"),
rss,
)
testing.expect(t, strings.contains(rss, "<link>https://example.com/</link>"), rss)
testing.expect(
t,
strings.contains(rss, `<atom:link href="https://example.com/index.xml"`),
rss,
)
// Exactly two <item>s — the home index page is skipped.
testing.expect_value(t, strings.count(rss, "<item>"), 2)
testing.expect(t, !strings.contains(rss, "HOME_SHOULD_BE_SKIPPED"), rss)
// First item: title and body are XML-escaped, link is verbatim.
testing.expect(t, strings.contains(rss, "<title>First &lt;Post&gt;</title>"), rss)
testing.expect(
t,
strings.contains(
rss,
"<description>&lt;p&gt;Hello &amp; welcome&lt;/p&gt;</description>",
),
rss,
)
testing.expect(
t,
strings.contains(rss, "<link>https://example.com/posts/first/</link>"),
rss,
)
// Undated pages get the sentinel pubDate (no date formatting/allocation).
testing.expect(
t,
strings.contains(rss, "<pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate>"),
rss,
)
// Well-formed at the boundaries.
testing.expect(t, strings.has_prefix(rss, `<?xml version="1.0"`), rss)
testing.expect(t, strings.has_suffix(rss, "</channel>\n</rss>"), rss)
}
flake.nix
+1 -15
View File
@@ -141,20 +141,13 @@
buildPhase = ''
runHook preBuild
odin build . -o:speed -no-bounds-check \
-define:THOR_DEFAULTS_PATH=$out/share/thor/defaults \
-out:${pname}-keep
odin build . -o:speed -no-bounds-check -out:${pname}-keep
runHook postBuild
'';
installPhase = ''
runHook preInstall
install -Dm755 ${pname}-keep $out/bin/${pname}
# Ship the built-in default layouts alongside the binary. The
# binary reads these at runtime from THOR_DEFAULTS_PATH (baked in
# above). Keep in sync with defaults.odin.
mkdir -p $out/share/thor
cp -r defaults $out/share/thor/defaults
runHook postInstall
'';
};
@@ -167,15 +160,8 @@
gdb
perf
bloaty # du for binaries?
# temp
hugo
pandoc
# IDE
unstable.helix
claude-code
typescript-language-server
vscode-langservers-extracted
]);
markdown/deflists.odin
+116 -243
View File
@@ -4,288 +4,140 @@ import cm "vendor:commonmark"
import "core:strings"
// DefList_Entry represents a single term-definition pair in a definition list.
DefList_Entry :: struct {
term: string,
definition: string,
}
// convert_deflists scans markdown text for definition list patterns and
// converts them to <dl><dt><dd> HTML blocks before cmark processing.
//
// A definition line starts with fewer than four columns of leading whitespace
// followed by a colon and a space (or end-of-line). The term is the nearest
// preceding non-blank line (immediately or within one blank line). Consecutive
// term+definition pairs are grouped into a single <dl> block, and consecutive
// definition lines under one term become multiple <dd> elements.
// A definition line starts with optional whitespace followed by a colon and
// a space. The term is the nearest preceding non-blank line (immediately or
// within one blank line). Consecutive term+definition pairs are grouped into
// a single <dl> block.
//
// The scanner walks the body one line at a time using byte offsets: it never
// splits the input, allocates nothing per line, and copies non-deflist text
// through verbatim. Matching is disabled inside fenced code blocks. Terms and
// definitions are rendered through cmark individually so that inline markdown
// (code, links, emphasis) is processed.
// Line_Kind classifies a line for convert_deflists's group-finding scan. Other
// covers fenced-code delimiters and their contents, where definition syntax is
// inert.
Line_Kind :: enum {
None,
Blank,
Term,
Def,
Other,
}
// Terms and definitions are rendered through cmark individually so that
// inline markdown (code, links, emphasis) is processed.
convert_deflists :: proc(body: string, allocator := context.allocator) -> string {
// Fast path: a body with no definition line at all cannot contain a
// definition list, so return it untouched — skipping the builder allocation
// and the classifying scan below. This is the common case (most pages have
// no definition lists), and the guard scan is leaner than the full loop.
if !has_def_line(body) {
return body
}
lines := strings.split(body, "\n", allocator = context.temp_allocator)
sb := strings.builder_make(context.temp_allocator)
// emitted_up_to is the byte offset up to which `body` has been copied into
// `sb`. It stays at 0 until the first deflist is emitted; if it is still 0
// at the end, nothing matched and we can return the body unchanged.
emitted_up_to := 0
first := true
need_blank := false
in_fence := false
fence_marker: u8 = 0
// The scan is definition-driven: it walks line by line, remembering the two
// previous lines' kinds and offsets, and only does group-finding work when it
// reaches a (rare) definition line — whose term is the immediately preceding
// line, or the one before a single blank line. This avoids a per-line forward
// lookahead: each line is classified once, with no rescanning.
prev1_kind, prev2_kind := Line_Kind.None, Line_Kind.None
prev1_pos, prev2_pos := 0, 0
pos := 0
for {
line, next, ok := line_at(body, pos)
if !ok {
break
i := 0
for i < len(lines) {
entries, matched, next := try_match_deflist(lines, i)
if matched {
html := render_deflist(entries)
if !first {
strings.write_string(&sb, "\n\n")
}
// Classify the line. Definition-list syntax is inert inside fenced code
// blocks, so fence delimiters and their contents are classified Other.
kind: Line_Kind
if in_fence {
if is_fence_line(line, fence_marker) {
in_fence = false
}
kind = .Other
} else if marker, opened := fence_open(line); opened {
in_fence = true
fence_marker = marker
kind = .Other
} else if is_def_line(line) {
kind = .Def
} else if is_blank_line(line) {
kind = .Blank
} else {
kind = .Term
}
// A definition-list group begins at the term preceding this def line
// (immediately, or across a single blank line). Flush pending passthrough
// text verbatim, then render the whole group directly into the builder.
if kind == .Def {
term_pos := -1
if prev1_kind == .Term {
term_pos = prev1_pos
} else if prev1_kind == .Blank && prev2_kind == .Term {
term_pos = prev2_pos
}
if term_pos >= 0 {
strings.write_string(&sb, body[emitted_up_to:term_pos])
group_end := write_deflist_group(&sb, body, term_pos)
// write_deflist_group emits a raw <dl> HTML block with no
// trailing newline, so the blank line the author left after the
// list is consumed as the </dl> line's terminator rather than a
// separating blank line. CommonMark (type-6 HTML block) then
// swallows the following block — e.g. a fenced code example —
// as raw, unescaped HTML, leaking its markup (a template's
// <h1>) into the document. Re-emit a newline so </dl> stands on
// its own line and the author's blank line still separates the
// block. Only when a blank line actually follows: fused
// non-blank fall-through content is left as-is.
if line, _, ok := line_at(body, group_end); ok && is_blank_line(line) {
strings.write_byte(&sb, '\n')
}
emitted_up_to = group_end
pos = group_end
prev1_kind, prev2_kind = .None, .None
in_fence = false
strings.write_string(&sb, html)
first = false
need_blank = true
i = next
continue
}
if need_blank {
strings.write_string(&sb, "\n\n")
need_blank = false
} else if !first {
strings.write_string(&sb, "\n")
}
strings.write_string(&sb, lines[i])
first = false
i += 1
}
prev2_kind, prev2_pos = prev1_kind, prev1_pos
prev1_kind, prev1_pos = kind, pos
pos = next
}
if emitted_up_to == 0 {
// No definition list anywhere; the body passes through unchanged.
return body
}
strings.write_string(&sb, body[emitted_up_to:])
return strings.clone(strings.to_string(sb), allocator)
}
// line_at returns the line beginning at byte offset `pos` (excluding its
// trailing newline) along with the offset of the following line. `pos` must be
// 0 or one past a '\n'. ok is false once `pos` reaches the end of `body`.
line_at :: proc(body: string, pos: int) -> (line: string, next: int, ok: bool) {
if pos >= len(body) {
return "", pos, false
}
nl := strings.index_byte(body[pos:], '\n')
if nl < 0 {
return body[pos:], len(body), true
}
return body[pos:pos + nl], pos + nl + 1, true
}
// try_match_deflist attempts to match a definition list group starting at
// lines[start]. A group is one or more term+definition pairs. Returns the
// matched entries, whether a match was found, and the index past the group.
try_match_deflist :: proc(
lines: []string,
start: int,
) -> (
entries: [dynamic]DefList_Entry,
ok: bool,
end: int,
) {
entries = make([dynamic]DefList_Entry, 0, allocator = context.temp_allocator)
end = start
// has_def_line reports whether any line in `body` is a definition line (see
// is_def_line). It is a cheap guard for convert_deflists: a body with no
// definition line cannot contain a definition list. There are no false
// negatives (every definition list requires a definition line); a false
// positive — e.g. a colon line that only appears inside a code fence — simply
// falls through to the full scan, which then finds no match and returns `body`.
has_def_line :: proc(body: string) -> bool {
pos := 0
for {
line, next := line_at(body, pos) or_break
if is_def_line(line) {
return true
}
pos = next
}
return false
}
// write_deflist_group renders the definition-list group beginning at `start`
// directly into `sb`, wrapped in <dl>...</dl>, and returns the byte offset one
// past the group (the start of the first line not consumed). The caller must
// have confirmed via term_has_def that a group starts at `start`, so the group
// always contains at least one term+definition pair. A trailing blank line that
// separates the group from following content is left unconsumed.
write_deflist_group :: proc(sb: ^strings.Builder, body: string, start: int) -> (end: int) {
strings.write_string(sb, "<dl>")
pos := start
for {
term_line, term_next, term_ok := line_at(body, pos)
if !term_ok || is_blank_line(term_line) || is_def_line(term_line) {
i := start
for i < len(lines) {
// A term must be non-blank and not itself a def line
if is_blank_line(lines[i]) || is_def_line(lines[i]) {
break
}
// The definition follows immediately or across a single blank line.
def_pos := term_next
def_line, def_next, def_ok := line_at(body, def_pos)
if def_ok && is_blank_line(def_line) {
def_pos = def_next
def_line, def_next, def_ok = line_at(body, def_pos)
// Look for a def line: immediately after or with one blank line
def_idx := i + 1
if def_idx < len(lines) && is_blank_line(lines[def_idx]) {
def_idx += 1
}
if !def_ok || !is_def_line(def_line) {
if def_idx >= len(lines) || !is_def_line(lines[def_idx]) {
break
}
// Emit the term with its first definition, then any consecutive
// definition lines as additional <dd>s under the same term.
strings.write_string(sb, "<dt>")
strings.write_string(sb, render_inline_md(strings.trim_space(term_line)))
strings.write_string(sb, "</dt><dd>")
strings.write_string(sb, render_inline_md(def_content(def_line)))
strings.write_string(sb, "</dd>")
pos = def_next
for {
cont, cont_next, cont_ok := line_at(body, pos)
if !cont_ok || !is_def_line(cont) {
break
}
strings.write_string(sb, "<dd>")
strings.write_string(sb, render_inline_md(def_content(cont)))
strings.write_string(sb, "</dd>")
pos = cont_next
}
// Found a term + def pair
append(
&entries,
DefList_Entry {
term = strings.trim_space(lines[i]),
definition = def_content(lines[def_idx]),
},
)
i = def_idx + 1
// Continue the group if another term+definition pair follows,
// optionally separated by one blank line. Otherwise stop here without
// consuming a trailing blank line.
blank_line, after_blank, blank_ok := line_at(body, pos)
if blank_ok && is_blank_line(blank_line) {
if term_has_def(body, after_blank) {
pos = after_blank
// After a pair, check if another pair follows (optionally
// separated by one blank line). If so, continue the group.
// If not, break without consuming the blank line.
if i < len(lines) && is_blank_line(lines[i]) {
after_blank := i + 1
if after_blank < len(lines) &&
!is_blank_line(lines[after_blank]) &&
!is_def_line(lines[after_blank]) {
// Check whether a def follows this potential term
check_def := after_blank + 1
if check_def < len(lines) && is_blank_line(lines[check_def]) {
check_def += 1
}
if check_def < len(lines) && is_def_line(lines[check_def]) {
i = after_blank
continue
}
}
break
}
// No blank line: loop and let the top re-test this line as a term.
}
strings.write_string(sb, "</dl>")
return pos
if len(entries) > 0 {
ok = true
end = i
}
return
}
// term_has_def reports whether the line at `pos` is a term that is immediately
// (or across one blank line) followed by a definition line.
term_has_def :: proc(body: string, pos: int) -> bool {
line, next, ok := line_at(body, pos)
if !ok || is_blank_line(line) || is_def_line(line) {
return false
}
def_pos := next
def_line, def_next, def_ok := line_at(body, def_pos)
if def_ok && is_blank_line(def_line) {
def_pos = def_next
def_line, def_next, def_ok = line_at(body, def_pos)
}
return def_ok && is_def_line(def_line)
}
// fence_open reports whether `line` opens a fenced code block and, if so, which
// marker character it uses. Leading spaces are ignored.
fence_open :: proc(line: string) -> (marker: u8, ok: bool) {
trimmed := strings.trim_left(line, " ")
if strings.has_prefix(trimmed, "```") {
return '`', true
}
if strings.has_prefix(trimmed, "~~~") {
return '~', true
}
return 0, false
}
// is_fence_line reports whether `line` is a fence using the given marker
// character (used to detect the closing fence of an open block).
is_fence_line :: proc(line: string, marker: u8) -> bool {
trimmed := strings.trim_left(line, " ")
if marker == '~' {
return strings.has_prefix(trimmed, "~~~")
}
return strings.has_prefix(trimmed, "```")
}
// is_def_line returns true if the line is a definition line: fewer than four
// columns of leading whitespace, a colon, then whitespace or end-of-line. Four
// or more columns of indentation is a code block, not a definition.
// is_def_line returns true if the line is a definition line:
// optional leading whitespace, a colon, then whitespace or end-of-line.
is_def_line :: proc(line: string) -> bool {
indent := 0
n := 0
for n < len(line) && (line[n] == ' ' || line[n] == '\t') {
indent += line[n] == '\t' ? 4 : 1
n += 1
}
if indent >= 4 {
trimmed := strings.trim_left(line, " \t")
if len(trimmed) < 1 || trimmed[0] != ':' {
return false
}
rest := line[n:]
if len(rest) < 1 || rest[0] != ':' {
return false
}
if len(rest) == 1 {
if len(trimmed) == 1 {
return true
}
return rest[1] == ' ' || rest[1] == '\t'
return trimmed[1] == ' ' || trimmed[1] == '\t'
}
// def_content extracts the definition text from a definition line,
@@ -302,6 +154,27 @@ is_blank_line :: proc(line: string) -> bool {
return strings.trim_space(line) == ""
}
// render_deflist builds the <dl> HTML block from a list of entries.
// Each term and definition is rendered through cmark to process inline
// markdown. Result lives in context.temp_allocator.
render_deflist :: proc(entries: [dynamic]DefList_Entry) -> string {
sb := strings.builder_make(context.temp_allocator)
strings.write_string(&sb, "<dl>")
for entry in entries {
term_html := render_inline_md(entry.term)
def_html := render_inline_md(entry.definition)
strings.write_string(&sb, "<dt>")
strings.write_string(&sb, term_html)
strings.write_string(&sb, "</dt><dd>")
strings.write_string(&sb, def_html)
strings.write_string(&sb, "</dd>")
}
strings.write_string(&sb, "</dl>")
return strings.to_string(sb)
}
// render_inline_md renders a snippet of markdown through cmark and strips
// the surrounding <p> tags. Result lives in context.temp_allocator.
render_inline_md :: proc(text: string) -> string {
markdown/deflists_fixtures_test.odin
-92
View File
@@ -1,92 +0,0 @@
#+test
package markdown
import "core:testing"
// Fixture-driven tests for convert_deflists. Every fixture is exact-matched
// against its oracle in testdata/deflists/expected/. See
// testdata/deflists/reference/README.md for how each expected file was decided
// (cross-checked against Pandoc and Hugo/Goldmark).
//
// The fixture inputs are the definition-list examples from
// https://www.markdownlang.com/extended/definition-lists.html
//
// Provenance of each expected file:
// 01,02,08,09 Pandoc and Hugo agree; oracle = that agreed HTML.
// 06 Both agree it is NOT a deflist; convert_deflists passes the
// markdown through unchanged, so expected = the raw input.
// 07 CommonMark's <4-space marker rule: we accept the 2-space colon
// (Pandoc does; Hugo does not).
// 05 Block nesting is unsupported; our flatten happens to match
// Pandoc's output exactly.
// 03,04 Block content in definitions is unsupported; expected captures
// our current degraded behavior (a <dl> for the term/first line,
// the rest falling through as markdown) as a regression guard.
Deflist_Fixture :: struct {
name: string,
input: string,
expected: string,
}
@(test)
test_deflist_fixtures :: proc(t: ^testing.T) {
cases := []Deflist_Fixture {
{
"01-basic",
#load(#directory + "testdata/deflists/01-basic.md", string),
#load(#directory + "testdata/deflists/expected/01-basic.html", string),
},
{
"02-multiple-defs",
#load(#directory + "testdata/deflists/02-multiple-defs.md", string),
#load(#directory + "testdata/deflists/expected/02-multiple-defs.html", string),
},
{
"03-multiline-def",
#load(#directory + "testdata/deflists/03-multiline-def.md", string),
#load(#directory + "testdata/deflists/expected/03-multiline-def.html", string),
},
{
"04-markdown-in-def",
#load(#directory + "testdata/deflists/04-markdown-in-def.md", string),
#load(#directory + "testdata/deflists/expected/04-markdown-in-def.html", string),
},
{
"05-nested",
#load(#directory + "testdata/deflists/05-nested.md", string),
#load(#directory + "testdata/deflists/expected/05-nested.html", string),
},
{
"06-compact",
#load(#directory + "testdata/deflists/06-compact.md", string),
#load(#directory + "testdata/deflists/expected/06-compact.html", string),
},
{
"07-extra-space",
#load(#directory + "testdata/deflists/07-extra-space.md", string),
#load(#directory + "testdata/deflists/expected/07-extra-space.html", string),
},
{
"08-glossary",
#load(#directory + "testdata/deflists/08-glossary.md", string),
#load(#directory + "testdata/deflists/expected/08-glossary.html", string),
},
{
"09-api-params",
#load(#directory + "testdata/deflists/09-api-params.md", string),
#load(#directory + "testdata/deflists/expected/09-api-params.html", string),
},
}
for c in cases {
result := convert_deflists(c.input, context.temp_allocator)
testing.expectf(
t,
result == c.expected,
"[%s]\n expected: %q\n got: %q",
c.name,
c.expected,
result,
)
}
}
markdown/deflists_test.odin
-11
View File
@@ -90,17 +90,6 @@ test_empty_body :: proc(t: ^testing.T) {
testing.expect_value(t, result, "")
}
// Regression: a fenced code block following a definition list must stay a
// separate block. Without a blank line after the emitted </dl>, CommonMark
// treats the fence as raw HTML continuation of the <dl> block and passes its
// contents through unescaped (leaking e.g. an example <h1> into the document).
@(test)
test_deflist_before_code_fence :: proc(t: ^testing.T) {
input := "term\n\n: def\n\n```\n<h1>example</h1>\n```"
result := convert_deflists(input, context.temp_allocator)
testing.expect(t, strings.contains(result, "</dl>\n\n```"))
}
@(test)
test_docs_md_pattern :: proc(t: ^testing.T) {
input := "content\n\n: `content` holds your pages.\n\n assets\n : `assets` contains files."
markdown/highlight.odin
+9 -19
View File
@@ -90,12 +90,9 @@ write_escaped :: proc(b: ^strings.Builder, s: string) {
}
}
// unescape_html returns a caller-owned string (free it with delete). When the
// input has no entities it returns a clone; otherwise it returns the builder's
// buffer. It must NOT destroy the builder — to_string aliases that buffer, so the
// returned string owns it.
unescape_html :: proc(s: string) -> string {
sb := strings.builder_make()
defer strings.builder_destroy(&sb)
start := 0
for i in 0 ..< len(s) {
@@ -122,28 +119,22 @@ unescape_html :: proc(s: string) -> string {
strings.write_string(&sb, replacement)
start = i + semi + 1
}
if start == 0 do return strings.clone(s)
if start == 0 do return s
if start < len(s) do strings.write_string(&sb, s[start:])
return strings.to_string(sb)
}
highlight_block :: proc(code: string, lang: string, file_path: string) -> string {
g, ok := ts.grammar(lang)
if !ok {
gc := ts.load_grammar(lang)
if gc == nil {
return code
}
parser := ts.open_parser(g)
if parser == nil {
return code
}
defer ts.parser_delete(parser)
raw_code := unescape_html(code)
defer delete(raw_code)
raw_c := strings.clone_to_cstring(raw_code)
defer delete(raw_c)
tree := ts.parser_parse_string(parser, nil, raw_c, u32(len(raw_code)))
tree := ts.parser_parse_string(gc.parser, nil, raw_c, u32(len(raw_code)))
if tree == nil {
return code
}
@@ -165,13 +156,12 @@ highlight_block :: proc(code: string, lang: string, file_path: string) -> string
}
}
if g.query == nil {
cursor := gc.cursor
if cursor == nil {
return code
}
cursor := ts.query_cursor_new()
defer ts.query_cursor_delete(cursor)
ts.query_cursor_exec(cursor, g.query, root)
ts.query_cursor_exec(cursor, gc.query, root)
captures := make([dynamic]Capture, 0, 64, context.temp_allocator)
defer delete(captures)
@@ -184,7 +174,7 @@ highlight_block :: proc(code: string, lang: string, file_path: string) -> string
}
cap := match.captures[capture_idx]
name_len: u32
name_c := ts.query_capture_name_for_id(g.query, cap.index, &name_len)
name_c := ts.query_capture_name_for_id(gc.query, cap.index, &name_len)
if name_c == nil {
continue
}
markdown/highlight_test.odin
-103
View File
@@ -1,103 +0,0 @@
#+test
package markdown
import ts "../treesitter"
import "core:mem"
import "core:strings"
import "core:testing"
// highlight_block returns the input string unchanged (borrowed) when it can't
// highlight — no grammar, no query, or zero captures — and a freshly-allocated
// string when it does. Free only the latter; identity (raw_data) tells them
// apart regardless of content.
@(private = "file")
free_highlight :: proc(out, code: string) {
if raw_data(out) != raw_data(code) {
delete(out)
}
}
// A builtin grammar (css) highlights into hl- spans, preserving the source text.
@(test)
test_highlight_block_css :: proc(t: ^testing.T) {
code := "a{color:red}"
out := highlight_block(code, "css", "test")
defer free_highlight(out, code)
testing.expect(t, strings.contains(out, "<span class=\"hl-"), "css should be highlighted")
testing.expect(t, strings.contains(out, "</span>"))
testing.expect(t, strings.contains(out, "color"))
testing.expect(t, strings.contains(out, "red"))
}
// The other builtin grammar (html) highlights, and the source's own '<'/'>' are
// HTML-escaped in the output (so they can't be confused with the wrapping spans).
@(test)
test_highlight_block_html_escapes_source :: proc(t: ^testing.T) {
code := "<p>hi</p>"
out := highlight_block(code, "html", "test")
defer free_highlight(out, code)
testing.expect(t, strings.contains(out, "<span class=\"hl-"), "html should be highlighted")
testing.expect(t, strings.contains(out, "hi"))
// The literal angle brackets of the source are escaped in the output.
testing.expect(t, strings.contains(out, "&lt;"))
testing.expect(t, strings.contains(out, "&gt;"))
}
// A language with no available grammar returns the input verbatim (best-effort:
// highlighting is optional). The returned string must be the same buffer, so
// free_highlight leaves it alone.
@(test)
test_highlight_block_unknown_lang_passthrough :: proc(t: ^testing.T) {
code := "fn main() {}"
out := highlight_block(code, "no-such-language", "test")
defer free_highlight(out, code)
testing.expect_value(t, out, code)
testing.expect(t, raw_data(out) == raw_data(code), "passthrough must return the input buffer")
}
// End-to-end: highlight_code finds a fenced code block in rendered HTML and
// rewrites its body with highlight spans while preserving the <pre><code> shell.
// Run under a scratch arena (as production does) so the intermediate allocations
// highlight_code doesn't individually free are reclaimed in one shot.
@(test)
test_highlight_code_wraps_css_block :: proc(t: ^testing.T) {
arena: mem.Dynamic_Arena
mem.dynamic_arena_init(&arena)
defer mem.dynamic_arena_destroy(&arena)
context.allocator = mem.dynamic_arena_allocator(&arena)
html := `<pre><code class="language-css">a{color:red}</code></pre>`
out := highlight_code(html, "test")
testing.expect(t, strings.contains(out, `<pre><code class="language-css">`))
testing.expect(t, strings.contains(out, "</code></pre>"))
testing.expect(
t,
strings.contains(out, "<span class=\"hl-"),
"block body should be highlighted",
)
}
// unescape_html must return a caller-owned allocation the caller frees exactly
// once. The buggy version frees its builder internally (defer builder_destroy)
// and returns the freed alias, so the caller's delete becomes a double-free that
// the test runner's tracking allocator flags.
@(test)
test_unescape_html_entities_owned :: proc(t: ^testing.T) {
got := unescape_html("&lt;a&gt;")
defer delete(got)
testing.expect_value(t, got, "<a>")
}
// Same contract on the no-entity path: the buggy version returns the borrowed
// input, so freeing it is a bad free (pointer not from the allocator).
@(test)
test_unescape_html_passthrough_owned :: proc(t: ^testing.T) {
got := unescape_html("plain text")
defer delete(got)
testing.expect_value(t, got, "plain text")
}
markdown/markdown.odin
+2 -24
View File
@@ -6,8 +6,6 @@ import "core:encoding/json"
import "core:log"
import "core:strings"
import diags "../diagnostics"
Extension :: enum {
Emoji,
Sidenotes,
@@ -21,11 +19,6 @@ Extension :: enum {
DEFAULT_EXTENSIONS :: bit_set[Extension]{.Emoji, .Sidenotes, .Alerts, .HeadingIDs, .DefLists}
EXTENSION_NAMES :: []string{
"emoji", "sidenotes", "alerts", "highlight", "sections",
"heading_ids", "deflists", "footnotes",
}
// Caller is responsible for freeing string
process :: proc(
body: string,
@@ -76,13 +69,8 @@ parse_extension_list :: proc(s: string) -> (result: bit_set[Extension]) {
e, ok := extension_from_name(name)
if !ok {
if name != "" {
suggestion := diags.suggest_correction(EXTENSION_NAMES, name)
if suggestion != "" {
log.warnf("unknown extension '%s' (did you mean '%s'?)", name, suggestion)
} else {
log.warnf("unknown extension '%s'", name)
}
}
continue
}
result += {e}
@@ -93,19 +81,9 @@ parse_extension_list :: proc(s: string) -> (result: bit_set[Extension]) {
// Given a map[Extension]bool, apply it to ext.
apply_extension_config :: proc(ext: ^bit_set[Extension], config: json.Object) {
for name, val in config {
// TODO: Silently discards invalid values.
enabled := val.(json.Boolean) or_continue
e, ok := extension_from_name(name)
if !ok {
lower := strings.to_lower(name, context.temp_allocator)
suggestion := diags.suggest_correction(EXTENSION_NAMES, lower)
if suggestion != "" {
log.warnf("unknown extension '%s' (did you mean '%s'?)", name, suggestion)
} else {
log.warnf("unknown extension '%s'", name)
}
continue
}
e := extension_from_name(name) or_continue
if enabled {
ext^ += {e}
markdown/testdata/deflists/01-basic.md
Vendored
-2
View File
@@ -1,2 +0,0 @@
Term
: Definition content
markdown/testdata/deflists/02-multiple-defs.md
Vendored
-7
View File
@@ -1,7 +0,0 @@
Markdown
: A lightweight markup language
: Created by John Gruber in 2004
HTML
: A standard markup language used to create web pages
: Used to create web pages
markdown/testdata/deflists/03-multiline-def.md
Vendored
-5
View File
@@ -1,5 +0,0 @@
Term
: This is the first line of definition content
This is the second line, needs at least one space indentation
This is a new paragraph, needs at least one space indentation and a preceding blank line
markdown/testdata/deflists/04-markdown-in-def.md
Vendored
-7
View File
@@ -1,7 +0,0 @@
Markdown Syntax
: **Markdown** supports multiple text formatting:
- *Italic* and **Bold**
- [Link](https://www.markdownlang.com)
- `Inline code`
- > Block quote
- Lists and other elements
markdown/testdata/deflists/05-nested.md
Vendored
-6
View File
@@ -1,6 +0,0 @@
Outer Term
: Outer Definition
Inner Term
: Inner Definition
: Another Inner Definition
markdown/testdata/deflists/06-compact.md
Vendored
-1
View File
@@ -1 +0,0 @@
Term: Definition
markdown/testdata/deflists/07-extra-space.md
Vendored
-2
View File
@@ -1,2 +0,0 @@
Term
: Definition
markdown/testdata/deflists/08-glossary.md
Vendored
-11
View File
@@ -1,11 +0,0 @@
API
: An application programming interface, a set of rules that allow different software applications to communicate with each other.
Framework
: A software library that provides a standardized structure for application development.
Git
: A distributed version control system used to track changes in a project development process.
IDE
: An integrated development environment, a software application that includes a code editor and various development tools.
markdown/testdata/deflists/09-api-params.md
Vendored
-14
View File
@@ -1,14 +0,0 @@
user_id
: **Required** - The unique identifier for the user.
: Type: `integer`
name
: **Optional** - The display name for the user.
: Type: `string`
: Default: `null`
status
: **Optional** - The user status.
: Type: `string`
: Allowed values: `active`, `inactive`, `suspended`
: Default: `active`
markdown/testdata/deflists/expected/01-basic.html
Vendored
-1
View File
@@ -1 +0,0 @@
<dl><dt>Term</dt><dd>Definition content</dd></dl>
markdown/testdata/deflists/expected/02-multiple-defs.html
Vendored
-1
View File
@@ -1 +0,0 @@
<dl><dt>Markdown</dt><dd>A lightweight markup language</dd><dd>Created by John Gruber in 2004</dd><dt>HTML</dt><dd>A standard markup language used to create web pages</dd><dd>Used to create web pages</dd></dl>
markdown/testdata/deflists/expected/03-multiline-def.html
Vendored
-3
View File
@@ -1,3 +0,0 @@
<dl><dt>Term</dt><dd>This is the first line of definition content</dd></dl> This is the second line, needs at least one space indentation
This is a new paragraph, needs at least one space indentation and a preceding blank line
markdown/testdata/deflists/expected/04-markdown-in-def.html
Vendored
-5
View File
@@ -1,5 +0,0 @@
<dl><dt>Markdown Syntax</dt><dd><strong>Markdown</strong> supports multiple text formatting:</dd></dl> - *Italic* and **Bold**
- [Link](https://www.markdownlang.com)
- `Inline code`
- > Block quote
- Lists and other elements
markdown/testdata/deflists/expected/05-nested.html
Vendored
-1
View File
@@ -1 +0,0 @@
<dl><dt>Outer Term</dt><dd>Outer Definition</dd><dt>Inner Term</dt><dd>Inner Definition</dd><dd>Another Inner Definition</dd></dl>
markdown/testdata/deflists/expected/06-compact.html
Vendored
-1
View File
@@ -1 +0,0 @@
Term: Definition
markdown/testdata/deflists/expected/07-extra-space.html
Vendored
-1
View File
@@ -1 +0,0 @@
<dl><dt>Term</dt><dd>Definition</dd></dl>
markdown/testdata/deflists/expected/08-glossary.html
Vendored
-1
View File
@@ -1 +0,0 @@
<dl><dt>API</dt><dd>An application programming interface, a set of rules that allow different software applications to communicate with each other.</dd><dt>Framework</dt><dd>A software library that provides a standardized structure for application development.</dd><dt>Git</dt><dd>A distributed version control system used to track changes in a project development process.</dd><dt>IDE</dt><dd>An integrated development environment, a software application that includes a code editor and various development tools.</dd></dl>
markdown/testdata/deflists/expected/09-api-params.html
Vendored
-1
View File
@@ -1 +0,0 @@
<dl><dt>user_id</dt><dd><strong>Required</strong> - The unique identifier for the user.</dd><dd>Type: <code>integer</code></dd><dt>name</dt><dd><strong>Optional</strong> - The display name for the user.</dd><dd>Type: <code>string</code></dd><dd>Default: <code>null</code></dd><dt>status</dt><dd><strong>Optional</strong> - The user status.</dd><dd>Type: <code>string</code></dd><dd>Allowed values: <code>active</code>, <code>inactive</code>, <code>suspended</code></dd><dd>Default: <code>active</code></dd></dl>
markdown/testdata/deflists/reference/COMPARISON.md
Vendored
-60
View File
@@ -1,60 +0,0 @@
# 3-way behavior comparison: ours vs Pandoc vs Hugo
Captured for later review (e.g. if we ever pursue "Option B" — full CommonMark
block semantics inside definitions). Compares `convert_deflists` output (Option A,
inline/flat) against Pandoc (`-f markdown -t html5`) and Hugo/Goldmark for the
four fixtures where behavior is interesting. Raw renderings live in `pandoc/` and
`hugo/`; ours is in `../expected/`. Whitespace/formatting differences ignored
below — only structural differences are described.
## 03-multiline-def — definition with a continuation line + a second paragraph
*How much multi-line content belongs to the `<dd>`?*
- **Ours:** only the **first line** goes in the `<dd>`. The continuation line and
the second paragraph fall out of the `<dl>` as **raw, unrendered markdown**.
- **Pandoc:** folds the continuation line **into** the `<dd>` (lazy continuation),
then **ejects** the second paragraph as a sibling `<p>` after `</dl>`.
- **Hugo:** keeps **everything inside** the `<dd>` — continuation line as text,
second paragraph as a nested `<p>`.
- All three differ. We capture the least, Pandoc the middle, Hugo the most.
## 04-markdown-in-def — definition containing a bullet list (with a blockquote)
*What happens to the nested list?*
- **Ours:** intro line becomes the `<dd>`; the **entire bullet list falls out as
raw markdown** (unrendered `- …` lines after `</dl>`).
- **Pandoc:** bullets **flattened into the `<dd>`** as inline text — no `<ul>`, the
`-` become literal dashes.
- **Hugo:** bullets become a **real nested `<ul>`/`<li>`** (blockquote and all)
inside the `<dd>`.
- All three differ; same shape as 03. We drop it, Pandoc absorbs-but-flattens,
Hugo builds true structure.
## 05-nested — a definition list nested under another definition
*Nest or flatten?*
- **Ours:** **flatten** — one flat `<dl>`; "Inner Term" is just another `<dt>`.
- **Pandoc:** **flatten** — structurally **identical to ours**.
- **Hugo:** **nests** a real inner `<dl>` inside the outer `<dd>`.
- Ours == Pandoc; Hugo is the odd one out.
## 07-extra-space — term then a 2-space-indented `: Definition`
*Is a 2-space colon a definition marker?*
- **Ours:** **yes** → clean `<dl>`.
- **Pandoc:** **yes** → structurally **identical to ours**.
- **Hugo:** **no** → rejects it, emits a plain `<p>`.
- Ours == Pandoc; Hugo is the odd one out.
## Pattern
Two clean groups:
- **Marker / flattening decisions (05, 07):** we land **exactly on Pandoc**. These
are the calls our CommonMark-marker + flat-inline design makes cleanly; Pandoc
agrees, Hugo diverges (nests / stricter marker).
- **Block-content-in-a-definition (03, 04):** we diverge from **both** tools, doing
**less** — we emit the `<dl>` for the term/first line and let the rest fall
through as raw markdown, rather than absorbing it into the `<dd>` (flattened like
Pandoc, or structured like Hugo).
In short: our behavior is **"Pandoc, minus the ability to pull block content into
a `<dd>`."** Single inline-line definitions (01, 02, 05, 07, 08, 09) match a real
engine; genuinely multi-block definitions (03, 04) stop at the first line and pass
the remainder through. Closing that gap is what Option B would entail.
markdown/testdata/deflists/reference/README.md
Vendored
-44
View File
@@ -1,44 +0,0 @@
# Definition-list test fixtures
Inputs (`../NN-name.md`) are the definition-list examples from
<https://www.markdownlang.com/extended/definition-lists.html>.
`pandoc/` and `hugo/` hold each input rendered by Pandoc (`-f markdown -t html5`)
and Hugo (Goldmark with `markup.goldmark.extensions.definitionList = true`).
They are retained as reference for a possible future "Option B" (full CommonMark
block semantics inside definitions). They are NOT the test oracle for the current
implementation.
`../expected/NN.html` is the oracle for `convert_deflists`: our inline fragment
format (no inter-tag newlines, no trailing newline), derived from the trusted
tool per the decisions below.
## Decisions (Option A — simple / inline)
Marker whitespace follows CommonMark: `:` with 0–3 spaces of indent is a
definition; 4+ (tab = 4) is a code block. Definition bodies are inline/flat.
Every fixture has a concrete exact-match oracle in `../expected/NN.html`.
| Fixture | Pandoc | Hugo | Decision | expected/ source |
|---------|--------|------|----------|------------------|
| 01-basic | `<dl>` | `<dl>` | support | hugo (agreed) |
| 02-multiple-defs | multi `<dd>` | multi `<dd>` | support | hugo (agreed) |
| 06-compact (`Term: Definition`) | `<p>` | `<p>` | NOT a deflist → passthrough | raw input |
| 07-extra-space (2-space `:`) | `<dl>` | `<p>` | support (accept, CommonMark ≤3-space rule) | hand-written |
| 08-glossary | `<dl>` | `<dl>` | support | hugo (agreed) |
| 09-api-params | multi `<dd>` | multi `<dd>` | support | hugo (agreed) |
| 05-nested | flattens | nested `<dl>` | flatten (matches Pandoc) | snapshot (== pandoc) |
| 03-multiline-def | ejects 2nd para | nests 2nd para in `<dd>` | degrade | snapshot (current output) |
| 04-markdown-in-def | flattens list | nested `<ul>`/`<blockquote>` | degrade | snapshot (current output) |
A detailed 3-way behavior comparison (ours vs Pandoc vs Hugo) for the diverging
fixtures is in `COMPARISON.md`, kept for Option-B review.
03/04/05 require block-level content inside `<dd>` (multi-paragraph, nested
lists/blockquotes, nested `<dl>`), which the inline preprocessor does not
produce. 05 flattens exactly as Pandoc does, so its snapshot equals Pandoc's
output. 03/04 emit a `<dl>` for the term and first line, then let the remaining
block content fall through as markdown; their `expected/` files snapshot that
current degraded behavior as a regression guard, not an endorsed rendering.
Revisit via Option B (full CommonMark block semantics) if real content needs it.
markdown/testdata/deflists/reference/hugo/01-basic.html
Vendored
-4
View File
@@ -1,4 +0,0 @@
<dl>
<dt>Term</dt>
<dd>Definition content</dd>
</dl>
markdown/testdata/deflists/reference/hugo/02-multiple-defs.html
Vendored
-8
View File
@@ -1,8 +0,0 @@
<dl>
<dt>Markdown</dt>
<dd>A lightweight markup language</dd>
<dd>Created by John Gruber in 2004</dd>
<dt>HTML</dt>
<dd>A standard markup language used to create web pages</dd>
<dd>Used to create web pages</dd>
</dl>
markdown/testdata/deflists/reference/hugo/03-multiline-def.html
Vendored
-7
View File
@@ -1,7 +0,0 @@
<dl>
<dt>Term</dt>
<dd>This is the first line of definition content
This is the second line, needs at least one space indentation
<p>This is a new paragraph, needs at least one space indentation and a preceding blank line</p>
</dd>
</dl>
markdown/testdata/deflists/reference/hugo/04-markdown-in-def.html
Vendored
-16
View File
@@ -1,16 +0,0 @@
<dl>
<dt>Markdown Syntax</dt>
<dd><strong>Markdown</strong> supports multiple text formatting:
<ul>
<li><em>Italic</em> and <strong>Bold</strong></li>
<li><a href="https://www.markdownlang.com">Link</a></li>
<li><code>Inline code</code></li>
<li>
<blockquote>
<p>Block quote</p>
</blockquote>
</li>
<li>Lists and other elements</li>
</ul>
</dd>
</dl>
markdown/testdata/deflists/reference/hugo/05-nested.html
Vendored
-10
View File
@@ -1,10 +0,0 @@
<dl>
<dt>Outer Term</dt>
<dd>Outer Definition
<dl>
<dt>Inner Term</dt>
<dd>Inner Definition</dd>
<dd>Another Inner Definition</dd>
</dl>
</dd>
</dl>
markdown/testdata/deflists/reference/hugo/06-compact.html
Vendored
-1
View File
@@ -1 +0,0 @@
<p>Term: Definition</p>
markdown/testdata/deflists/reference/hugo/07-extra-space.html
Vendored
-2
View File
@@ -1,2 +0,0 @@
<p>Term
: Definition</p>
markdown/testdata/deflists/reference/hugo/08-glossary.html
Vendored
-10
View File
@@ -1,10 +0,0 @@
<dl>
<dt>API</dt>
<dd>An application programming interface, a set of rules that allow different software applications to communicate with each other.</dd>
<dt>Framework</dt>
<dd>A software library that provides a standardized structure for application development.</dd>
<dt>Git</dt>
<dd>A distributed version control system used to track changes in a project development process.</dd>
<dt>IDE</dt>
<dd>An integrated development environment, a software application that includes a code editor and various development tools.</dd>
</dl>
markdown/testdata/deflists/reference/hugo/09-api-params.html
Vendored
-14
View File
@@ -1,14 +0,0 @@
<dl>
<dt>user_id</dt>
<dd><strong>Required</strong> - The unique identifier for the user.</dd>
<dd>Type: <code>integer</code></dd>
<dt>name</dt>
<dd><strong>Optional</strong> - The display name for the user.</dd>
<dd>Type: <code>string</code></dd>
<dd>Default: <code>null</code></dd>
<dt>status</dt>
<dd><strong>Optional</strong> - The user status.</dd>
<dd>Type: <code>string</code></dd>
<dd>Allowed values: <code>active</code>, <code>inactive</code>, <code>suspended</code></dd>
<dd>Default: <code>active</code></dd>
</dl>
markdown/testdata/deflists/reference/pandoc/01-basic.html
Vendored
-6
View File
@@ -1,6 +0,0 @@
<dl>
<dt>Term</dt>
<dd>
Definition content
</dd>
</dl>
markdown/testdata/deflists/reference/pandoc/02-multiple-defs.html
Vendored
-16
View File
@@ -1,16 +0,0 @@
<dl>
<dt>Markdown</dt>
<dd>
A lightweight markup language
</dd>
<dd>
Created by John Gruber in 2004
</dd>
<dt>HTML</dt>
<dd>
A standard markup language used to create web pages
</dd>
<dd>
Used to create web pages
</dd>
</dl>
markdown/testdata/deflists/reference/pandoc/03-multiline-def.html
Vendored
-9
View File
@@ -1,9 +0,0 @@
<dl>
<dt>Term</dt>
<dd>
This is the first line of definition content This is the second line,
needs at least one space indentation
</dd>
</dl>
<p>This is a new paragraph, needs at least one space indentation and a
preceding blank line</p>
markdown/testdata/deflists/reference/pandoc/04-markdown-in-def.html
Vendored
-9
View File
@@ -1,9 +0,0 @@
<dl>
<dt>Markdown Syntax</dt>
<dd>
<strong>Markdown</strong> supports multiple text formatting: -
<em>Italic</em> and <strong>Bold</strong> - <a
href="https://www.markdownlang.com">Link</a> - <code>Inline code</code>
- &gt; Block quote - Lists and other elements
</dd>
</dl>
markdown/testdata/deflists/reference/pandoc/05-nested.html
Vendored
-13
View File
@@ -1,13 +0,0 @@
<dl>
<dt>Outer Term</dt>
<dd>
Outer Definition
</dd>
<dt>Inner Term</dt>
<dd>
Inner Definition
</dd>
<dd>
Another Inner Definition
</dd>
</dl>
markdown/testdata/deflists/reference/pandoc/06-compact.html
Vendored
-1
View File
@@ -1 +0,0 @@
<p>Term: Definition</p>
markdown/testdata/deflists/reference/pandoc/07-extra-space.html
Vendored
-6
View File
@@ -1,6 +0,0 @@
<dl>
<dt>Term</dt>
<dd>
Definition
</dd>
</dl>
markdown/testdata/deflists/reference/pandoc/08-glossary.html
Vendored
-22
View File
@@ -1,22 +0,0 @@
<dl>
<dt>API</dt>
<dd>
An application programming interface, a set of rules that allow
different software applications to communicate with each other.
</dd>
<dt>Framework</dt>
<dd>
A software library that provides a standardized structure for
application development.
</dd>
<dt>Git</dt>
<dd>
A distributed version control system used to track changes in a project
development process.
</dd>
<dt>IDE</dt>
<dd>
An integrated development environment, a software application that
includes a code editor and various development tools.
</dd>
</dl>
markdown/testdata/deflists/reference/pandoc/09-api-params.html
Vendored
-33
View File
@@ -1,33 +0,0 @@
<dl>
<dt>user_id</dt>
<dd>
<strong>Required</strong> - The unique identifier for the user.
</dd>
<dd>
Type: <code>integer</code>
</dd>
<dt>name</dt>
<dd>
<strong>Optional</strong> - The display name for the user.
</dd>
<dd>
Type: <code>string</code>
</dd>
<dd>
Default: <code>null</code>
</dd>
<dt>status</dt>
<dd>
<strong>Optional</strong> - The user status.
</dd>
<dd>
Type: <code>string</code>
</dd>
<dd>
Allowed values: <code>active</code>, <code>inactive</code>,
<code>suspended</code>
</dd>
<dd>
Default: <code>active</code>
</dd>
</dl>
markdown/toc.odin
+1 -17
View File
@@ -1,6 +1,5 @@
package markdown
import "core:log"
import "core:strings"
// generate_toc scans rendered HTML for <h1>-<h6> tags with id attributes and
@@ -29,23 +28,8 @@ generate_toc :: proc(html: string, allocator := context.allocator) -> string {
current_level = level
strings.write_string(&b, "<ul>\n")
} else if level > current_level {
if level - current_level > 1 {
log.warnf(
"heading level skips from h%d to h%d (\"%s\"); " +
"table of contents inserts empty levels to keep valid nesting",
current_level,
level,
text,
)
}
// The first step down nests inside the current, still-open <li>.
// Each further skipped level needs its own <li> to host the deeper
// <ul>, since a <ul> may only contain <li> — otherwise a jump of
// more than one level emits stacked, parentless <ul> tags.
strings.write_string(&b, "<ul>\n")
current_level += 1
for current_level < level {
strings.write_string(&b, "<li>\n<ul>\n")
strings.write_string(&b, "<ul>\n")
current_level += 1
}
} else if level < current_level {
markdown/toc_test.odin
-35
View File
@@ -1,35 +0,0 @@
#+test
package markdown
import "core:strings"
import "core:testing"
@(test)
test_toc_sequential_nesting :: proc(t: ^testing.T) {
html := `<h1 id="a">A</h1><h2 id="b">B</h2><h1 id="c">C</h1>`
result := generate_toc(html, context.temp_allocator)
testing.expect(t, strings.contains(result, `<a href="#a">A</a>`))
testing.expect(t, strings.contains(result, `<a href="#b">B</a>`))
testing.expect(t, strings.contains(result, `<a href="#c">C</a>`))
// A well-formed list never places a <ul> directly inside a <ul>.
testing.expect(t, !strings.contains(result, "<ul>\n<ul>"))
}
// Regression: a heading level that jumps by more than one (h1 -> h3) must not
// emit stacked, parentless <ul> tags. Each skipped level is wrapped in an <li>
// so the list stays valid.
@(test)
test_toc_level_jump_stays_valid :: proc(t: ^testing.T) {
html := `<h1 id="a">A</h1><h3 id="b">B</h3>`
result := generate_toc(html, context.temp_allocator)
testing.expect(t, !strings.contains(result, "<ul>\n<ul>"))
// The intermediate level is hosted by an <li>.
testing.expect(t, strings.contains(result, "<li>\n<ul>"))
testing.expect(t, strings.contains(result, `<a href="#b">B</a>`))
}
@(test)
test_toc_empty_when_no_headings :: proc(t: ^testing.T) {
result := generate_toc("<p>no headings here</p>", context.temp_allocator)
testing.expect_value(t, result, "")
}
menus.odin
+28 -26
View File
@@ -4,6 +4,7 @@ import "core:encoding/json"
import "core:fmt"
import "core:log"
import "core:mem"
import "core:os"
import "core:strings"
DEFAULT_WEIGHT :: 10
@@ -111,32 +112,42 @@ parse_page_menus :: proc(
return result
}
// build_menus populates site.menus by merging every source into one menu set:
// 1. Auto-menus (sections + root-level pages)
// 2. Config menus (thor.json "menus" key), already parsed into site.menus
// 3. Page frontmatter menus
// All three are combined per menu name and sorted by weight, then name.
// build_menus populates site.menus:
// 1. Config menus (thor.json "menus" key present) — exclusive, preserves array order
// 2. Auto-menus (sections + root-level pages) + page frontmatter menus — merged, sorted
//
// If "menus" is present but empty ({}) it means explicit opt-out: no menus at
// all, and page frontmatter menus are ignored.
// If "menus" is present but empty ({}) it means explicit opt-out: no menus.
// Config menus cannot be mixed with page frontmatter menus (error).
build_menus :: proc(site: ^Site) {
// Explicit opt-out: "menus": {} present but empty → no menus; page
// frontmatter menus are ignored (with a warning).
// TODO(TODOS.md): silently discarding frontmatter here is harsh; revisit.
if site.menus != nil && len(site.menus) == 0 {
if site.menus != nil {
has_menus := false
for page in site.pages {
if len(page.menus) > 0 {
log.warnf(
"menus: config has empty menus but pages have frontmatter menu entries; ignoring page menus",
)
has_menus = true
break
}
}
// Already populated from config in site_apply_config
if len(site.menus) == 0 {
// Explicit opt-out ("menus": {})
if has_menus {
log.warnf(
"menus: config has empty menus but pages have frontmatter menu entries; ignoring page menus",
)
}
return
}
// Config menus active
if has_menus {
log.fatalf("menus: cannot mix config menus with frontmatter menus")
os.exit(1)
}
warn_all_duplicate_weights(site)
return
}
// Merge all sources: auto section/root menus, config menus (already in
// site.menus if present), then page frontmatter menus.
// No config menus — auto-generate, then merge page menus on top
collect_auto_menus(site)
merge_page_menus(site)
warn_all_duplicate_weights(site)
@@ -242,17 +253,8 @@ collect_auto_menus :: proc(site: ^Site) {
return
}
if site.menus == nil {
site.menus = make(map[string][]Menu_Entry, alloc)
}
// Merge into any existing "main" (e.g. config menus) rather than clobbering.
if existing, ok := site.menus["main"]; ok {
merged := make([dynamic]Menu_Entry, 0, len(existing) + len(entries), alloc)
append(&merged, ..existing)
append(&merged, ..entries[:])
entries = merged
}
sort_menu_entries(entries[:])
site.menus = make(map[string][]Menu_Entry, alloc)
site.menus["main"] = entries[:]
}
menus_test.odin
+5 -65
View File
@@ -100,6 +100,7 @@ test_menus_nil_input :: proc(t: ^testing.T) {
@(test)
test_menus_invalid_type :: proc(t: ^testing.T) {
context.logger = log.nil_logger()
arena: mem.Dynamic_Arena
mem.dynamic_arena_init(&arena)
defer mem.dynamic_arena_destroy(&arena)
@@ -112,6 +113,7 @@ test_menus_invalid_type :: proc(t: ^testing.T) {
@(test)
test_menus_array_non_string :: proc(t: ^testing.T) {
context.logger = log.nil_logger()
arena: mem.Dynamic_Arena
mem.dynamic_arena_init(&arena)
defer mem.dynamic_arena_destroy(&arena)
@@ -128,6 +130,7 @@ test_menus_array_non_string :: proc(t: ^testing.T) {
@(test)
test_menus_object_non_object_value :: proc(t: ^testing.T) {
context.logger = log.nil_logger()
arena: mem.Dynamic_Arena
mem.dynamic_arena_init(&arena)
defer mem.dynamic_arena_destroy(&arena)
@@ -143,6 +146,7 @@ test_menus_object_non_object_value :: proc(t: ^testing.T) {
@(test)
test_menus_non_numeric_weight :: proc(t: ^testing.T) {
context.logger = log.nil_logger()
arena: mem.Dynamic_Arena
mem.dynamic_arena_init(&arena)
defer mem.dynamic_arena_destroy(&arena)
@@ -157,6 +161,7 @@ test_menus_non_numeric_weight :: proc(t: ^testing.T) {
@(test)
test_menus_empty_title :: proc(t: ^testing.T) {
context.logger = log.nil_logger()
arena: mem.Dynamic_Arena
mem.dynamic_arena_init(&arena)
defer mem.dynamic_arena_destroy(&arena)
@@ -421,71 +426,6 @@ test_auto_menus_no_duplicate_with_frontmatter :: proc(t: ^testing.T) {
testing.expect_value(t, main[0].name, "Ideas")
}
// --- config + frontmatter + auto merge tests ---
@(test)
test_build_menus_config_merges_frontmatter :: proc(t: ^testing.T) {
site: Site
mem.dynamic_arena_init(&site.arena)
defer mem.dynamic_arena_destroy(&site.arena)
context.allocator = site_allocator(&site)
// Simulate config menus as site_apply_config would populate them.
site.menus = make(map[string][]Menu_Entry, site_allocator(&site))
cfg := make([dynamic]Menu_Entry, 0, 1, site_allocator(&site))
append(&cfg, Menu_Entry{name = "GitHub", url = "https://example.com", weight = 20})
site.menus["main"] = cfg[:]
// A root-level page opting into "main" via frontmatter.
page := make_page("Ideas", "/ideas/")
page.menus = parse_page_menus(parse_raw(`"main"`), page, site_allocator(&site))
site.pages = make(#soa[dynamic]Page, 0, 1, site_allocator(&site))
append(&site.pages, page)
build_menus(&site)
main, ok := site.menus["main"]
testing.expect(t, ok)
// GitHub (config, weight 20) + Ideas (frontmatter, nil → DEFAULT_WEIGHT 10).
testing.expect(t, len(main) == 2, "expected config + frontmatter merged")
testing.expect_value(t, main[0].name, "Ideas")
testing.expect_value(t, main[1].name, "GitHub")
}
@(test)
test_build_menus_config_merges_auto :: proc(t: ^testing.T) {
site: Site
mem.dynamic_arena_init(&site.arena)
defer mem.dynamic_arena_destroy(&site.arena)
context.allocator = site_allocator(&site)
// Config menu in a different bucket ("footer"); auto-gen owns "main".
site.menus = make(map[string][]Menu_Entry, site_allocator(&site))
cfg := make([dynamic]Menu_Entry, 0, 1, site_allocator(&site))
append(&cfg, Menu_Entry{name = "GitHub", url = "https://example.com", weight = 20})
site.menus["footer"] = cfg[:]
// A section (non-index) page triggers an auto "main" section entry.
page := make_page("Post", "/blog/post/")
page.section = "blog"
site.pages = make(#soa[dynamic]Page, 0, 1, site_allocator(&site))
append(&site.pages, page)
build_menus(&site)
footer, ok_f := site.menus["footer"]
testing.expect(t, ok_f, "config footer menu should be preserved")
testing.expect(t, len(footer) == 1)
testing.expect_value(t, footer[0].name, "GitHub")
main, ok_m := site.menus["main"]
testing.expect(t, ok_m, "auto-generated main menu should exist alongside config")
testing.expect(t, len(main) == 1)
testing.expect_value(t, main[0].name, "Blog")
}
// --- warn_duplicate_weights tests ---
@(test)
minify.odin
+21 -97
View File
@@ -12,31 +12,18 @@ Range :: struct {
end: u32,
}
// minify_html collapses insignificant whitespace, strips comments, and minifies
// inline <style> bodies (via minify_css). The caller owns html_parser and
// css_parser and passes them in; both may be reused across calls. On success it
// returns (owned string, true) and the caller must free the string. On failure
// (nil html_parser, parse failure, or syntax errors) it returns ("", false) and
// the caller should keep the original source unchanged.
minify_html :: proc(
html_parser: ts.Parser,
css_parser: ts.Parser,
source: string,
) -> (
minified: string,
ok: bool,
) {
if html_parser == nil {
return minified, ok
minify_html :: proc(source: string) -> string {
gc := ts.ensure_parser("html")
if gc == nil {
return source
}
source_c := strings.clone_to_cstring(source)
defer delete(source_c)
tree := ts.parser_parse_string(html_parser, nil, source_c, u32(len(source)))
tree := ts.parser_parse_string(gc.parser, nil, source_c, u32(len(source)))
if tree == nil {
ok = false
return minified, ok
return source
}
defer ts.tree_delete(tree)
@@ -44,24 +31,20 @@ minify_html :: proc(
if ts.node_has_error(root) {
log.warnf("minify: HTML parse errors, skipping minification")
ok = false
return minified, ok
return source
}
comments: [dynamic]Range
defer delete(comments)
preserves: [dynamic]Range
defer delete(preserves)
styles: [dynamic]Range
defer delete(styles)
collect_html_ranges(root, source, &comments, &preserves, &styles)
collect_html_ranges(root, source, &comments, &preserves)
sb := strings.builder_make()
ci := 0
pi := 0
si := 0
i := 0
last_written: u8 = 0
@@ -78,27 +61,6 @@ minify_html :: proc(
continue
}
if si < len(styles) && u32(i) >= styles[si].start {
s := styles[si]
seg := source[i:s.end]
if minified, mok := minify_css(css_parser, seg); mok {
strings.write_string(&sb, minified)
if len(minified) > 0 {
last_written = minified[len(minified) - 1]
}
delete(minified)
} else {
// CSS minify skipped/failed: keep the style body verbatim.
strings.write_string(&sb, seg)
if len(seg) > 0 {
last_written = seg[len(seg) - 1]
}
}
i = int(s.end)
si += 1
continue
}
if ci < len(comments) && u32(i) >= comments[ci].start {
i = int(comments[ci].end)
ci += 1
@@ -131,10 +93,7 @@ minify_html :: proc(
}
}
minified, ok = strings.to_string(sb), true
return minified, ok
return strings.to_string(sb)
}
collect_html_ranges :: proc(
@@ -142,7 +101,6 @@ collect_html_ranges :: proc(
source: string,
comments: ^[dynamic]Range,
preserves: ^[dynamic]Range,
styles: ^[dynamic]Range,
) {
child_count := ts.node_named_child_count(node)
for i in 0 ..< child_count {
@@ -154,14 +112,7 @@ collect_html_ranges :: proc(
comments,
Range{start = ts.node_start_byte(child), end = ts.node_end_byte(child)},
)
} else if type_str == "style_element" {
// Preserve the <style> tags but minify the CSS body (the raw_text
// child) via the CSS-aware pass.
raw, ok := style_raw_text(child)
if ok {
append(styles, Range{start = ts.node_start_byte(raw), end = ts.node_end_byte(raw)})
}
} else if type_str == "script_element" {
} else if type_str == "script_element" || type_str == "style_element" {
append(
preserves,
Range{start = ts.node_start_byte(child), end = ts.node_end_byte(child)},
@@ -174,27 +125,14 @@ collect_html_ranges :: proc(
Range{start = ts.node_start_byte(child), end = ts.node_end_byte(child)},
)
} else {
collect_html_ranges(child, source, comments, preserves, styles)
collect_html_ranges(child, source, comments, preserves)
}
} else {
collect_html_ranges(child, source, comments, preserves, styles)
collect_html_ranges(child, source, comments, preserves)
}
}
}
// style_raw_text returns the raw_text (CSS body) child of a style_element.
// ok is false for an empty <style></style>, which has no raw_text child.
style_raw_text :: proc(style_element: ts.Node) -> (raw: ts.Node, ok: bool) {
child_count := ts.node_named_child_count(style_element)
for i in 0 ..< child_count {
child := ts.node_named_child(style_element, u32(i))
if string(ts.node_type(child)) == "raw_text" {
return child, true
}
}
return {}, false
}
html_tag_name :: proc(element: ts.Node, source: string) -> string {
child_count := ts.node_named_child_count(element)
for i in 0 ..< child_count {
@@ -230,23 +168,18 @@ is_css_delim :: proc(c: u8) -> bool {
return false
}
// minify_css collapses insignificant whitespace and strips comments. The caller
// owns `parser` and passes it in; it may be reused across calls. On success it
// returns (owned string, true) and the caller must free the string. On failure
// (nil parser, parse failure, or syntax errors) it returns ("", false) and the
// caller should keep the original source unchanged.
minify_css :: proc(parser: ts.Parser, source: string) -> (minified: string, ok: bool) {
if parser == nil {
return minified, ok
minify_css :: proc(source: string) -> string {
gc := ts.ensure_parser("css")
if gc == nil {
return source
}
source_c := strings.clone_to_cstring(source)
defer delete(source_c)
tree := ts.parser_parse_string(parser, nil, source_c, u32(len(source)))
tree := ts.parser_parse_string(gc.parser, nil, source_c, u32(len(source)))
if tree == nil {
ok = false
return minified, ok
return source
}
defer ts.tree_delete(tree)
@@ -254,8 +187,7 @@ minify_css :: proc(parser: ts.Parser, source: string) -> (minified: string, ok:
if ts.node_has_error(root) {
log.warnf("minify: CSS parse errors, skipping minification")
ok = false
return minified, ok
return source
}
comments: [dynamic]Range
@@ -292,14 +224,7 @@ minify_css :: proc(parser: ts.Parser, source: string) -> (minified: string, ok:
next = source[j]
}
// Emit a single separating space, but never a leading space
// (last_written == 0), a doubled space (last_written == ' ',
// which happens when a stripped comment sits between two
// whitespace runs), or one adjacent to a delimiter.
if last_written != 0 &&
last_written != ' ' &&
!is_css_delim(last_written) &&
!is_css_delim(next) {
if !is_css_delim(last_written) && !is_css_delim(next) {
strings.write_byte(&sb, ' ')
last_written = ' '
}
@@ -311,8 +236,7 @@ minify_css :: proc(parser: ts.Parser, source: string) -> (minified: string, ok:
}
}
minified, ok = strings.to_string(sb), true
return minified, ok
return strings.to_string(sb)
}
collect_css_comments :: proc(node: ts.Node, comments: ^[dynamic]Range) {
minify_test.odin
-98
View File
@@ -1,98 +0,0 @@
#+test
package main
import ts "treesitter"
import "core:strings"
import "core:testing"
// The treesitter registry is thread-safe and self-initializing (grammar() lazily
// binds it to the heap), and parsers are caller-owned. Each test opens its own
// parser(s) via ts.open_parser, passes them to minify_*, and frees the returned
// string — so the multi-threaded test runner needs no shared state or locking.
@(private = "file")
open_parsers :: proc(t: ^testing.T) -> (html_parser, css_parser: ts.Parser) {
hg, hok := ts.grammar("html")
cg, cok := ts.grammar("css")
testing.expect(t, hok, "html grammar should load")
testing.expect(t, cok, "css grammar should load")
return ts.open_parser(hg), ts.open_parser(cg)
}
@(private = "file")
expect_css :: proc(t: ^testing.T, parser: ts.Parser, input, want: string) {
got, ok := minify_css(parser, input)
testing.expect(t, ok)
defer delete(got)
testing.expect_value(t, got, want)
}
// --- minify_html: inline <style> bodies get CSS-minified ---
@(test)
test_minify_html_inline_style :: proc(t: ^testing.T) {
hp, cp := open_parsers(t)
defer ts.parser_delete(hp)
defer ts.parser_delete(cp)
input := `<html><head><style>
body {
color: red;
}
</style></head><body><p>hi</p></body></html>`
result, ok := minify_html(hp, cp, input)
testing.expect(t, ok)
defer delete(result)
// The CSS body is collapsed by minify_css (delimiters lose surrounding
// whitespace)...
testing.expect(t, strings.contains(result, "body{color:red;}"))
// ...while the <style> tags themselves survive.
testing.expect(t, strings.contains(result, "<style>"))
testing.expect(t, strings.contains(result, "</style>"))
}
@(test)
test_minify_html_empty_style :: proc(t: ^testing.T) {
hp, cp := open_parsers(t)
defer ts.parser_delete(hp)
defer ts.parser_delete(cp)
// An empty <style></style> has no raw_text child; must not crash.
input := `<html><head><style></style></head><body></body></html>`
result, ok := minify_html(hp, cp, input)
testing.expect(t, ok)
defer delete(result)
testing.expect(t, strings.contains(result, "<style></style>"))
}
// --- minify_css: whitespace around stripped comments / leading edge ---
@(test)
test_minify_css_leading_and_comment :: proc(t: ^testing.T) {
cg, cok := ts.grammar("css")
testing.expect(t, cok, "css grammar should load")
cp := ts.open_parser(cg)
defer ts.parser_delete(cp)
// Leading whitespace and a comment between two whitespace runs must not
// leave a leading space or a doubled space.
expect_css(t, cp, "\n/* c */\na {}", "a{}")
expect_css(t, cp, " a {}", "a{}")
expect_css(t, cp, "a {}\n/* trailing */", "a{}")
}
@(test)
test_minify_html_script_preserved :: proc(t: ^testing.T) {
hp, cp := open_parsers(t)
defer ts.parser_delete(hp)
defer ts.parser_delete(cp)
// <script> stays byte-for-byte (whitespace-significant JS).
input := "<html><body><script>\nconst s = \"a b\";\n</script></body></html>"
result, ok := minify_html(hp, cp, input)
testing.expect(t, ok)
defer delete(result)
testing.expect(t, strings.contains(result, "const s = \"a b\";"))
}
mustache/TODOS.md
-1
View File
@@ -4,4 +4,3 @@
- [ ] Simplify `trim_standalone_whitespace` — four temp arrays (`li_buf`, `ri_buf`, `left_done`, `right_done`) could potentially be reduced to two by merging the done-tracking into the index buffers using sentinel values (e.g. `-2` = already trimmed).
- [ ] Adjacent standalone tags don't capture indent for the second tag (e.g. ` {{>a}}\n {{>b}}` — `{{>b}}` gets `indent=""` because its left text was consumed by `{{>a}}`'s right-trim). Blocks recover via `deindent_blocks`, but partials/parents have no fallback.
- [ ] lazy load partials ? (with mutex)
- [ ] Support changing delimiters (Set Delimiter tag `{{=<% %>=}}`). Delimiters `{{`/`}}`/`{{{`/`}}}` are hardcoded in [tokenizer.odin](./tokenizer.odin); needs an `=` sigil case, mutable open/close delimiters threaded through `tokenize`, and wiring up `spec/specs/delimiters.json` in `spec_test.odin` (fixture present but not run).
mustache/diag_cases.odin
-149
View File
@@ -1,149 +0,0 @@
package mustache
import "core:os"
import "core:path/filepath"
import "core:strings"
Diag_Case :: struct {
name: string,
enable: bool,
should: string, // "error" | "warn" | "pass"
expected: string, // exact formatted diagnostic output
input: string,
}
load_diag_cases :: proc(path: string, allocator := context.temp_allocator) -> []Diag_Case {
dirs := [?]string{#directory, path}
full_path, _ := filepath.join(dirs[:], allocator)
raw_data, err := os.read_entire_file(full_path, allocator)
if err != nil {
return {}
}
content := string(raw_data)
lines := strings.split(content, "\n", allocator)
cases := make([dynamic]Diag_Case, 0, 16, allocator)
current: Diag_Case
has_current := false
// Multiline block state
in_block := false
block_key: string
block_indent := -1
block_sb: strings.Builder
for line in lines {
raw := strings.trim_right(line, "\r")
if in_block {
// Check if this line belongs to the multiline block
is_indented := len(raw) == 0 || raw[0] == ' ' || raw[0] == '\t'
if is_indented {
if block_indent < 0 && len(raw) > 0 {
// Determine indentation from first content line
block_indent = 0
for block_indent < len(raw) && raw[block_indent] == ' ' {
block_indent += 1
}
}
// Strip block indentation
stripped := raw
if block_indent > 0 && len(stripped) >= block_indent {
stripped = stripped[block_indent:]
}
if strings.builder_len(block_sb) > 0 {
strings.write_string(&block_sb, "\n")
}
strings.write_string(&block_sb, stripped)
continue
} else {
// End of multiline block
in_block = false
strings.write_string(&block_sb, "\n") // YAML | trailing newline
value := strings.clone(strings.to_string(block_sb), allocator)
strings.builder_destroy(&block_sb)
set_field(&current, block_key, value)
block_indent = -1
// Fall through to process this line
}
}
trimmed := strings.trim_space(raw)
// Entry separator
if trimmed == "---" {
if has_current {
append(&cases, current)
current = {}
has_current = false
}
continue
}
// Skip blank lines outside blocks
if trimmed == "" {
continue
}
// Multiline block start: "key: |"
if strings.has_suffix(trimmed, ": |") {
colon := strings.last_index(trimmed, ":")
if colon > 0 {
block_key = strings.trim_space(trimmed[:colon])
strings.builder_init(&block_sb, allocator)
in_block = true
has_current = true
continue
}
}
// Simple key: value
colon := strings.index(trimmed, ":")
if colon > 0 {
key := strings.trim_space(trimmed[:colon])
value := strings.trim_space(trimmed[colon + 1:])
// Strip matching outer quotes
if len(value) >= 2 {
if (value[0] == '"' && value[len(value) - 1] == '"') ||
(value[0] == '\'' && value[len(value) - 1] == '\'') {
value = value[1:len(value) - 1]
}
}
set_field(&current, key, value)
has_current = true
}
}
// Flush trailing multiline block
if in_block {
strings.write_string(&block_sb, "\n") // YAML | trailing newline
value := strings.clone(strings.to_string(block_sb), allocator)
strings.builder_destroy(&block_sb)
set_field(&current, block_key, value)
}
// Flush last entry
if has_current {
append(&cases, current)
}
return cases[:]
}
set_field :: proc(c: ^Diag_Case, key, value: string) {
switch key {
case "name":
c.name = value
case "enable":
c.enable = value == "true"
case "should":
c.should = value
case "expected":
c.expected = value
case "input":
c.input = value
}
}
mustache/diagnostic.odin
+243 -38
View File
@@ -1,6 +1,61 @@
package mustache
import diags "../diagnostics"
import "core:fmt"
import "core:os"
import "core:strings"
import "core:terminal/ansi"
import "core:unicode/utf8"
// line_col returns the 1-indexed line and column for a byte offset in source.
// Newlines ('\n') separate lines; '\r' is treated as part of '\r\n'. Column is
// counted in bytes from the start of the line.
line_col :: proc(source: string, pos_in: int) -> (line: int, col: int) {
pos := pos_in
if pos < 0 {
return 1, 1
}
if pos > len(source) {
pos = len(source)
}
line = 1
col = 1
for i := 0; i < pos; i += 1 {
if source[i] == '\n' {
line += 1
col = 1
} else {
col += 1
}
}
return
}
// line_text returns the Nth (1-indexed) line of source, without the trailing
// newline. Returns "" if line is out of range.
line_text :: proc(source: string, line: int) -> string {
if line < 1 {
return ""
}
current := 1
start := 0
for i := 0; i < len(source); i += 1 {
if current == line {
end := i
for end < len(source) && source[end] != '\n' {
end += 1
}
return source[start:end]
}
if source[i] == '\n' {
current += 1
start = i + 1
}
}
if current == line {
return source[start:]
}
return ""
}
// context_extent returns the byte offset of the start of the line containing
// pos, plus the byte offsets of the start and end of the mustache tag at pos.
@@ -60,11 +115,68 @@ context_extent :: proc(
return line_start, pos, pos + 1
}
// format_tag_error is a mustache-aware wrapper around diags.format_error.
// It uses context_extent to compute the caret underline for `{{ }}` tags,
// falling back to a single-rune underline when pos is not inside a tag.
// When `span > 0`, the caret extends from pos to pos + span.
format_tag_error :: proc(
// should_colorize returns true if stderr is a TTY and color output is wanted.
should_colorize :: proc() -> bool {
return os.is_tty(os.stderr)
}
// count_lines returns the number of '\n'-separated lines in source.
// A trailing newline does not add an extra line.
count_lines :: proc(source: string) -> int {
if len(source) == 0 {
return 1
}
n := 1
for c in source {
if c == '\n' {
n += 1
}
}
// Drop phantom last line if source ends with '\n'.
if len(source) > 0 && source[len(source) - 1] == '\n' {
n -= 1
}
return n
}
// digit_count returns the number of decimal digits in n (min 1).
digit_count :: proc(n: int) -> int {
if n <= 0 {
return 1
}
c := 0
x := n
for x > 0 {
c += 1
x /= 10
}
return c
}
// display_width returns the number of terminal cells `s` occupies.
// For ASCII this is byte length; for UTF-8 we count runes (combining
// marks and wide CJK chars are still approximate).
display_width :: proc(s: string) -> int {
return utf8.rune_count_in_string(s)
}
// format_error produces a rust-style multi-line diagnostic string.
//
// <msg>
// --> <path>:<line>:<col>
// |
// N | <source line N-2>
// N | <source line N-1>
// N | <source line N — the error line>
// | ^^^^^^^^^^^ <hint>
// N | <source line N+1>
// N | <source line N+2>
// |
//
// `context_before`/`context_after` lines of context are shown around the
// error line. Line numbers are right-aligned to the width of the largest
// line number shown.
format_error :: proc(
path: string,
source: string,
pos: int,
@@ -75,25 +187,135 @@ format_tag_error :: proc(
colorize: bool = false,
span: int = 0,
) -> string {
caret_start := pos
caret_end := pos + 1
if span > 0 {
caret_end = pos + span
} else {
_, ts, te := context_extent(source, pos)
caret_start = ts
caret_end = te
line, col := line_col(source, pos)
total_lines := count_lines(source)
start_line := line - context_before
if start_line < 1 {
start_line = 1
}
return diags.format_error(
path, source, pos, msg, hint,
context_before, context_after,
colorize,
caret_start, caret_end,
)
end_line := line + context_after
if end_line > total_lines {
end_line = total_lines
}
// Width of the line-number column (right-align).
width := digit_count(end_line)
if width < 1 {
width = 1
}
sb := strings.builder_make(context.temp_allocator)
defer strings.builder_destroy(&sb)
color := colorize
red, faint, reset := "", "", ""
if color {
red = ansi.CSI + ansi.FG_RED + ansi.SGR
faint = ansi.CSI + ansi.FAINT + ansi.SGR
reset = ansi.CSI + ansi.RESET + ansi.SGR
}
// Header line: message.
strings.write_string(&sb, msg)
strings.write_byte(&sb, '\n')
// Location line: " --> path:line:col" (width spaces + arrow).
strings.write_string(&sb, faint)
for _ in 0 ..< width {
strings.write_byte(&sb, ' ')
}
strings.write_string(&sb, "--> ")
strings.write_string(&sb, reset)
fmt.sbprintf(&sb, "%s:%d:%d\n", path, line, col)
// Top gutter line.
write_gutter(&sb, width, faint, reset)
// Caret extent for the error line.
line_start, token_start, token_end := context_extent(source, pos)
if span > 0 {
token_start = pos
token_end = pos + span
}
caret_start_col := token_start - line_start + 1
caret_end_col := token_end - line_start + 1
if caret_end_col <= caret_start_col {
caret_end_col = caret_start_col + 1
}
// Context lines.
for n in start_line ..= end_line {
// Line number (right-aligned, faint).
num_str := fmt.tprintf("%d", n)
strings.write_string(&sb, faint)
for _ in 0 ..< width - len(num_str) {
strings.write_byte(&sb, ' ')
}
strings.write_string(&sb, num_str)
strings.write_string(&sb, " | ")
strings.write_string(&sb, reset)
strings.write_string(&sb, line_text(source, n))
strings.write_byte(&sb, '\n')
// After the error line, emit the caret row.
if n == line {
strings.write_string(&sb, faint)
for _ in 0 ..< width + 1 {
strings.write_byte(&sb, ' ')
}
strings.write_string(&sb, "| ")
strings.write_string(&sb, reset)
for _ in 1 ..< caret_start_col {
strings.write_byte(&sb, ' ')
}
if color {
strings.write_string(&sb, red)
}
for _ in 0 ..< caret_end_col - caret_start_col {
strings.write_byte(&sb, '^')
}
if color {
strings.write_string(&sb, reset)
}
if hint != "" {
strings.write_byte(&sb, ' ')
if color {
strings.write_string(&sb, faint)
}
strings.write_string(&sb, hint)
if color {
strings.write_string(&sb, reset)
}
}
strings.write_byte(&sb, '\n')
}
}
// Trailing gutter line for visual closure.
write_gutter(&sb, width, faint, reset)
return strings.to_string(sb)
}
// write_gutter emits a faint pipe-only gutter line: `<width+1 spaces> |`.
write_gutter :: proc(sb: ^strings.Builder, width: int, faint: string, reset: string) {
strings.write_string(sb, faint)
for _ in 0 ..< width + 1 {
strings.write_byte(sb, ' ')
}
strings.write_string(sb, "|")
strings.write_string(sb, reset)
strings.write_byte(sb, '\n')
}
// format_render_error produces a diagnostic for an Error value using the
// template's path and source for context. Returns "" for nil errors.
// If the error carries its own source/path (from tag_error), those are used
// instead of the passed-in template — this ensures errors inside partials
// point at the correct file.
format_render_error :: proc(err: Error, tmpl: Template, colorize: bool = false) -> string {
if err == nil {
return ""
@@ -104,22 +326,5 @@ format_render_error :: proc(err: Error, tmpl: Template, colorize: bool = false)
if path == "" {
path = "<input>"
}
caret_start := b.pos
caret_end := b.pos + 1
if b.span > 0 {
caret_end = b.pos + b.span
} else {
_, ts, te := context_extent(source, b.pos)
caret_start = ts
caret_end = te
}
return diags.format_error(
path, source, b.pos, b.msg,
hint = b.hint,
colorize = colorize,
caret_start = caret_start,
caret_end = caret_end,
)
return format_error(path, source, b.pos, b.msg, hint = b.hint, colorize = colorize, span = b.span)
}
mustache/diagnostic_test.odin
+101 -288
View File
@@ -1,22 +1,100 @@
#+test
package mustache
import "core:encoding/json"
import "core:fmt"
import "core:log"
import "core:os"
import "core:path/filepath"
import "core:strings"
import "core:testing"
// ---------------------------------------------------------------------------
// format_tag_error — golden output tests
// line_col / line_text / count_lines / digit_count — primitive helpers
// ---------------------------------------------------------------------------
@(test)
test_line_col_basic :: proc(t: ^testing.T) {
src := "abc\ndef\nghi"
cases := [?]struct {
pos: int,
line: int,
col: int,
}{{0, 1, 1}, {2, 1, 3}, {3, 1, 4}, {4, 2, 1}, {6, 2, 3}, {7, 2, 4}, {8, 3, 1}}
for c in cases {
l, col := line_col(src, c.pos)
testing.expect(t, l == c.line, fmt.tprintf("pos %d: line %d, want %d", c.pos, l, c.line))
testing.expect(t, col == c.col, fmt.tprintf("pos %d: col %d, want %d", c.pos, col, c.col))
}
}
@(test)
test_line_col_empty :: proc(t: ^testing.T) {
l, col := line_col("", 0)
testing.expect_value(t, l, 1)
testing.expect_value(t, col, 1)
}
@(test)
test_line_col_negative :: proc(t: ^testing.T) {
l, col := line_col("abc", -1)
testing.expect_value(t, l, 1)
testing.expect_value(t, col, 1)
}
@(test)
test_line_col_past_end :: proc(t: ^testing.T) {
l, col := line_col("abc", 100)
testing.expect_value(t, l, 1)
testing.expect_value(t, col, 4)
}
@(test)
test_line_text_first :: proc(t: ^testing.T) {
src := "first\nsecond\nthird"
testing.expect_value(t, line_text(src, 1), "first")
testing.expect_value(t, line_text(src, 2), "second")
testing.expect_value(t, line_text(src, 3), "third")
}
@(test)
test_line_text_trailing_newline :: proc(t: ^testing.T) {
src := "first\nsecond\n"
testing.expect_value(t, line_text(src, 1), "first")
testing.expect_value(t, line_text(src, 2), "second")
testing.expect_value(t, line_text(src, 3), "")
}
@(test)
test_line_text_out_of_range :: proc(t: ^testing.T) {
testing.expect_value(t, line_text("abc", 5), "")
testing.expect_value(t, line_text("abc", 0), "")
}
@(test)
test_count_lines :: proc(t: ^testing.T) {
testing.expect_value(t, count_lines(""), 1)
testing.expect_value(t, count_lines("abc"), 1)
testing.expect_value(t, count_lines("a\nb"), 2)
testing.expect_value(t, count_lines("a\nb\n"), 2)
testing.expect_value(t, count_lines("a\nb\nc"), 3)
}
@(test)
test_digit_count :: proc(t: ^testing.T) {
testing.expect_value(t, digit_count(0), 1)
testing.expect_value(t, digit_count(1), 1)
testing.expect_value(t, digit_count(9), 1)
testing.expect_value(t, digit_count(10), 2)
testing.expect_value(t, digit_count(99), 2)
testing.expect_value(t, digit_count(100), 3)
testing.expect_value(t, digit_count(-5), 1)
}
// ---------------------------------------------------------------------------
// format_error — golden output tests
// ---------------------------------------------------------------------------
@(test)
test_format_error_basic :: proc(t: ^testing.T) {
src := "line 1\nline 2\n{{bad}}\nline 4\nline 5"
out := format_tag_error("p.html", src, 14, "unknown key 'bad'", "", colorize = false)
out := format_error("p.html", src, 14, "unknown key 'bad'", "", colorize = false)
expected := `unknown key 'bad'
--> p.html:3:1
|
@@ -34,7 +112,7 @@ test_format_error_basic :: proc(t: ^testing.T) {
@(test)
test_format_error_with_hint :: proc(t: ^testing.T) {
src := "line 1\nline 2\n{{titel}}\nline 4\nline 5"
out := format_tag_error(
out := format_error(
"post.html",
src,
14,
@@ -59,7 +137,7 @@ test_format_error_with_hint :: proc(t: ^testing.T) {
@(test)
test_format_error_no_hint_omits_trailing_space :: proc(t: ^testing.T) {
src := "{{bad}}"
out := format_tag_error("p.html", src, 0, "msg", "", colorize = false)
out := format_error("p.html", src, 0, "msg", "", colorize = false)
// Caret line ends immediately after the carets — no trailing space.
testing.expect(t, strings.contains(out, "^^^^^^^\n"), out)
testing.expect(t, !strings.contains(out, "^^^^^^^ \n"), out)
@@ -72,7 +150,7 @@ test_format_error_no_hint_omits_trailing_space :: proc(t: ^testing.T) {
@(test)
test_format_error_first_line_only_after_context :: proc(t: ^testing.T) {
src := "{{bad}}\nline 2\nline 3\nline 4\nline 5"
out := format_tag_error("p.html", src, 0, "msg", "", colorize = false)
out := format_error("p.html", src, 0, "msg", "", colorize = false)
expected := `msg
--> p.html:1:1
|
@@ -88,7 +166,7 @@ test_format_error_first_line_only_after_context :: proc(t: ^testing.T) {
@(test)
test_format_error_last_line_only_before_context :: proc(t: ^testing.T) {
src := "line 1\nline 2\nline 3\nline 4\n{{bad}}"
out := format_tag_error("p.html", src, 28, "msg", "", colorize = false)
out := format_error("p.html", src, 28, "msg", "", colorize = false)
expected := `msg
--> p.html:5:1
|
@@ -104,7 +182,7 @@ test_format_error_last_line_only_before_context :: proc(t: ^testing.T) {
@(test)
test_format_error_short_source_clamped :: proc(t: ^testing.T) {
src := "x\n{{bad}}\ny"
out := format_tag_error("p.html", src, 2, "msg", "", colorize = false)
out := format_error("p.html", src, 2, "msg", "", colorize = false)
expected := `msg
--> p.html:2:1
|
@@ -120,7 +198,7 @@ test_format_error_short_source_clamped :: proc(t: ^testing.T) {
@(test)
test_format_error_single_line_source :: proc(t: ^testing.T) {
src := "{{bad}}"
out := format_tag_error("p.html", src, 0, "msg", "", colorize = false)
out := format_error("p.html", src, 0, "msg", "", colorize = false)
expected := `msg
--> p.html:1:1
|
@@ -136,7 +214,7 @@ test_format_error_two_digit_line_numbers :: proc(t: ^testing.T) {
// 12-line source; error on line 9. end_line=11 → width=2.
src := "l01\nl02\nl03\nl04\nl05\nl06\nl07\nl08\n{{bad}}\nl10\nl11\nl12"
// Position of `{{bad}}`: 8 lines of "l0N\n" = 8*4 = 32 bytes.
out := format_tag_error("p.html", src, 32, "msg", "", colorize = false)
out := format_error("p.html", src, 32, "msg", "", colorize = false)
testing.expect(t, strings.contains(out, " --> p.html:9:1\n"), out)
testing.expect(t, strings.contains(out, " |\n"), out)
testing.expect(t, strings.contains(out, " 7 | l07\n"), out)
@@ -163,7 +241,7 @@ test_format_error_three_digit_line_numbers :: proc(t: ^testing.T) {
pos += len(parts[i - 1]) + 1
}
out := format_tag_error("p.html", src, pos, "msg", "", colorize = false)
out := format_error("p.html", src, pos, "msg", "", colorize = false)
testing.expect(t, strings.contains(out, " --> p.html:100:1\n"), out)
testing.expect(t, strings.contains(out, " |\n"), out)
testing.expect(t, strings.contains(out, " 98 | l098\n"), out)
@@ -178,7 +256,7 @@ test_format_error_three_digit_line_numbers :: proc(t: ^testing.T) {
@(test)
test_caret_at_column_1 :: proc(t: ^testing.T) {
src := "{{bad}} at start"
out := format_tag_error("p.html", src, 0, "msg", "", colorize = false)
out := format_error("p.html", src, 0, "msg", "", colorize = false)
// Caret line should start with "^" right after "| " (no leading spaces).
testing.expect(t, strings.contains(out, " | ^^^^^^^\n"), out)
}
@@ -187,7 +265,7 @@ test_caret_at_column_1 :: proc(t: ^testing.T) {
test_caret_at_column_N :: proc(t: ^testing.T) {
src := " {{bad}}"
// pos=4 is the first '{'. Line 1, col 5.
out := format_tag_error("p.html", src, 4, "msg", "", colorize = false)
out := format_error("p.html", src, 4, "msg", "", colorize = false)
// 4 leading spaces, then 7 carets.
testing.expect(t, strings.contains(out, " | ^^^^^^^\n"), out)
}
@@ -195,7 +273,7 @@ test_caret_at_column_N :: proc(t: ^testing.T) {
@(test)
test_caret_width_matches_token :: proc(t: ^testing.T) {
src := "{{x}}"
out := format_tag_error("p.html", src, 0, "msg", "", colorize = false)
out := format_error("p.html", src, 0, "msg", "", colorize = false)
// {{x}} is 5 chars wide.
testing.expect(t, strings.contains(out, " | ^^^^^\n"), out)
}
@@ -207,7 +285,7 @@ test_caret_width_matches_token :: proc(t: ^testing.T) {
@(test)
test_context_before_zero :: proc(t: ^testing.T) {
src := "l1\nl2\nl3\n{{bad}}\nl5\nl6"
out := format_tag_error(
out := format_error(
"p.html",
src,
9,
@@ -231,7 +309,7 @@ test_context_before_zero :: proc(t: ^testing.T) {
@(test)
test_context_after_zero :: proc(t: ^testing.T) {
src := "l1\nl2\nl3\n{{bad}}\nl5\nl6"
out := format_tag_error(
out := format_error(
"p.html",
src,
9,
@@ -255,7 +333,7 @@ test_context_after_zero :: proc(t: ^testing.T) {
@(test)
test_context_both_zero :: proc(t: ^testing.T) {
src := "l1\nl2\nl3\n{{bad}}\nl5\nl6"
out := format_tag_error(
out := format_error(
"p.html",
src,
9,
@@ -282,7 +360,7 @@ test_context_both_zero :: proc(t: ^testing.T) {
@(test)
test_gutter_pipes_align_with_source_pipe :: proc(t: ^testing.T) {
src := "l1\n{{bad}}\nl3"
out := format_tag_error("p.html", src, 3, "msg", "", colorize = false)
out := format_error("p.html", src, 3, "msg", "", colorize = false)
// All "|" characters should appear at the same column.
// For width=1: source line is "N | ...", so "|" at col 2.
// Empty gutter is " |" (width+1 spaces + "|"), so "|" at col 2.
@@ -304,7 +382,7 @@ test_gutter_pipes_align_with_source_pipe :: proc(t: ^testing.T) {
@(test)
test_arrow_points_at_pipe :: proc(t: ^testing.T) {
src := "{{bad}}"
out := format_tag_error("p.html", src, 0, "msg", "", colorize = false)
out := format_error("p.html", src, 0, "msg", "", colorize = false)
// For width=1: arrow line is " --> ..." so ">" at col 3.
// Pipe lines are " |" so "|" at col 2.
lines := strings.split(out, "\n", context.temp_allocator)
@@ -346,7 +424,7 @@ test_format_render_error_dispatch :: proc(t: ^testing.T) {
}
b := body(parse_err)
out := format_tag_error("test.html", src, b.pos, b.msg, colorize = false)
out := format_error("test.html", src, b.pos, b.msg, colorize = false)
testing.expect(t, strings.contains(out, "unclosed section"), out)
testing.expect(t, strings.contains(out, "test.html:"), out)
}
@@ -479,268 +557,3 @@ test_parse_error_pipe_parse_in_inverted_keeps_double_braces :: proc(t: ^testing.
fmt.tprintf("msg should contain literal '{{^', got %q", b.msg),
)
}
// ---------------------------------------------------------------------------
// Data-driven diagnostic cases (from DIAGNOSTIC_TESTS.yaml)
//
// write_diagnostic_tests generates diagnostic_test_cases_test.odin from the
// enabled cases in DIAGNOSTIC_TESTS.yaml. On the first run (or when the YAML
// changes), it regenerates the file and fails. On the second run, each case
// runs as an individual test via run_diag_case.
// ---------------------------------------------------------------------------
Diag_Context :: struct {
items: [5]string,
word: string,
params: json.Object,
now: string,
}
diag_ctx :: proc() -> Diag_Context {
params_raw := `{ "author": { "name": "foo bar"}}`
params_val, err := json.parse(params_raw, allocator = context.temp_allocator)
assert(err == nil)
params, ok := params_val.(json.Object)
assert(ok)
return Diag_Context {
items = {"a", "b", "c", "d", "e"},
word = "Hello, World!",
params = params,
now = "2026-08-10T16:21:55-04:00",
}
}
diag_partials :: proc() -> map[string]Template {
dirs := [?]string{#directory, "fixtures/layouts/partials"}
partials_dir, _ := filepath.join(dirs[:], context.temp_allocator)
partials := load_partials(partials_dir, allocator = context.temp_allocator)
// #directory expands to an absolute, build-environment-specific path (e.g.
// /build/... inside the nix sandbox vs a local checkout). Strip it so
// partial diagnostics reference a stable path across environments, keeping
// the golden expected output deterministic.
for key, tmpl in partials {
t := tmpl
rel := strings.trim_prefix(t.path, #directory)
t.path = strings.trim_left(rel, "/")
partials[key] = t
}
return partials
}
diag_slug :: proc(name: string) -> string {
sb: strings.Builder
strings.builder_init(&sb, context.temp_allocator)
prev_underscore := false
for r in name {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
strings.write_rune(&sb, r)
prev_underscore = false
} else if r >= 'A' && r <= 'Z' {
strings.write_rune(&sb, r + 32)
prev_underscore = false
} else {
if !prev_underscore && strings.builder_len(sb) > 0 {
strings.write_rune(&sb, '_')
prev_underscore = true
}
}
}
s := strings.to_string(sb)
if len(s) > 0 && s[len(s) - 1] == '_' {
s = s[:len(s) - 1]
}
return s
}
render_silenced :: proc(
tmpl: Template,
data: any,
partials: map[string]Template = nil,
allocator := context.allocator,
warnings: ^[dynamic]Error = nil,
) -> (string, Error) {
saved := context.logger
context.logger = log.nil_logger()
result, err := render(tmpl, data, partials, allocator, warnings)
context.logger = saved
return result, err
}
run_diag_case :: proc(t: ^testing.T, name: string) {
cases := load_diag_cases("../DIAGNOSTIC_TESTS.yaml")
ctx := diag_ctx()
partials := diag_partials()
for c in cases {
if c.name != name do continue
switch c.should {
case "error":
tmpl, perr := parse(c.input, "<test>", allocator = context.temp_allocator)
if perr != nil {
formatted := format_render_error(perr, tmpl, colorize = false)
testing.expect_value(t, formatted, c.expected)
return
}
_, rerr := render_silenced(tmpl, ctx, partials, context.temp_allocator)
testing.expect(
t,
rerr != nil,
fmt.tprintf("[%s] should error but render succeeded", c.name),
)
if rerr == nil do return
formatted := format_render_error(rerr, tmpl, colorize = false)
testing.expect_value(t, formatted, c.expected)
case "pass":
tmpl, perr := parse(c.input, "<test>", allocator = context.temp_allocator)
testing.expect(
t,
perr == nil,
fmt.tprintf(
"[%s] should parse, got: %s",
c.name,
perr != nil ? format_render_error(perr, tmpl, colorize = false) : "",
),
)
if perr != nil do return
_, rerr := render_silenced(tmpl, ctx, partials, context.temp_allocator)
testing.expect(
t,
rerr == nil,
fmt.tprintf(
"[%s] should not error, got: %s",
c.name,
rerr != nil ? format_render_error(rerr, tmpl, colorize = false) : "",
),
)
case "ok":
tmpl, perr := parse(c.input, "<test>", allocator = context.temp_allocator)
testing.expect(
t,
perr == nil,
fmt.tprintf(
"[%s] should parse, got: %s",
c.name,
perr != nil ? format_render_error(perr, tmpl, colorize = false) : "",
),
)
if perr != nil do return
result, rerr := render_silenced(tmpl, ctx, partials, context.temp_allocator)
testing.expect(
t,
rerr == nil,
fmt.tprintf(
"[%s] should not error, got: %s",
c.name,
rerr != nil ? format_render_error(rerr, tmpl, colorize = false) : "",
),
)
if rerr != nil do return
testing.expect_value(t, result, c.expected)
case "warn":
tmpl, perr := parse(c.input, "<test>", allocator = context.temp_allocator)
testing.expect(
t,
perr == nil,
fmt.tprintf(
"[%s] should parse, got: %s",
c.name,
perr != nil ? format_render_error(perr, tmpl, colorize = false) : "",
),
)
if perr != nil do return
warnings := make([dynamic]Error, 0, 4, context.temp_allocator)
_, rerr := render_silenced(tmpl, ctx, partials, context.temp_allocator, &warnings)
testing.expect(
t,
rerr == nil,
fmt.tprintf(
"[%s] should not error, got: %s",
c.name,
rerr != nil ? format_render_error(rerr, tmpl, colorize = false) : "",
),
)
if rerr != nil do return
testing.expect(
t,
len(warnings) > 0,
fmt.tprintf("[%s] should produce a warning", c.name),
)
if len(warnings) == 0 do return
formatted := format_render_error(warnings[0], tmpl, colorize = false)
testing.expect_value(t, formatted, c.expected)
case:
testing.expectf(t, false, "[%s] unknown should value: '%s'", c.name, c.should)
}
return
}
testing.expectf(t, false, "case '%s' not found in DIAGNOSTIC_TESTS.yaml", name)
}
@(test)
write_diagnostic_tests :: proc(t: ^testing.T) {
cases := load_diag_cases("../DIAGNOSTIC_TESTS.yaml")
testing.expect(t, len(cases) > 0, "should load diagnostic cases")
sb: strings.Builder
strings.builder_init(&sb, context.temp_allocator)
strings.write_string(&sb, "#+test\n")
strings.write_string(&sb, "package mustache\n\n")
strings.write_string(&sb, "import \"core:testing\"\n\n")
count := 0
for c in cases {
if !c.enable do continue
if c.should == "" {
testing.expectf(t, false, "[%s] has enable: true but no should value", c.name)
continue
}
count += 1
slug := diag_slug(c.name)
strings.write_string(&sb, "@(test)\n")
strings.write_string(&sb, "test_diag_")
strings.write_string(&sb, slug)
strings.write_string(&sb, " :: proc(t: ^testing.T) {\n")
strings.write_string(&sb, "\trun_diag_case(t, \"")
strings.write_string(&sb, c.name)
strings.write_string(&sb, "\")\n}\n\n")
}
testing.expect(t, count > 0, "at least one case should be enabled")
generated := strings.to_string(sb)
dirs := [?]string{#directory, "diagnostic_test_cases_test.odin"}
gen_path, _ := filepath.join(dirs[:], context.temp_allocator)
existing_data, _ := os.read_entire_file_from_path(gen_path, context.temp_allocator)
existing := string(existing_data)
if existing == generated do return
werr := os.write_entire_file(gen_path, transmute([]byte)generated)
if werr != nil {
testing.expectf(t, false, "failed to write %s: %v", gen_path, werr)
return
}
testing.expectf(
t,
false,
"diagnostic_test_cases_test.odin regenerated (%d cases), re-run tests to see results",
count,
)
}
mustache/diagnostic_test_cases_test.odin
-45
View File
@@ -1,45 +0,0 @@
#+test
package mustache
import "core:testing"
@(test)
test_diag_empty_tag :: proc(t: ^testing.T) {
run_diag_case(t, "empty tag")
}
@(test)
test_diag_empty_raw_tag :: proc(t: ^testing.T) {
run_diag_case(t, "empty raw tag")
}
@(test)
test_diag_missing_pipe_arguement :: proc(t: ^testing.T) {
run_diag_case(t, "Missing pipe arguement")
}
@(test)
test_diag_first_last_with_zero_argument :: proc(t: ^testing.T) {
run_diag_case(t, "first/last with zero argument")
}
@(test)
test_diag_first_last_with_negative_argument :: proc(t: ^testing.T) {
run_diag_case(t, "first/last with negative argument")
}
@(test)
test_diag_first_succeeds_on_params_author_name :: proc(t: ^testing.T) {
run_diag_case(t, "first succeeds on params.author.name")
}
@(test)
test_diag_first_last_in_partial :: proc(t: ^testing.T) {
run_diag_case(t, "first/last in partial")
}
@(test)
test_diag_pipes_are_ignored_in_comments :: proc(t: ^testing.T) {
run_diag_case(t, "Pipes are ignored in comments")
}
mustache/fixtures/layouts/partials/footer.html
-12
View File
@@ -1,12 +0,0 @@
<footer>
<menu>
<a class="goto-top opacity-0" href="#">{{> icons/chevron_up}}</a>
{{#params.social}}
<a href="{{url}}" target="_blank" rel="noopener noreferrer me" title="{{name}}">{{>* icon}}</a>
{{/params.social}}
</menu>
<p>Proudly built with <a href="https://github.com/sbrow/thor/">Thor</a> and <a
href="https://edwardtufte.github.io/tufte-css/">Tufte CSS</a>
</p>
<p><small>&copy;</small> {{! now | format "2006" }} {{params.author.name | first -3 }}</p>
</footer>
mustache/mustache.odin
+25 -153
View File
@@ -5,8 +5,6 @@ import "core:fmt"
import "core:log"
import "core:strings"
import diags "../diagnostics"
// A hypothetical maximum context depth. Trying to pass more than this many items
// to render(tpl, data), or nesting templates further than this depth would be
// an error.
@@ -25,7 +23,6 @@ Context_Stack :: [dynamic]any
Error_Kind :: enum {
Syntax, // parse-time: malformed template
Data, // render-time: template fine, data wrong (e.g. filter misuse)
Warning, // non-fatal: operation succeeded but user should know
}
Error_Body :: struct {
@@ -156,22 +153,18 @@ parse :: proc(
tmpl: Template,
err: Error,
) {
tmpl.source = source
tmpl.path = path
tokens, terr := tokenize(source, tokens_allocator)
if terr != nil {
return tmpl, terr
return {}, terr
}
tmpl.nodes, err = parse_tokens(tokens[:], source, allocator)
if err != nil {
delete(tmpl.nodes)
e := body(err)
e.source = source
e.path = path
return {}, e
return {}, err
}
tmpl.source = source
tmpl.path = path
deindent_blocks(tmpl.nodes[:], allocator)
return tmpl, nil
}
@@ -181,8 +174,6 @@ render :: proc(
data: any,
partials: map[string]Template = nil,
allocator := context.allocator,
warnings: ^[dynamic]Error = nil,
tools: ^Tool_Registry = nil,
) -> (
result: string,
err: Error,
@@ -208,15 +199,7 @@ render :: proc(
}
all_nodes := tmpl.nodes[:]
err = render_nodes(
tmpl,
all_nodes,
&ctx,
partials,
&builder,
warnings = warnings,
tools = tools,
)
err = render_nodes(tmpl, all_nodes, &ctx, partials, &builder)
if err != nil {
return result, err
}
@@ -280,22 +263,9 @@ parse_section :: proc(
pos^ += 1
case .Variable:
if len(strings.trim_space(tok.value)) == 0 {
return Error_Body {
msg = "Empty tag",
pos = tok.pos,
kind = .Syntax,
hint = "tag must contain a key",
}
}
idx := len(nodes)
append(nodes, Node{kind = .Variable})
pipe_key, perr := parse_pipeline(
tok.value,
&nodes[idx].filters,
tok.pos,
tag_content_base(source, tok.pos),
)
pipe_key, perr := parse_pipeline(tok.value, &nodes[idx].filters, tok.pos, tag_content_base(source, tok.pos))
if perr != nil {
return Error_Body {
msg = fmt.tprintf("pipe parse error in '{{{{%s}}}}': %v", tok.value, perr),
@@ -307,22 +277,9 @@ parse_section :: proc(
pos^ += 1
case .Unescaped:
if len(strings.trim_space(tok.value)) == 0 {
return Error_Body {
msg = "Empty tag",
pos = tok.pos,
kind = .Syntax,
hint = "tag must contain a key",
}
}
idx := len(nodes)
append(nodes, Node{kind = .Unescaped})
pipe_key, perr := parse_pipeline(
tok.value,
&nodes[idx].filters,
tok.pos,
tag_content_base(source, tok.pos),
)
pipe_key, perr := parse_pipeline(tok.value, &nodes[idx].filters, tok.pos, tag_content_base(source, tok.pos))
if perr != nil {
return Error_Body {
msg = fmt.tprintf("pipe parse error in '{{{{&%s}}}}': %v", tok.value, perr),
@@ -342,12 +299,7 @@ parse_section :: proc(
content_start := 0
if pos^ < len(tokens) {content_start = tokens[pos^].pos}
append(nodes, Node{kind = .Section, pos = tok.pos})
pipe_key, perr := parse_pipeline(
tok.value,
&nodes[idx].filters,
tok.pos,
tag_content_base(source, tok.pos),
)
pipe_key, perr := parse_pipeline(tok.value, &nodes[idx].filters, tok.pos, tag_content_base(source, tok.pos))
if perr != nil {
return Error_Body {
msg = fmt.tprintf("pipe parse error in '{{{{#%s}}}}': %v", tok.value, perr),
@@ -368,12 +320,7 @@ parse_section :: proc(
content_start := 0
if pos^ < len(tokens) {content_start = tokens[pos^].pos}
append(nodes, Node{kind = .Inverted, pos = tok.pos})
pipe_key, perr := parse_pipeline(
tok.value,
&nodes[idx].filters,
tok.pos,
tag_content_base(source, tok.pos),
)
pipe_key, perr := parse_pipeline(tok.value, &nodes[idx].filters, tok.pos, tag_content_base(source, tok.pos))
if perr != nil {
return Error_Body {
msg = fmt.tprintf("pipe parse error in '{{{{^%s}}}}': %v", tok.value, perr),
@@ -596,8 +543,6 @@ render_template :: proc(
b: ^strings.Builder,
blocks: map[string]Block_Override,
indent: string,
warnings: ^[dynamic]Error = nil,
tools: ^Tool_Registry = nil,
) -> Error {
if len(indent) > 0 {
state := Indent_State {
@@ -605,9 +550,9 @@ render_template :: proc(
at_line_start = false,
}
strings.write_string(b, indent) // first line always gets indent
return render_nodes(pt, pt.nodes[:], ctx, partials, b, blocks, &state, warnings, tools)
return render_nodes(pt, pt.nodes[:], ctx, partials, b, blocks, &state)
}
return render_nodes(pt, pt.nodes[:], ctx, partials, b, blocks, nil, warnings, tools)
return render_nodes(pt, pt.nodes[:], ctx, partials, b, blocks, nil)
}
write_indented :: proc(
@@ -651,8 +596,6 @@ render_nodes :: proc(
b: ^strings.Builder,
blocks: map[string]Block_Override = nil,
indent_state: ^Indent_State = nil,
warnings: ^[dynamic]Error = nil,
tools: ^Tool_Registry = nil,
) -> Error {
i := 0
for i < len(nodes) {
@@ -676,15 +619,7 @@ render_nodes :: proc(
warn_unknown_key(current, ctx[:], node)
}
if len(node.filters) > 0 {
transformed, perr := apply_pipeline(
val,
node.filters[:],
node.pos,
ctx[:],
current,
warnings,
tools,
)
transformed, perr := apply_pipeline(val, node.filters[:], node.pos, ctx[:])
if perr != nil {
return tag_error(perr, current)
}
@@ -703,15 +638,7 @@ render_nodes :: proc(
warn_unknown_key(current, ctx[:], node)
}
if len(node.filters) > 0 {
transformed, perr := apply_pipeline(
val,
node.filters[:],
node.pos,
ctx[:],
current,
warnings,
tools,
)
transformed, perr := apply_pipeline(val, node.filters[:], node.pos, ctx[:])
if perr != nil {
return tag_error(perr, current)
}
@@ -726,15 +653,7 @@ render_nodes :: proc(
warn_unknown_key(current, ctx[:], node)
}
if len(node.filters) > 0 {
transformed, perr := apply_pipeline(
val,
node.filters[:],
node.pos,
ctx[:],
current,
warnings,
tools,
)
transformed, perr := apply_pipeline(val, node.filters[:], node.pos, ctx[:])
if perr != nil {
return tag_error(perr, current)
}
@@ -756,8 +675,6 @@ render_nodes :: proc(
b,
blocks,
indent_state,
warnings,
tools,
) or_return
}
} else {
@@ -771,7 +688,6 @@ render_nodes :: proc(
b,
blocks,
indent_state,
warnings,
) or_return
}
}
@@ -783,15 +699,7 @@ render_nodes :: proc(
warn_unknown_key(current, ctx[:], node)
}
if len(node.filters) > 0 {
transformed, perr := apply_pipeline(
val,
node.filters[:],
node.pos,
ctx[:],
current,
warnings,
tools,
)
transformed, perr := apply_pipeline(val, node.filters[:], node.pos, ctx[:])
if perr != nil {
return tag_error(perr, current)
}
@@ -806,8 +714,6 @@ render_nodes :: proc(
b,
blocks,
indent_state,
warnings,
tools,
) or_return
}
i += 1 + len(node.children)
@@ -822,7 +728,7 @@ render_nodes :: proc(
if !found {
warn_missing_partial(current, partials, node, name)
} else {
render_template(pt, ctx, partials, b, nil, node.indent, warnings, tools) or_return
render_template(pt, ctx, partials, b, nil, node.indent) or_return
if indent_state != nil {
indent_state.at_line_start = false
}
@@ -857,8 +763,6 @@ render_nodes :: proc(
&temp,
content_blocks,
nil,
warnings,
tools,
) or_return
at_ls := true
write_indented(b, node.indent, strings.to_string(temp), &at_ls)
@@ -871,8 +775,6 @@ render_nodes :: proc(
b,
content_blocks,
indent_state,
warnings,
tools,
) or_return
}
i += 1 + len(node.children)
@@ -885,16 +787,7 @@ render_nodes :: proc(
warn_missing_partial(current, partials, node, node.key)
} else {
warn_unmatched_block_overrides(current, pt, parent_children)
render_template(
pt,
ctx,
partials,
b,
merged,
node.indent,
warnings,
tools,
) or_return
render_template(pt, ctx, partials, b, merged, node.indent) or_return
if indent_state != nil {
indent_state.at_line_start = false
}
@@ -949,7 +842,7 @@ warn_unknown_key :: proc(current: Template, ctx: []any, node: Node) {
hint := ""
if len(available) > 0 {
suggestion := diags.suggest_correction(available, missing)
suggestion := suggest_correction(available, missing)
if suggestion != "" {
hint = fmt.tprintf("did you mean '%s'?", suggestion)
}
@@ -959,14 +852,7 @@ warn_unknown_key :: proc(current: Template, ctx: []any, node: Node) {
if path == "" {
path = "<input>"
}
diag := format_tag_error(
path,
current.source,
node.pos,
msg,
hint,
colorize = diags.should_colorize(),
)
diag := format_error(path, current.source, node.pos, msg, hint, colorize = should_colorize())
log.warnf("%s", diag)
}
@@ -980,7 +866,7 @@ warn_missing_partial :: proc(
) {
hint := ""
available := collect_partial_names(partials)
suggestion := diags.suggest_correction(available, name)
suggestion := suggest_correction(available, name)
if suggestion != "" {
hint = fmt.tprintf("did you mean '%s'?", suggestion)
}
@@ -989,14 +875,7 @@ warn_missing_partial :: proc(
if path == "" {
path = "<input>"
}
diag := format_tag_error(
path,
current.source,
node.pos,
msg,
hint,
colorize = diags.should_colorize(),
)
diag := format_error(path, current.source, node.pos, msg, hint, colorize = should_colorize())
log.warnf("%s", diag)
}
@@ -1032,7 +911,7 @@ warn_unmatched_block_overrides :: proc(
}
hint := ""
suggestion := diags.suggest_correction(available, child.key)
suggestion := suggest_correction(available, child.key)
if suggestion != "" {
hint = fmt.tprintf("did you mean '%s'?", suggestion)
}
@@ -1045,13 +924,13 @@ warn_unmatched_block_overrides :: proc(
if path == "" {
path = "<input>"
}
diag := format_tag_error(
diag := format_error(
path,
current.source,
child.pos,
msg,
hint,
colorize = diags.should_colorize(),
colorize = should_colorize(),
)
log.warnf("%s", diag)
}
@@ -1075,13 +954,6 @@ warn_context_depth :: proc(current: Template, node: Node) {
if path == "" {
path = "<input>"
}
diag := format_tag_error(
path,
current.source,
node.pos,
msg,
"",
colorize = diags.should_colorize(),
)
diag := format_error(path, current.source, node.pos, msg, "", colorize = should_colorize())
log.warnf("%s", diag)
}
mustache/mustache_test.odin
+4 -4
View File
@@ -1,6 +1,7 @@
#+test
package mustache
import "core:log"
import "core:mem"
import "core:strings"
import "core:testing"
@@ -127,9 +128,10 @@ leak_repeated_render :: proc(t: ^testing.T) {
// A deeply nested map pushes the context stack past MAX_CONTEXT_DEPTH (16).
// The depth warning must be non-fatal: rendering still succeeds.
/*
@(test)
test_context_depth_warns :: proc(t: ^testing.T) {
context.logger = log.nil_logger()
AMT :: MAX_CONTEXT_DEPTH + 2
// Build AMT nested {x: {...}} levels; the innermost holds `leaf`.
@@ -155,9 +157,7 @@ test_context_depth_warns :: proc(t: ^testing.T) {
return
}
result, rerr := render_silenced(tmpl, data, allocator = context.temp_allocator)
result, rerr := render(tmpl, data, allocator = context.temp_allocator)
testing.expect(t, rerr == nil, "depth warning must be non-fatal")
testing.expect_value(t, result, "found")
}
*/
mustache/partials.odin
-48
View File
@@ -1,48 +0,0 @@
package mustache
import "core:fmt"
import "core:log"
import "core:os"
import "core:strings"
load_partials :: proc(
dir: string,
base: string = "",
allocator := context.allocator,
) -> map[string]Template {
partials := make(map[string]Template, allocator)
entries, err := os.read_all_directory_by_path(dir, context.temp_allocator)
if err != nil {
log.warnf("cannot read partials dir %s: %v", dir, err)
return partials
}
defer os.file_info_slice_delete(entries, context.temp_allocator)
for entry in entries {
if entry.type == .Regular {
if !strings.has_suffix(entry.name, ".html") do continue
stripped := entry.name[:len(entry.name) - len(".html")]
key := fmt.tprintf("%s%s", base, stripped)
data, ferr := os.read_entire_file_from_path(entry.fullpath, allocator)
if ferr != nil do continue
source := string(data)
path := strings.clone(entry.fullpath, allocator)
tmpl, perr := parse(source, path, allocator = allocator)
if perr != nil {
b := body(perr)
log.warnf("failed to parse partial %s: %s", entry.fullpath, b.msg)
continue
}
partials[key] = tmpl
} else if entry.type == .Directory {
sub_base := fmt.tprintf("%s%s/", base, entry.name)
sub := load_partials(entry.fullpath, sub_base, allocator)
for k, v in sub {
partials[k] = v
}
}
}
return partials
}
mustache/pipes.odin
+13 -251
View File
@@ -4,10 +4,8 @@ import "base:runtime"
import "core:fmt"
import "core:log"
import "core:reflect"
import "core:strconv"
import "core:strings"
import diags "../diagnostics"
import "core:time"
import "core:time/datetime"
// MAX_PIPES was chosen arbitrarily. It holds no performance or logical
@@ -22,10 +20,6 @@ DEFAULT_DATE_FORMAT :: "2 Jan 2006"
Pipe_Op :: enum {
Format,
Group_By,
First,
Last,
Rel_Url,
Tool,
}
// Pipe op names are derived from the enum via reflection (lowercased).
@@ -65,7 +59,6 @@ Pipe_Filter :: struct {
op: string,
args: [dynamic; MAX_PIPE_ARGS]string,
op_pos: int,
end_pos: int,
}
Group :: struct {
@@ -138,7 +131,11 @@ parse_pipeline :: proc(
pipe_count := strings.count(content, "|")
if pipe_count > MAX_PIPES {
return "", Error_Body {
msg = fmt.tprintf("pipe expression has %d filters, max is %d", pipe_count, MAX_PIPES),
msg = fmt.tprintf(
"pipe expression has %d filters, max is %d",
pipe_count,
MAX_PIPES,
),
pos = pos,
kind = .Syntax,
}
@@ -202,7 +199,6 @@ parse_pipeline :: proc(
filter := Pipe_Filter {
op = tokens[0],
op_pos = content_base + op_offset_in_content,
end_pos = content_base + op_offset_in_content + len(seg),
}
for j in 1 ..< len(tokens) {
append(&filter.args, tokens[j])
@@ -224,28 +220,14 @@ apply_pipeline :: proc(
filters: []Pipe_Filter,
pos: int,
ctx: []any,
tmpl: Template,
warnings: ^[dynamic]Error = nil,
tools: ^Tool_Registry = nil,
) -> (
current: any,
err: Error,
) {
current = value
for &filter in filters {
result, ferr := apply_filter(current, &filter, pos, ctx, tmpl, tools)
if ferr != nil {
b := body(ferr)
if b.kind != .Warning {
return result, ferr
}
if warnings != nil {
append(warnings, ferr)
} else {
log.warnf("%s", format_render_error(ferr, tmpl))
}
}
current = result
current = apply_filter(current, &filter, pos, ctx) or_return
log.debugf("applied: filter=%v before=%s after=%s pos=%d", filter, value, current, pos)
}
return
}
@@ -274,21 +256,11 @@ resolve_format_string :: proc(name: string, ctx: []any, pos: int) -> (string, Er
return str, nil
}
apply_filter :: proc(
value: any,
filter: ^Pipe_Filter,
pos: int,
ctx: []any,
tmpl: Template,
tools: ^Tool_Registry = nil,
) -> (
any,
Error,
) {
apply_filter :: proc(value: any, filter: ^Pipe_Filter, pos: int, ctx: []any) -> (any, Error) {
op, ok := pipe_op_from_string(filter.op)
if !ok {
hint := ""
suggestion := diags.suggest_correction(pipe_op_candidates(), filter.op)
suggestion := suggest_correction(pipe_op_candidates(), filter.op)
if suggestion != "" {
hint = fmt.tprintf("did you mean '%s'?", suggestion)
}
@@ -303,8 +275,7 @@ apply_filter :: proc(
switch op {
case .Group_By:
span := filter.end_pos - filter.op_pos
return apply_group_by(value, filter.args[:], filter.op_pos, span)
return apply_group_by(value, filter.args[:], pos)
case .Format:
str, ok := reflect.as_string(value)
if !ok {
@@ -343,89 +314,10 @@ apply_filter :: proc(
} else {
return any{new_clone(str2, context.temp_allocator), typeid_of(string)}, nil
}
case .First:
span := filter.end_pos - filter.op_pos
return take(value, filter.args[:], true, tmpl, filter.op_pos, span)
case .Last:
span := filter.end_pos - filter.op_pos
return take(value, filter.args[:], false, tmpl, filter.op_pos, span)
case .Rel_Url:
bv, _ := base_value(value)
str, ok := reflect.as_string(bv)
if !ok {
return value, Error_Body {
msg = "rel_url may only be used on strings",
pos = pos,
kind = .Data,
}
}
result := rel_url(str, ctx)
return any{new_clone(result, context.temp_allocator), typeid_of(string)}, nil
case .Tool:
if len(filter.args) != 1 {
return value, Error_Body {
msg = fmt.tprintf(
"tool expects 1 argument (the tool name), got %d",
len(filter.args),
),
pos = filter.op_pos,
span = filter.end_pos - filter.op_pos,
kind = .Data,
}
}
bv, _ := base_value(value)
str, ok := reflect.as_string(bv)
if !ok {
return value, Error_Body {
msg = "tool may only be used on strings",
pos = pos,
kind = .Data,
}
}
result, terr := run_tool(tools, filter.args[0], str, filter.op_pos)
if terr != nil {
return value, terr
}
return any{new_clone(result, context.temp_allocator), typeid_of(string)}, nil
}
return {}, nil
}
// rel_url prefixes a root-relative path (e.g. "/faq/") with the site's
// base_path so links resolve correctly under a subpath deployment. Values that
// are already absolute — a scheme ("http://"), protocol-relative ("//"), an
// in-page fragment ("#"), or a "mailto:" — are returned unchanged, which keeps
// external menu and footer links intact. base_path is read from the context
// stack the same way `format` reads date_format/timezone.
rel_url :: proc(path: string, ctx: []any) -> string {
if path == "" {
return path
}
if strings.has_prefix(path, "http://") ||
strings.has_prefix(path, "https://") ||
strings.has_prefix(path, "//") ||
strings.has_prefix(path, "#") ||
strings.has_prefix(path, "mailto:") {
return path
}
base := ""
if raw := resolve_name("base_path", ctx); raw != nil {
if s, ok := reflect.as_string(raw); ok {
base = s
}
}
if base == "" {
return path
}
// Only prefix root-relative paths; leave document-relative ones alone.
if path[0] == '/' {
return fmt.tprintf("%s%s", base, path)
}
return path
}
apply_format :: proc(
iso: string,
args: []string,
@@ -463,40 +355,19 @@ apply_format :: proc(
}
// Groups preserve first-appearance order from the input list.
apply_group_by :: proc(
value: any,
args: []string,
pos: int,
span: int,
) -> (
result: any,
err: Error,
) {
apply_group_by :: proc(value: any, args: []string, pos: int) -> (result: any, err: Error) {
if len(args) != 1 {
hint := ""
if len(args) == 0 {
hint = "missing argument"
} else {
hint = "too many arguments"
}
return nil, Error_Body {
msg = fmt.tprintf("group_by expects 1 argument, got %d", len(args)),
pos = pos,
span = span,
kind = .Data,
hint = hint,
}
}
field := args[0]
elem_info, count, data := list_info(value)
if elem_info == nil {
return nil, Error_Body {
msg = "group_by expects a list",
pos = pos,
span = span,
kind = .Data,
}
return nil, Error_Body{msg = "group_by expects a list", pos = pos, kind = .Data}
}
groups := make([dynamic]Group, 0, 8, context.temp_allocator)
@@ -538,112 +409,3 @@ apply_group_by :: proc(
return groups, nil
}
take :: proc(
value: any,
args: []string,
from_start: bool,
tmpl: Template,
op_pos: int,
span: int,
) -> (
any,
Error,
) {
unwrapped, _ := base_value(value)
elem_info, count, data := list_info(unwrapped)
is_list := elem_info != nil
from_start := from_start
warning: Error = nil
str: string
if !is_list {
s, is_str := reflect.as_string(unwrapped)
if !is_str {
return nil, Error_Body {
msg = "requires a list or string",
pos = op_pos,
span = span,
kind = .Data,
source = tmpl.source,
path = tmpl.path,
}
}
str = s
count = len(str)
}
n := 1
if !is_list && len(args) == 0 {
warning = Error_Body {
msg = "defaulting to n=1 for string",
pos = op_pos,
span = span,
kind = .Warning,
source = tmpl.source,
path = tmpl.path,
hint = "consider specifying n explicitly",
}
}
if len(args) > 0 {
parsed, ok := strconv.parse_int(args[0])
if !ok {
return nil, Error_Body {
msg = fmt.tprintf("invalid argument '%s'", args[0]),
pos = op_pos,
span = span,
kind = .Data,
source = tmpl.source,
path = tmpl.path,
}
}
n = parsed
}
if n == 0 {
return nil, Error_Body {
msg = "argument must not be zero",
pos = op_pos,
span = span,
kind = .Data,
source = tmpl.source,
path = tmpl.path,
}
}
if n < 0 {
op_name := from_start ? "first" : "last"
alt_name := from_start ? "last" : "first"
warning = Error_Body {
msg = fmt.tprintf("%s %d is equivalent to %s %d", op_name, n, alt_name, 0 - n),
pos = op_pos,
span = span,
kind = .Warning,
source = tmpl.source,
path = tmpl.path,
hint = "consider using the clearer form",
}
from_start = !from_start
n = 0 - n
}
start: int
end: int
if from_start {
start = 0
end = min(count, n)
} else {
start = max(0, count - n)
end = count
}
if is_list {
result := make([dynamic]any, 0, max(0, end - start), context.temp_allocator)
for j in start ..< end {
append(&result, extract_list_element(elem_info, data, j))
}
return any{new_clone(result, context.temp_allocator), typeid_of([dynamic]any)}, warning
} else {
return any{new_clone(str[start:end], context.temp_allocator), typeid_of(string)}, warning
}
}
mustache/pipes_test.odin
+4 -145
View File
@@ -1,4 +1,3 @@
#+feature dynamic-literals
#+test
package mustache
@@ -568,11 +567,8 @@ test_format_mst_no_offset_no_timezone :: proc(t: ^testing.T) {
@(test)
test_unknown_pipe_op_suggestion :: proc(t: ^testing.T) {
filter := Pipe_Filter {
op = "formats",
op_pos = 0,
}
_, err := apply_filter("2026-01-15", &filter, 0, nil, {})
filter := Pipe_Filter{op = "formats", op_pos = 0}
_, err := apply_filter("2026-01-15", &filter, 0, nil)
testing.expect(t, err != nil, "should error on unknown op")
b := body(err)
testing.expect(t, strings.contains(b.hint, "format"), "hint should suggest 'format'")
@@ -581,146 +577,9 @@ test_unknown_pipe_op_suggestion :: proc(t: ^testing.T) {
@(test)
test_unknown_pipe_op_no_suggestion :: proc(t: ^testing.T) {
filter := Pipe_Filter {
op = "xyz",
op_pos = 0,
}
_, err := apply_filter("2026-01-15", &filter, 0, nil, {})
filter := Pipe_Filter{op = "xyz", op_pos = 0}
_, err := apply_filter("2026-01-15", &filter, 0, nil)
testing.expect(t, err != nil, "should error on unknown op")
b := body(err)
testing.expect(t, b.hint == "", "no suggestion expected for 'xyz'")
}
// Group 1: Lists
First_Last_List_Data: map[string][5]string = {
"items" = [5]string{"foo", "bar", "bat", "qux", "quz"},
}
@(test)
test_first_list_no_args_equals_first_one :: proc(t: ^testing.T) {
tpl_a, _ := parse(
"{{#items | first}}<li>{{.}}</li>{{/items}}",
"<test>",
allocator = context.temp_allocator,
)
result_a, _ := render(tpl_a, First_Last_List_Data, {}, context.temp_allocator)
tpl_b, _ := parse(
"{{#items | first 1}}<li>{{.}}</li>{{/items}}",
"<test>",
allocator = context.temp_allocator,
)
result_b, _ := render(tpl_b, First_Last_List_Data, {}, context.temp_allocator)
testing.expect_value(t, result_a, "<li>foo</li>")
testing.expect_value(t, result_a, result_b)
}
@(test)
test_first_list_n :: proc(t: ^testing.T) {
data := First_Last_List_Data
tpl, _ := parse(
"{{#items | first 3}}<li>{{.}}</li>{{/items}}",
"<test>",
allocator = context.temp_allocator,
)
result, _ := render(tpl, data, {}, context.temp_allocator)
testing.expect_value(t, result, "<li>foo</li><li>bar</li><li>bat</li>")
}
@(test)
test_first_list_negative_equals_last :: proc(t: ^testing.T) {
data := First_Last_List_Data
tpl, _ := parse(
"{{#items | first -2}}<li>{{.}}</li>{{/items}}",
"<test>",
allocator = context.temp_allocator,
)
result, _ := render(tpl, data, {}, context.temp_allocator)
testing.expect_value(t, result, "<li>qux</li><li>quz</li>")
}
@(test)
test_last_list_no_args_equals_last_one :: proc(t: ^testing.T) {
data := First_Last_List_Data
tpl_a, _ := parse(
"{{#items | last}}<li>{{.}}</li>{{/items}}",
"<test>",
allocator = context.temp_allocator,
)
result_a, _ := render(tpl_a, data, {}, context.temp_allocator)
tpl_b, _ := parse(
"{{#items | last 1}}<li>{{.}}</li>{{/items}}",
"<test>",
allocator = context.temp_allocator,
)
result_b, _ := render(tpl_b, data, {}, context.temp_allocator)
testing.expect_value(t, result_a, "<li>quz</li>")
testing.expect_value(t, result_b, "<li>quz</li>")
}
@(test)
test_last_list_n :: proc(t: ^testing.T) {
data := First_Last_List_Data
tpl, _ := parse(
"{{#items | last 3}}<li>{{.}}</li>{{/items}}",
"<test>",
allocator = context.temp_allocator,
)
result, _ := render(tpl, data, {}, context.temp_allocator)
testing.expect_value(t, result, "<li>bat</li><li>qux</li><li>quz</li>")
}
@(test)
test_last_list_negative_equals_first :: proc(t: ^testing.T) {
data := First_Last_List_Data
tpl, _ := parse(
"{{#items | last -2}}<li>{{.}}</li>{{/items}}",
"<test>",
allocator = context.temp_allocator,
)
result, _ := render(tpl, data, {}, context.temp_allocator)
testing.expect_value(t, result, "<li>foo</li><li>bar</li>")
}
// Group 2: Strings
First_Last_String_Data := map[string]string {
"word" = "Hello, World!",
}
@(test)
test_first_string_n :: proc(t: ^testing.T) {
data := First_Last_String_Data
tpl, _ := parse("{{word | first 5}}", "<test>", allocator = context.temp_allocator)
result, _ := render(tpl, data, {}, context.temp_allocator)
testing.expect_value(t, result, "Hello")
}
@(test)
test_last_string_n :: proc(t: ^testing.T) {
data := First_Last_String_Data
tpl, _ := parse("{{word | last 6}}", "<test>", allocator = context.temp_allocator)
result, _ := render(tpl, data, {}, context.temp_allocator)
testing.expect_value(t, result, "World!")
}
@(test)
test_first_string_negative_equals_last :: proc(t: ^testing.T) {
data := First_Last_String_Data
tpl, _ := parse("{{word | first -6}}", "<test>", allocator = context.temp_allocator)
result, _ := render(tpl, data, {}, context.temp_allocator)
testing.expect_value(t, result, "World!")
}
@(test)
test_last_string_negative_equals_first :: proc(t: ^testing.T) {
data := First_Last_String_Data
tpl, _ := parse("{{word | last -5}}", "<test>", allocator = context.temp_allocator)
result, _ := render(tpl, data, {}, context.temp_allocator)
testing.expect_value(t, result, "Hello")
}
mustache/spec_test.odin
+6 -1
View File
@@ -3,6 +3,7 @@ package mustache
import "core:encoding/json"
import "core:fmt"
import "core:log"
import "core:os"
import "core:path/filepath"
import "core:testing"
@@ -54,6 +55,10 @@ run_one_test :: proc(
test: json.Object,
name, template_src, expected: string,
) -> bool {
// Spec tests deliberately exercise missing keys, missing partials, etc.
// Silence the warnings to keep test output readable.
context.logger = log.nil_logger()
partials := make_map(map[string]Template, context.temp_allocator)
if "partials" in test {
@@ -79,7 +84,7 @@ run_one_test :: proc(
}
defer delete(tmpl.nodes)
result, rerr := render_silenced(tmpl, test["data"], partials)
result, rerr := render(tmpl, test["data"], partials)
if rerr != nil {
testing.expectf(t, false, "[%s] render error", name)
return false
mustache/suggest.odin
+25 -3
View File
@@ -3,8 +3,6 @@ package mustache
import "base:runtime"
import "core:strings"
import diags "../diagnostics"
// collect_struct_keys enumerates the visible field names of a struct value,
// including fields promoted via `using`-embedded structs.
collect_struct_keys :: proc(val: any, allocator := context.temp_allocator) -> []string {
@@ -132,7 +130,7 @@ validate_key_path :: proc(
continue
}
available := collect_map_keys(current, allocator)
if diags.suggest_correction(available, parts[i]) != "" {
if suggest_correction(available, parts[i]) != "" {
return false, parts[i], available
}
return true, "", nil
@@ -157,6 +155,30 @@ validate_key_path :: proc(
return true, "", nil
}
// suggest_correction returns the closest match from `available` to `missing`
// using Levenshtein distance, or "" if no good match exists. The threshold
// scales with the length of the missing key.
suggest_correction :: proc(available: []string, missing: string) -> string {
if len(available) == 0 || len(missing) == 0 {
return ""
}
threshold := max(2, len(missing) / 3)
best: string
best_dist := threshold + 1
for candidate in available {
if abs(len(candidate) - len(missing)) > threshold {
continue
}
d := strings.levenshtein_distance(missing, candidate)
if d <= threshold && d < best_dist {
best = candidate
best_dist = d
}
}
return best
}
// collect_partial_names enumerates the keys of the partials map.
collect_partial_names :: proc(
partials: map[string]Template,
mustache/suggest_test.odin
+25 -3
View File
@@ -6,8 +6,6 @@ import "core:encoding/json"
import "core:fmt"
import "core:testing"
import diags "../diagnostics"
Inner :: struct {
foo: string,
bar: int,
@@ -155,6 +153,30 @@ test_validate_map_path_silent :: proc(t: ^testing.T) {
testing.expect_value(t, ok, true)
}
@(test)
test_suggest_correction_exact :: proc(t: ^testing.T) {
available := []string{"title", "page.title", "body"}
testing.expect_value(t, suggest_correction(available, "page_titel"), "page.title")
}
@(test)
test_suggest_correction_close :: proc(t: ^testing.T) {
available := []string{"title", "body", "now"}
testing.expect_value(t, suggest_correction(available, "titel"), "title")
}
@(test)
test_suggest_correction_no_match :: proc(t: ^testing.T) {
available := []string{"completely_different", "unrelated"}
testing.expect_value(t, suggest_correction(available, "page_titel"), "")
}
@(test)
test_suggest_correction_empty :: proc(t: ^testing.T) {
testing.expect_value(t, suggest_correction([]string{}, "anything"), "")
testing.expect_value(t, suggest_correction([]string{"a"}, ""), "")
}
@(test)
test_warn_no_false_positive_for_valid_keys :: proc(t: ^testing.T) {
Data :: struct {
@@ -222,7 +244,7 @@ test_validate_key_path_map_typo :: proc(t: ^testing.T) {
ok, missing, available := validate_key_path(ctx[:], "params.authr")
testing.expect(t, !ok, "typo of map key should not be ok")
testing.expect_value(t, missing, "authr")
suggestion := diags.suggest_correction(available, "authr")
suggestion := suggest_correction(available, "authr")
testing.expect_value(t, suggestion, "author")
}
mustache/tools.odin
-319
View File
@@ -1,319 +0,0 @@
package mustache
import "core:fmt"
import "core:os"
import "core:strings"
import "core:sync"
// The `tool` pipe runs a user-configured external program over a site asset.
// A tool is a command string that is itself a mustache template; rendering it
// yields the command line, which is tokenized into argv and run via
// os.process_exec (no shell). The command template has three variables:
//
// {{in}} absolute path to the source file the tool should read
// {{out}} absolute path the tool should write its result to
// {{#minify}} section, non-empty when the site is building with minify on
//
// {{in}}/{{out}} are optional per tool. If a command omits {{in}}, thor feeds
// the source file on the tool's stdin; if it omits {{out}}, thor captures the
// tool's stdout. Stdin/stdout is the preferred form (it will compose best with
// the future `fingerprint` pipe); {{in}}/{{out}} is the fallback for tools that
// can't stream. The pipe takes an asset path (e.g. "/css/main.css"), runs the
// tool, and returns the root-relative URL of the written output.
// Tool_Registry is the host-supplied configuration for the `tool` pipe. The
// host (thor) owns one per build and threads a pointer to it through render, so
// it never enters the template context stack and can't be resolved as a key.
// The config fields are written once before rendering and only read thereafter;
// `manifest` is the sole mutable state and is guarded by `mu` — needed because
// render may go parallel and because os.process_start is not thread-safe.
Tool_Registry :: struct {
commands: map[string]string, // tool name -> command template
input_dir: string, // base dir for resolving {{in}} (the assets dir)
output_dir: string, // base dir for the written output (the output dir)
minify: bool,
manifest: map[string]string, // (name\x00input) -> output URL, per build
mu: sync.Mutex,
}
// tool_registry_init prepares `reg` for one build: the input/output base dirs,
// the minify flag, and a fresh (empty) manifest. `commands` is configured
// separately (from thor.json) and left untouched. The manifest is temp-allocated
// — build-scoped, discarded when the build's temp allocator is reset. Nothing
// needs to be cleaned up between builds: the host recreates the whole registry.
tool_registry_init :: proc(reg: ^Tool_Registry, input_dir, output_dir: string, minify: bool) {
reg.input_dir = input_dir
reg.output_dir = output_dir
reg.minify = minify
reg.manifest = make(map[string]string, context.temp_allocator)
}
// run_tool executes the named tool over `input` (an asset path) and returns the
// root-relative URL of the produced file. Any failure is a hard error (.Data).
run_tool :: proc(
reg: ^Tool_Registry,
name: string,
input: string,
pos: int,
) -> (
url: string,
err: Error,
) {
if reg == nil {
return "", Error_Body{msg = "tool pipe is not available here", pos = pos, kind = .Data}
}
// Absolute URLs (CDN links, protocol-relative) have no local source; pass
// them through so a mixed stylesheet list stays safe.
if is_absolute_url(input) {
return input, nil
}
cmd_tmpl, ok := reg.commands[name]
if !ok {
return "", Error_Body {
msg = fmt.tprintf("unknown tool '%s' (add it to thor.json \"tools\")", name),
pos = pos,
kind = .Data,
}
}
// Logical path -> in/out/url. Strip a single leading '/'.
rel := input
if len(rel) > 0 && rel[0] == '/' {
rel = rel[1:]
}
if rel == "" {
return "", Error_Body{msg = "tool: empty asset path", pos = pos, kind = .Data}
}
in_path := fmt.tprintf("%s/%s", reg.input_dir, rel)
out_path := fmt.tprintf("%s/%s", reg.output_dir, rel)
out_url := fmt.tprintf("/%s", rel)
if !os.exists(in_path) {
return "", Error_Body {
msg = fmt.tprintf("tool '%s': source file not found: %s", name, in_path),
pos = pos,
kind = .Data,
}
}
// Render the command template. Done OUTSIDE the lock so a command template
// that (mistakenly) uses a `tool` pipe can't deadlock on reg.mu. The
// registry fields it reads are immutable during rendering.
cmd_str, rerr := render_command(cmd_tmpl, in_path, out_path, reg.minify, pos)
if rerr != nil {
return "", rerr
}
argv := tokenize_command(cmd_str, context.temp_allocator)
if len(argv) == 0 {
return "", Error_Body {
msg = fmt.tprintf("tool '%s': command is empty", name),
pos = pos,
kind = .Data,
}
}
if !find_on_path(argv[0]) {
return "", Error_Body {
msg = fmt.tprintf("tool '%s': program '%s' not found on PATH", name, argv[0]),
pos = pos,
kind = .Data,
}
}
sync.lock(&reg.mu)
defer sync.unlock(&reg.mu)
key := fmt.tprintf("%s\x00%s", name, input)
if cached, hit := reg.manifest[key]; hit {
return cached, nil
}
// How the tool speaks I/O is inferred from whether the command references
// the paths thor substituted:
// {{in}} present -> the tool opens the source file itself
// {{in}} absent -> thor feeds the source file on stdin
// {{out}} present -> the tool writes the output file itself
// {{out}} absent -> thor captures stdout and writes it to out_path
// Either way the run ends with a real file at out_path.
uses_in := strings.contains(cmd_str, in_path)
uses_out := strings.contains(cmd_str, out_path)
// Ensure the output's parent dir exists (for the tool's {{out}} write or
// thor's stdout write).
if idx := strings.last_index(out_path, "/"); idx >= 0 {
if merr := os.make_directory_all(out_path[:idx]); merr != nil && merr != .Exist {
return "", Error_Body {
msg = fmt.tprintf("tool '%s': cannot create %s: %v", name, out_path[:idx], merr),
pos = pos,
kind = .Data,
}
}
}
desc := os.Process_Desc {
command = argv,
}
// Feed the source on stdin unless the tool takes it as a {{in}} path.
stdin_file: ^os.File
if !uses_in {
f, oerr := os.open(in_path)
if oerr != nil {
return "", Error_Body {
msg = fmt.tprintf("tool '%s': cannot open %s: %v", name, in_path, oerr),
pos = pos,
kind = .Data,
}
}
stdin_file = f
desc.stdin = f
}
defer if stdin_file != nil {
os.close(stdin_file)
}
state, stdout, stderr, xerr := os.process_exec(desc, context.temp_allocator)
if xerr != nil {
return "", Error_Body {
msg = fmt.tprintf("tool '%s': failed to run '%s': %v", name, argv[0], xerr),
pos = pos,
kind = .Data,
}
}
if state.exit_code != 0 {
return "", Error_Body {
msg = fmt.tprintf(
"tool '%s': '%s' exited %d\n%s",
name,
argv[0],
state.exit_code,
strings.trim_space(string(stderr)),
),
pos = pos,
kind = .Data,
}
}
// If the tool streamed to stdout, persist it as the output file.
if !uses_out {
if werr := os.write_entire_file(out_path, stdout); werr != nil {
return "", Error_Body {
msg = fmt.tprintf("tool '%s': cannot write %s: %v", name, out_path, werr),
pos = pos,
kind = .Data,
}
}
}
// Memoize for the rest of this build. `key` and `out_url` are already on the
// temp allocator — the same build-scoped lifetime as the manifest — so they
// are stored directly, no clone needed.
if reg.manifest == nil {
reg.manifest = make(map[string]string, context.temp_allocator)
}
reg.manifest[key] = out_url
return out_url, nil
}
// render_command renders a tool's command template with the in/out/minify
// context. Paths render through mustache's default HTML escaping; the paths we
// generate are controlled and contain no HTML-special characters, so this is
// harmless in practice.
render_command :: proc(
tmpl_src, in_path, out_path: string,
minify: bool,
pos: int,
) -> (
string,
Error,
) {
tmpl, perr := parse(tmpl_src, "", context.temp_allocator, context.temp_allocator)
if perr != nil {
b := body(perr)
return "", Error_Body {
msg = fmt.tprintf("invalid tool command template: %s", b.msg),
pos = pos,
kind = .Data,
}
}
// `in`/`out` are Odin keywords, so the context is a map rather than a struct.
data := make(map[string]any, 3, context.temp_allocator)
data["in"] = in_path
data["out"] = out_path
data["minify"] = minify
out, rerr := render(tmpl, data, allocator = context.temp_allocator)
if rerr != nil {
b := body(rerr)
return "", Error_Body {
msg = fmt.tprintf("failed to render tool command: %s", b.msg),
pos = pos,
kind = .Data,
}
}
return out, nil
}
// is_absolute_url reports whether a path points outside the local site.
is_absolute_url :: proc(s: string) -> bool {
return(
strings.has_prefix(s, "http://") ||
strings.has_prefix(s, "https://") ||
strings.has_prefix(s, "//") \
)
}
// tokenize_command splits a command line into argv on whitespace, treating a
// double-quoted span as a single token with the quotes stripped. No escape
// sequences or single quotes (kept deliberately minimal — commands come from a
// trusted config, not untrusted input).
tokenize_command :: proc(cmd: string, allocator := context.temp_allocator) -> []string {
out := make([dynamic]string, 0, 8, allocator)
i := 0
for i < len(cmd) {
for i < len(cmd) && is_pipe_space(cmd[i]) {
i += 1
}
if i >= len(cmd) {
break
}
if cmd[i] == '"' {
j := i + 1
for j < len(cmd) && cmd[j] != '"' {
j += 1
}
append(&out, strings.clone(cmd[i + 1:j], allocator))
i = j + 1 if j < len(cmd) else j
} else {
start := i
for i < len(cmd) && !is_pipe_space(cmd[i]) {
i += 1
}
append(&out, strings.clone(cmd[start:i], allocator))
}
}
return out[:]
}
// find_on_path reports whether `prog` is executable: an explicit path is checked
// directly, otherwise each PATH entry is probed.
find_on_path :: proc(prog: string) -> bool {
if strings.contains(prog, "/") {
return os.exists(prog)
}
path := os.get_env_alloc("PATH", context.temp_allocator)
if path == "" {
return false
}
sep := ";" when ODIN_OS == .Windows else ":"
for dir in strings.split_iterator(&path, sep) {
if dir == "" {
continue
}
candidate := fmt.tprintf("%s/%s", dir, prog)
if os.exists(candidate) {
return true
}
}
return false
}
mustache/tools_test.odin
-208
View File
@@ -1,208 +0,0 @@
#+test
package mustache
import "core:fmt"
import "core:os"
import "core:strings"
import "core:testing"
// tool_test_dirs makes a fresh in/out dir pair under the system temp dir and
// returns their paths (unique per name).
tool_test_dirs :: proc(name: string) -> (in_dir, out_dir: string) {
base, _ := os.temp_directory(context.temp_allocator)
in_dir = fmt.tprintf("%s/thor_tool_test/%s/in", base, name)
out_dir = fmt.tprintf("%s/thor_tool_test/%s/out", base, name)
os.make_directory_all(in_dir)
os.make_directory_all(out_dir)
return
}
// Tool_Test_Ctx carries the asset path as a context value (mustache resolves
// tag keys from the context, not from quoted literals).
Tool_Test_Ctx :: struct {
p: string,
}
@(test)
test_tool_unknown_tool :: proc(t: ^testing.T) {
reg: Tool_Registry // no commands configured
tpl, _ := parse(`{{ p | tool nope }}`)
defer delete_template(&tpl)
_, err := render(tpl, Tool_Test_Ctx{p = "/a.css"}, tools = &reg)
testing.expect(t, err != nil, "unknown tool should be a hard error")
}
@(test)
test_tool_unavailable_without_registry :: proc(t: ^testing.T) {
// No registry threaded through (tools == nil): the pipe must fail rather
// than silently do nothing.
tpl, _ := parse(`{{ p | tool anything }}`)
defer delete_template(&tpl)
_, err := render(tpl, Tool_Test_Ctx{p = "/a.css"})
testing.expect(t, err != nil, "tool pipe without a registry should error")
}
@(test)
test_tool_arg_count :: proc(t: ^testing.T) {
reg: Tool_Registry
// Zero args.
tpl0, _ := parse(`{{ p | tool }}`)
defer delete_template(&tpl0)
_, err0 := render(tpl0, Tool_Test_Ctx{p = "/a.css"}, tools = &reg)
testing.expect(t, err0 != nil, "tool with no name should error")
// Two args.
tpl2, _ := parse(`{{ p | tool one two }}`)
defer delete_template(&tpl2)
_, err2 := render(tpl2, Tool_Test_Ctx{p = "/a.css"}, tools = &reg)
testing.expect(t, err2 != nil, "tool with two args should error")
}
@(test)
test_tool_absolute_url_passthrough :: proc(t: ^testing.T) {
// A CDN URL has no local source; it should pass through untouched.
reg: Tool_Registry
url, err := run_tool(&reg, "anything", "https://cdn.example.com/x.css", 0)
testing.expect(t, err == nil, "absolute URL should not error")
testing.expect_value(t, url, "https://cdn.example.com/x.css")
}
@(test)
test_tool_source_not_found :: proc(t: ^testing.T) {
in_dir, out_dir := tool_test_dirs("missing_src")
commands := make(map[string]string, context.temp_allocator)
commands["cp"] = "cp {{in}} {{out}}"
reg := Tool_Registry {
commands = commands,
input_dir = in_dir,
output_dir = out_dir,
}
_, err := run_tool(&reg, "cp", "/nope.css", 0)
testing.expect(t, err != nil, "missing source file should be a hard error")
}
@(test)
test_tool_missing_binary :: proc(t: ^testing.T) {
in_dir, out_dir := tool_test_dirs("missing_bin")
_ = os.write_entire_file_from_string(fmt.tprintf("%s/a.css", in_dir), "body{}")
commands := make(map[string]string, context.temp_allocator)
commands["ghost"] = "thor-tool-does-not-exist-xyz {{in}} {{out}}"
reg := Tool_Registry {
commands = commands,
input_dir = in_dir,
output_dir = out_dir,
}
_, err := run_tool(&reg, "ghost", "/a.css", 0)
testing.expect(t, err != nil, "missing binary should be caught by PATH check")
}
@(test)
test_tool_runs :: proc(t: ^testing.T) {
when ODIN_OS != .Windows {
in_dir, out_dir := tool_test_dirs("runs")
os.make_directory_all(fmt.tprintf("%s/css", in_dir))
_ = os.write_entire_file_from_string(
fmt.tprintf("%s/css/main.css", in_dir),
"body { color: red }",
)
commands := make(map[string]string, context.temp_allocator)
commands["copyit"] = "cp {{in}} {{out}}"
reg := Tool_Registry {
commands = commands,
input_dir = in_dir,
output_dir = out_dir,
}
tpl, _ := parse(`{{ p | tool copyit }}`)
defer delete_template(&tpl)
result, err := render(tpl, Tool_Test_Ctx{p = "/css/main.css"}, tools = &reg)
defer delete(result)
testing.expect(t, err == nil, "cp tool should succeed")
testing.expect_value(t, result, "/css/main.css")
// The tool wrote the output file.
written := fmt.tprintf("%s/css/main.css", out_dir)
data, rerr := os.read_entire_file_from_path(written, context.temp_allocator)
testing.expect(t, rerr == nil, "output file should exist")
testing.expect_value(t, string(data), "body { color: red }")
// The result was memoized; a second call for the same asset returns the
// same URL from the registry (mutated through the pointer).
url2, err2 := run_tool(&reg, "copyit", "/css/main.css", 0)
testing.expect(t, err2 == nil, "memoized call should not error")
testing.expect_value(t, url2, "/css/main.css")
testing.expect(t, len(reg.manifest) == 1, "result should be memoized")
}
}
// tool_io_case runs `cmd` over a source file containing `content` and returns
// the resulting output-file contents. Exercises the in/out ↔ stdin/stdout 2×2.
tool_io_case :: proc(name, cmd, content: string) -> (out: string, ok: bool) {
in_dir, out_dir := tool_test_dirs(name)
_ = os.write_entire_file_from_string(fmt.tprintf("%s/a.css", in_dir), content)
commands := make(map[string]string, context.temp_allocator)
commands[name] = cmd
reg := Tool_Registry {
commands = commands,
input_dir = in_dir,
output_dir = out_dir,
}
_, err := run_tool(&reg, name, "/a.css", 0)
if err != nil {
return "", false
}
data, rerr := os.read_entire_file_from_path(
fmt.tprintf("%s/a.css", out_dir),
context.temp_allocator,
)
return string(data), rerr == nil
}
@(test)
test_tool_stdin_stdout :: proc(t: ^testing.T) {
when ODIN_OS != .Windows {
// `cat` with no args: reads stdin (thor feeds the source), writes stdout
// (thor captures and persists it).
out, ok := tool_io_case("stream", "cat", "body{a:1}")
testing.expect(t, ok, "cat (stdin→stdout) should produce output")
testing.expect_value(t, out, "body{a:1}")
}
}
@(test)
test_tool_in_path_stdout :: proc(t: ^testing.T) {
when ODIN_OS != .Windows {
// `cat {{in}}`: tool opens the file by path, streams to stdout.
out, ok := tool_io_case("in_stream", "cat {{in}}", "body{b:2}")
testing.expect(t, ok, "cat {{in}} (file→stdout) should produce output")
testing.expect_value(t, out, "body{b:2}")
}
}
@(test)
test_tool_stdin_out_path :: proc(t: ^testing.T) {
when ODIN_OS != .Windows {
// `tee {{out}}`: thor feeds stdin, tool writes the file itself.
out, ok := tool_io_case("stream_out", "tee {{out}}", "body{c:3}")
testing.expect(t, ok, "tee {{out}} (stdin→file) should produce output")
testing.expect_value(t, out, "body{c:3}")
}
}
@(test)
test_tool_minify_section :: proc(t: ^testing.T) {
// The command template is itself mustache: {{#minify}} should expand only
// when the registry has minify on. Verify via the rendered command string.
cmd, err := render_command("prog {{#minify}}--min{{/minify}} {{in}}", "/a", "/b", true, 0)
testing.expect(t, err == nil, "render_command should succeed (minify on)")
testing.expect(t, strings.contains(cmd, "--min"), "minify section should render when on")
cmd2, err2 := render_command("prog {{#minify}}--min{{/minify}} {{in}}", "/a", "/b", false, 0)
testing.expect(t, err2 == nil, "render_command should succeed (minify off)")
testing.expect(t, !strings.contains(cmd2, "--min"), "minify section should be empty when off")
}
render.odin
+13 -55
View File
@@ -2,9 +2,6 @@ package main
import "mustache"
import diags "diagnostics"
import ts "treesitter"
import "core:encoding/json"
import "core:fmt"
import "core:log"
@@ -43,12 +40,12 @@ load_template :: proc(vfs: ^VFS, virtual_path: string) -> mustache.Template {
b := mustache.body(err)
log.errorf(
"%s",
mustache.format_tag_error(
mustache.format_error(
entry.fs_path,
source,
b.pos,
b.msg,
colorize = diags.should_colorize(),
colorize = mustache.should_colorize(),
),
)
os.exit(1)
@@ -139,19 +136,13 @@ render_template :: proc(
ctx: Template_Context,
partials: map[string]mustache.Template,
reported_errors: ^map[string]bool,
tools: ^mustache.Tool_Registry,
) -> string {
result, err := mustache.render(
content_tpl,
[]any{ctx.site, ctx.page, ctx},
partials,
tools = tools,
)
result, err := mustache.render(content_tpl, []any{ctx.site, ctx.page, ctx}, partials)
if err != nil {
formatted := mustache.format_render_error(
err,
content_tpl,
colorize = diags.should_colorize(),
colorize = mustache.should_colorize(),
)
if formatted in reported_errors^ {
return ""
@@ -191,16 +182,6 @@ render_site :: proc(site: ^Site) {
timezone = site.tz,
}
// Prepare the `tool` pipe registry for this build (commands were filled from
// thor.json at config time). A pointer to it is threaded into every
// mustache.render call.
mustache.tool_registry_init(
&site.tool_registry,
site.assets_dir,
site.output_dir,
.Minify in site.features,
)
// Find home page
home: Page
has_home := false
@@ -222,24 +203,6 @@ render_site :: proc(site: ^Site) {
}
}
// Parsers for minification, created once and reused across every page
// (render is single-threaded). Owned here; freed when render_site returns.
html_parser, css_parser: ts.Parser
if .Minify in site.features {
if g, gok := ts.grammar("html"); gok {
html_parser = ts.open_parser(g)
}
if g, gok := ts.grammar("css"); gok {
css_parser = ts.open_parser(g)
}
}
defer if html_parser != nil {
ts.parser_delete(html_parser)
}
defer if css_parser != nil {
ts.parser_delete(css_parser)
}
// Render individual content pages (skip all index pages)
for page in pages {
if page._is_index {
@@ -248,9 +211,7 @@ render_site :: proc(site: ^Site) {
tpl := get_template(&site.vfs, page.layout, &template_cache)
html := render_page_html(page, site, tpl, partials, ctx, &errors)
if .Minify in site.features {
if m, mok := minify_html(html_parser, css_parser, html); mok {
html = m
}
html = minify_html(html)
}
write_page(site.output_dir, page.permalink, html)
}
@@ -280,9 +241,7 @@ render_site :: proc(site: ^Site) {
&errors,
)
if .Minify in site.features {
if m, mok := minify_html(html_parser, css_parser, html); mok {
html = m
}
html = minify_html(html)
}
write_page(site.output_dir, fmt.aprintf("/%s/", section), html)
}
@@ -292,9 +251,7 @@ render_site :: proc(site: ^Site) {
home_tpl := get_template(&site.vfs, "home", &template_cache)
home_html := render_home_html(home, site, home_tpl, partials, ctx, &errors)
if .Minify in site.features {
if m, mok := minify_html(html_parser, css_parser, home_html); mok {
home_html = m
}
home_html = minify_html(home_html)
}
write_file(fmt.tprintf("%s/index.html", site.output_dir), home_html)
}
@@ -333,7 +290,7 @@ render_page_html :: proc(
ctx.page = page
ctx.og = og_for_page(site.og, page)
ctx.params = merge_params(site.params, page.params)
return render_template(content_tpl, ctx, partials, seen, &site.tool_registry)
return render_template(content_tpl, ctx, partials, seen)
}
render_home_html :: proc(
@@ -357,7 +314,7 @@ render_home_html :: proc(
ctx.og = og_for_page(site.og, home)
ctx.params = merge_params(site.params, home.params)
return render_template(content_tpl, ctx, partials, seen, &site.tool_registry)
return render_template(content_tpl, ctx, partials, seen)
}
render_section :: proc(
@@ -396,7 +353,7 @@ render_section :: proc(
}
ctx.posts = posts
ctx.params = merge_params(site.params, ctx.page.params)
return render_template(content_tpl, ctx, partials, seen, &site.tool_registry)
return render_template(content_tpl, ctx, partials, seen)
}
load_partials :: proc(vfs: ^VFS) -> map[string]mustache.Template {
@@ -420,12 +377,12 @@ load_partials :: proc(vfs: ^VFS) -> map[string]mustache.Template {
b := mustache.body(err)
log.errorf(
"%s",
mustache.format_tag_error(
mustache.format_error(
entry.fs_path,
source,
b.pos,
b.msg,
colorize = diags.should_colorize(),
colorize = mustache.should_colorize(),
),
)
os.exit(1)
@@ -486,3 +443,4 @@ write_file :: proc(path: string, html: string) {
log.errorf("cannot write %s: %v", path, err)
}
}
site.odin
-60
View File
@@ -10,7 +10,6 @@ import "core:time/datetime"
import "core:time/timezone"
import md "markdown"
import "mustache"
// Site_Context holds the site date that is accessible in templates.
@@ -18,12 +17,6 @@ Site_Context :: struct {
title: string,
description: string,
base_url: string,
// base_path is the path component of base_url, normalized to a leading slash
// with no trailing slash ("" for root-hosted sites). Templates prefix internal
// root-relative links with it (directly, or via the `rel_url` pipe) so that
// subpath deployments — e.g. project pages at https://user.github.io/proj —
// resolve correctly.
base_path: string,
params: json.Object,
og: Open_Graph,
menus: map[string][]Menu_Entry,
@@ -49,11 +42,6 @@ Site :: struct {
tz: ^datetime.TZ_Region,
grammars: string,
queries: string,
// tool_registry is the `tool` pipe's state: `commands` is filled from
// thor.json at config time; the per-build I/O dirs, minify flag, and memo
// are set by mustache.tool_registry_init in render_site. A pointer is
// threaded into mustache.render.
tool_registry: mustache.Tool_Registry,
}
Date_Preferences :: struct {
@@ -85,7 +73,6 @@ Config_File :: struct {
date: Date_Preferences,
grammars: string,
queries: string,
tools: map[string]string,
}
// Configuration loaded from command line arguments. Gets folded in to Site
@@ -140,7 +127,6 @@ init_site :: proc(site: ^Site, flags: Flags) {
}
site_apply_cli_flags(site, flags)
site.base_path = site_base_path(site.base_url, site_allocator(site))
site.config_path = path
site.og = og_for_site(site)
@@ -225,52 +211,6 @@ site_apply_config :: proc(site: ^Site, config: Config_File, config_dir: string)
site.grammars = expand_path(config.grammars, site_allocator(site))
site.queries = expand_path(config.queries, site_allocator(site))
if config.tools != nil {
site.tool_registry.commands = config.tools
}
}
// site_base_path extracts the path component of base_url, normalized to a
// leading slash with no trailing slash. Returns "" for root-hosted sites.
//
// "https://ex.com/thor/" → "/thor"
// "https://ex.com/a/b" → "/a/b"
// "https://ex.com" → ""
// "http://localhost:8080/" → ""
site_base_path :: proc(base_url: string, allocator := context.allocator) -> string {
url := base_url
// Strip scheme so the first '/' we find is the start of the path, not the
// "//" separating scheme and host.
if idx := strings.index(url, "://"); idx >= 0 {
url = url[idx + 3:]
}
// The path begins at the first '/' after the host.
slash := strings.index_byte(url, '/')
if slash < 0 {
return ""
}
path := url[slash:]
// Drop any query or fragment.
if idx := strings.index_byte(path, '?'); idx >= 0 {
path = path[:idx]
}
if idx := strings.index_byte(path, '#'); idx >= 0 {
path = path[:idx]
}
// Trim trailing slashes; a bare "/" normalizes to "".
for len(path) > 0 && path[len(path) - 1] == '/' {
path = path[:len(path) - 1]
}
if path == "" {
return ""
}
return strings.clone(path, allocator)
}
// expand_path replaces a leading ~/ with $HOME/.
site/assets/css/main.css
-34
View File
@@ -1,34 +0,0 @@
body>header {
background: var(--pico-primary-background);
position: sticky;
top: 0;
nav>ul li a {
color: var(--pico-primary-inverse);
}
}
/* TODO: Should probaby use a container for the page content like Odin does */
.toc {
position: fixed;
height: calc(100vh - 116px);
width: 280px;
top: 116px;
left: 0;
overflow-y: auto;
--pico-nav-element-spacing-vertical: 0.25rem;
& ul {
display: block;
&>li {
margin-left: 1rem;
width: 100%;
}
}
}
main>article {
width: 60%;
margin: auto;
}
site/content/docs.md
+37 -369
View File
@@ -3,21 +3,6 @@
"date": "2026-07-22T08:54:00-04:00",
"toc": true
}
<!--
This document serves as the primary reference material for building Thor sites.
It should include reference and description of all
- markdown extensions
- site configuration options
- commandline flags
- frontmatter options
etc.
TODO: Use consistent spacing with mustache tags
-->
[TOC]
## Introduction
@@ -25,60 +10,19 @@ This guide assumes you have either read [The Guide](../guide), or have built a [
## Features
Thor has many features and content processors available. In an effort to provide the best out of the box experience, most of them are enabled by default. Those that aren't are chosen because either:
1. Significant Performance loses.
2. Unexpected behaviour (content mangling)
If you feel that an extension is in the wrong category, let is know! [^issues]
[^issues]: TODO: Add a link to issues from here.
Thor has many features and content processors available. In an effort to provide the best out of the box experience, most of them are enabled by default.
### Opt-In Features
#### Footnotes
TODO: #### deflist syntax
By default, Thor uses Tufte style margin notes instead of footnotes. However, if you prefer Hugo style footnotes, you can enable them in the [config](#configuration) file.
TODO: Explain what tufte style is.
#### Syntax Highlighting
The highlight extension enables server-side syntax highlighting, powered by TreeSitter. This feature requires downloading a grammar file for each language you want to highlight, and will raise your build times from several milliseconds to over 250
TODO: #### footnotes
#### Minify
If enabled, `minify` will perform simple whitespace removal on all your output `.html` files, and any `.css` files in your `assets` directories. Minifying JavaScript is not supported (yet).
Inline CSS will be minified.
#### Sections
The `sections` extension wraps each of your content sections in a `<section>` block. Sections are opened just before each heading, and closed before the next heading, or the end the document, whichever comes first.
```markdown
<!-- Example -->
## First Section
content
## Second Section
content
```
Becomes:
```html
<section>
<h2>First Section</h2>
<p>content</p>
</section>
<section>
<h2>Second Section</h2>
<p>content</p>
</section>
```
TODO: Do we minify inline css?
### Opt-Out Features
@@ -86,18 +30,16 @@ Becomes:
- sidenotes/marginnotes
- syntax highlighting
- heading ids
- Table Of Contents Generation
- TODO: Table Of Contents Generation
- [GitHub style alerts](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#alerts)
- deflist syntax[^deflist]
[^deflist]: This may become opt-in in the future.
## Directories
Like Hugo, a Thor project is a collection of specially named directories, plus an optional config file. All directories are optional, but it is recommended to at least have a `content` directory.
content
: `content` holds your pages and page bundles. If not found, thor will look for content files in the root of your current working directory. See [Content](#content) for more details.
: `content` holds your pages and page bundles. See [Content](#content). If not found, thor will look for content files in the root of your current working directory.
assets
: `assets` contains any static files for your site (favicon.ico, etc.), as well as files you want to send through the asset pipeline (CSS or JS files).
@@ -111,7 +53,7 @@ layouts
All of these names can be remapped in `thor.json`.[^remap]
[^remap]: While directories can be remapped at the site level, [modules](#modules) must (currently) adhere to the defaults.
[^remap]: While directories can be remapped at the site level, modules must (currently) adhere to the defaults.
### Assets
@@ -142,83 +84,40 @@ TODO: Don't forget to highlight differences from Hugo.
## Templates
Sites are built using one or more template files written with an extended version of [mustache](https://mustache.github.io). The [mustache manual](https://mustache.github.io/mustache.5.html) has great explainations and a lot of examples, but this section should tell you everything you need to know.
Sites are built using one or more template files written in an extended version of [mustache](https://mustache.github.io) templates. The [mustache manual](https://mustache.github.io/mustache.5.html) has great explainations and a lot of examples if you want to know more, but I'll summarize them for you here.
The beauty of Mustache is that there is very little syntax; there are just 10 symbols you need learn: `{{`, `{{&`, `{{^`, `{{>`, `{{<`, `{{#`, `/}}`, `{{$`, `{{!`, and `|`.
TODO: ^^ Badly worded sentence ^^
TODOS: Gotta describe base templates somewhere. (the same way we describe the partials)
### Tags
The beauty of Mustache is that there is very little syntax; there are just 10 symbols you need learn: `{{`, `{{&`, `{{^`, `{{>`, `{{<`, `{{#`, `/}}`, `{{$`, `{{!`, and `|`.
#### Variables
In order to display a scalar (non-list) value in your template, simply wrap it in double curly braces. e.g. `{{ page.title }}`.
In order to display a scalar (not-list) value in your template, simply wrap it in double curly braces. e.g. `{{ page.title }}`.
This content will be HTML escaped (for safety), so if the value you're rendering contains HTML, you'll need to use the raw syntax instead `{{& page.title}}`[^raw] which will output the value without escaping your HTML. Do so at your own risk.[^risk]
This content will be HTML escaped (for safety), so if the value you're rendering contains HTML, you'll need to use the raw syntex instead `{{& page.title}}`[^raw] which will output the value without stripping or re-writing content.
[^risk]: Or don't, if you don't know why this is dangerous.
[^raw]: Mustache also supports raw output through triple brace syntax (`{{{ raw }}}`), but `{{& raw }}` is preferred, as it's easy to accidentily insert too many braces.
Leading and trailing whitespace(s) are ignored by the parser, so the folllowing are all equivilent: `{{& title }}`, `{{&title}}`, `{{& title}}`, `{{&title }}`.
[^raw]: Official Triple brace syntax (`{{{ raw }}}`) is also supported, but `{{& raw }}` is preferred, as it's easy to accidentily insert too many braces.
In most[^most] cases, invalid keys will be silently ignored (nothing between the braces will appear), in keeping with the official mustache spec.
[^most]: TODO: in what cases won't it? spelllcheck + strict mode? Also, people don't care about the spec, they care about how the app works. Also maybe mention strict mode here.
[^most]: TODO: in what cases won't it? spelllcheck + strict mode?
Leading and trailing whitespace(s) are ignored by the parser, so the folllowing are all equivilent: `{{& title }}`, `{{&title}}`, `{{& title}}`.
#### Sections
> Sections render blocks of text zero or more times, depending on the value of the key in the current context.
Sections allow you to render text conditionally, and also allow you to render lists[^iffor]. They are opened with `{{#key}}` and closed with `{{/key}}`.
[^iffor]: These would be your `if` and `for` blocks in a logic-ful language.
```mustache
{{og.title}}
Shown if OpenGraph title is set!
{{/og.title}}
<ul>
{{#pages}}
<li>{{name}}</li>
{{/pages}}
</ul>
```
TODO:
#### Inverted Sections
All of the syntax we've described so far renders text when its value is *not* empty. Inverted sections render text when the value given to them *is* empty.[^unless] This allows you to display special conent when no values exist, for example.
> [!NOTE]
> Empty values include `""` (an empty string), `[]` (an empty list), `false`
Inverted sections are opened with `{{^key}}` and closed with `{{/key}}`.
```mustache
<ul>
{{#pages}}<li>{{name}}</li>{{/pages}}
</ul>
{{^pages}}
We don't have anything to show you right now, come back later!
{{/pages}}
```
[^unless]: This would be your `if not` or `unless` keyword in a logic-ful template.
TODO:
#### Partials
Single file templates have limited functionality. Fortunately, mustache lets you break your documents up into multiple files through the use of partials.
TODO:
To include a partial, you use the `{{>name}}` syntax, where `name` is the path to the partial, relative to the `{{layouts_dir}}/partials` of the module. For example, with the default theme, the `footer` template exists in `{{theme_dir}}/layouts/partials/footer.html`, so you'd include it by adding `{{>footer}}` to you template.
See [Partials](#partials-2) for more information.
<!--
If you want to, your partials can include partials, creating a tree of files. But don't go crazy with it.
> [!WARNING]
> You are limited in the number of partials you can nest, to around 13.
> Please see [context depth](#depth-limit) to learn more.
-->
TODO: Talk about MAX_CONTEXT_DEPTH (currently 16) and how it limits the total number of nested templates to 13. (3 for `[site, page, ctx]`)
##### Dynamic Partials
@@ -228,8 +127,6 @@ TODO:
#### Parents
TODO:
Parents can be used by theme developers to create complex templates with multiple content slots.
#### Summary
`{{page.title}}` for normal values
@@ -238,7 +135,7 @@ Parents can be used by theme developers to create complex templates with multipl
`{{#params.is_starred}}<i class="fas fa-star"></i>{{/params.is_starred}}` for conditional content.
`{{^pages}}No pages yet!{{/pages}}` to draw content when the value is empty.
### Block Names
### Section Names
When building your own templates, you are of course free to pick whatever names you choose for your partials and content slots. However, sticking to conventions helps create consistency in the ecosystem, and reduces friction when relying on built-in templates.
@@ -298,14 +195,6 @@ When building your page(s), the following keys are accessible to your template f
: The HTML rendered page content
`page.url`
: The full url to the page. .e.g. `https://example.com/about`
`page.rel_url`
: The url to the page, minus the `site.base_url`, e.g. `/about`.
TODO: write
#### Site
@@ -322,24 +211,20 @@ TODO: Write
`stylesheets`
: A list of paths to css files the users wants to include globally. These are rendered by the `{{> styles }}` partial.
: A list of paths to css files the users wants to include globally. These are rendered by the `{{> styles }}` partial, and can be omitted on sites that use custom templates.
TODO: ^^ Bad sentence? ^^
`scripts`
: A list of `<script>` tags the users wants to include globally. These are rendered by the `{{> scripts }}` partial.
#### The Context Stack
When building your page(s), each template is fed a Context[^ctx] stack that contains all the data you should need to build your page.
When building your page(s), each template is fed a Context[^ctx] stack that contains all the data should you need to build your page.
[^ctx]: (for developers) `Template_Context` is distinct from Odin's implicit `context` parameter.
[^ctx]: (for developers) `Template_Context` is not the same as Odin's implicit `context` parameter.
The context stack is initialized[^init] as `[site, page, context]`, meaning if you use a key like `{{title}}` in your template, it will first attempt to look up `context.title`, then `page.title`, and finally `site.title`, using the first available value it can find.
The context stack is initialized[^init] as `[site, page, context]`, meaning if you use a key like `{{title}}` in your template, it will look up `context.title`, `page.title`, and then finally `site.title`, using the first available value it can find.
[^init]: TODO: Don't use programmer speak.
As you descend into sections/partials, new contexts are placed onto the stack, so they resolve first. (LIFO order).
TODO: As you descend into sections/partials, new contexts are placed onto the stack, so they resolve first. (LIFO order)
```mustache
{{<base}}
@@ -357,16 +242,10 @@ As you descend into sections/partials, new contexts are placed onto the stack, s
{{/base}}
```
##### Depth Limit
For technical reasons, the context depth is limited to 16. This shouldn't matter to most users, but if you're running into errors, please see the [FAQ](./FAQ) for more information.
---
### Filters
Because mustache is a logic-less language, (there are no `for` or `if` keywords), things like grouping, sorting, and filtering, are not possible using standard syntax.
To get around this, Thor extends mustache to include a handful of data filters in the form of pipes. Bash users will feel right at home here.
Because mustache is a logic-less language, (there are no `for` or `if` tags), Thor extends mustache to include a handful of data filters in the form of pipes. Bash users will feel right at home here.
`group_by <field>`
@@ -384,34 +263,25 @@ To get around this, Thor extends mustache to include a handful of data filters i
{{/pages}}
```
`sort_by <field> [asc|desc]`
`sort_by <field> <asc|desc>`
: Allows you to sort pages by the given field.
`first [n]`
`first <n>`
: Given a list, returns only the first `n` items. `n` defaults to 1. Given a string, returns the first `n` runes of the string. To help catch mistakes, `n` is required for strings.
`last [n]`
`last <n>`
: Given a list, returns only the last `n` items. `n` defaults to 1. Given a string, returns the first `n` runes of the string. To help catch mistakes, `n` is required for strings.
`format ["format string"]`
`format "<format string>"`
: Used to display a date in a particular format. See [DateTimes](#datetimes). If no format is given, the default will be used.
`rel_url`
: Should only be used by theme designers or when building a site where the homepage is not the website root (i.e. https://example.com/my-cool-site/).
Will take a url like `/about` and convert it to `https://example.com/my-cool-site/about`
`tool`
: Allows you to run your assets / content through any external tools you've configured. Most commonly used for css processors like Tailwind or Sass. See [Tools](#tools) for more info.
#### Chaining Pipes
Thor allows you to chain two or more pipes, to allow complex data manipulation. However you are limited to no more than **8 pipes**[^pipes] for any given tag. If you believe you need more than 8 pipes, please [open an issue](../issues) with a **clear** and **concrete** example of the problem you are facing.
Thor allows you to chain two or more pipes, to allow complex data manipulation. However for performance and stylistic reasons, you are limited to no more than **8 pipes**[^pipes] for any given tag. If you believe you need more than 8 pipes, please [open an issue](../issues) with a **concrete example** of the problem you are facing.
[^pipes]: TODO: This number must be kept in-sync with `MAX_PIPES`.
@@ -475,7 +345,7 @@ TODO: We keep referring to blocks and tags, should probably use consistent langu
TODO: Should this be a subsection of a "Data Types" section?
Dates are strings in one of the following formats:
<!--
| Format | Time zone |
| --------------------------- | ---------------------------- |
| `2023-10-15T13:18:50-07:00` | `America/Los_Angeles` |
@@ -484,220 +354,18 @@ Dates are strings in one of the following formats:
| `2023-10-15T13:18:50` | Default is local system time |
| `2023-10-15` | Default is local system time |
| `15 Oct 2023` | Default is local system time |
-->
<table>
<thead>
<tr>
<th>Format</th>
<th>Time zone</th>
</tr>
</thead>
<tbody>
<tr>
<td><code class="inline-code">2023-10-15T13:18:50-07:00</code></td>
<td><code class="inline-code">America/Los_Angeles</code></td>
</tr>
<tr>
<td><code class="inline-code">2023-10-15T13:18:50-0700</code></td>
<td><code class="inline-code">America/Los_Angeles</code></td>
</tr>
<tr>
<td><code class="inline-code">2023-10-15T13:18:50Z</code></td>
<td><code class="inline-code">Etc/UTC</code></td>
</tr>
<tr>
<td><code class="inline-code">2023-10-15T13:18:50</code></td>
<td>Default is&nbsp;local system time</td>
</tr>
<tr>
<td><code class="inline-code">2023-10-15</code></td>
<td>Default is&nbsp;local system time</td>
</tr>
<tr>
<td><code class="inline-code">15 Oct 2023</code></td>
<td>Default is&nbsp;local system time</td>
</tr>
</tbody>
</table>
If you want to display a date in a different format, you can use the `| format` Filter. With no argument, it will default to formatting the date with `site.date_format`.
## Configuration
Thor can be configured through commandline flags, content [frontmatter](#frontmatter), and the configuration file. Settings take precedence in that order, so command line flags will always override configured values.
You can run `thor -help` to list all the available flags.
### Config File
The `thor.json` file at the root of your project is the heart of your Thor site. Hopefully, you'll write your site specific metadata here once, and you'll never have to touch it again.
`title`
: The title of your site. Will be rendered inside the `{{> title}}` partial.
`description`
: A brief description of your site. Can be used by theme authors, or as the default [OpenGraph](#open-graph) description.
`base_url`
: The root URL prepended to all links on your site.
`date.format`
: The default format to display dates in. Write the date you'd like to see formatted as [the go format string]. TODO: Explain
`date.timezone`
: The time zone to localize all formatted dates to. Will attempt to use the local system's zone if left unset.
TODO: validate the default
TODO: Explain format
`markdown_extensions`
: This setting lets you control which markdown extensions are enabled on your site. It should be a mapping of extension names to either `true` or `false`.
The following keys are available:
- `emoji`
- `sidenotes`
- `alerts`
- `highlight`
- `sections`
- `headingids`
- `deflists`
- `footnotes`
`content_dir`, `assets_dir`, `output_dir`, `layouts_dir`
: These allow you to change the directory structure of your app. See [Directories](#directories) for more info.
> [!Note]
> This will not affect the directory structure of any of your modules.
`modules`
: This setting lets you layer modules onto the Union File System. See [modules](#modules) for more info.
`params`
: Site-wide user customizable data. See [params](#params) for more info.
`og`
: Site-wide Open Graph metadata. See [Open Graph](#open-graph) for more info.
`menus`
Menu configuration for the site. Note that (unlike Hugo) if your configuration contains this key, frontmatter `menus` will be ignored. See [menus](#menus) for more info.
`tools`
Allows you to configure external content/asset processing tools. Too... See [Tools](#tools) for more info.
## Tools
Configuring tools will allow you to create custom pipelines for your content and assets. For example, you could create a tailwindcss tool to pre-process your css files. You could also create a conversion tool to make all your images be a certain format or maximum size.
Tools are configured in your config file and utilized in your template files:
> [!DANGER]: You should never run a tool you have not configured yourself, as running arbetrary commands can be extremely dangerous.
```mustache
{{!layouts/partials/styles.css}}
{{#params.stylesheets}}<link rel="stylesheet" href="{{. | tool tailwind}}">
{{/params.stylesheets}}
```
Essentially, each tool is just a bash-style commandline string with a few special variables:
<!-- TODO: It might be a terrible idea to combine mustache templates with shell scripts :/ -->
```json5
// thor.json
{
"tools": {
// For demonstration purposes only, not recommended (see below).
"tailwind": "tailwindcss -i {{in}} -o {{out}}",
}
}
```
Both `{{in}}` and `{{out}}` exist to support tools that can't read from `stdin` and/or can't write to `stdout`. If your tool can do those things, it is ***highly recommended*** to avoid `{{in}}` + `{{out}}`, as each requires thor to make **a full round-trip** to the file system, rather than keeping everything in memory.
```json5
// thor.json
{
"tools": {
// The recommended way to configure Tailwind 4.3.3
"tailwind": "tailwindcss {{#minify}}--minify{{/minify}} -i -",
}
}
```
### Special Variables
`in`
: The path to a file created by thor, containing the content that will be processed by your tool.
`out`
: The path to a file, created by your tool, that will be represents the file output by the tool, that will be ingested by thor.
`minify`
: True if the `minify` config option has been set, or `--production` flag has been passed.
- it is up to you to have the necessary binaries installed - thor won't do it for you.
- every tool you want to use must be configured - there is no default <!-- TODO: Should there be? Perhaps we could include support for them in a future, interactive site builder setup TUI. -->
## Open Graph
Thor supports [Open Graph](https://ogp.me/) tags, allowing you to customize how your site looks when linked to on social media or in apps like Discord. It is accessible through the `{{>opengraph}}` template, which should be automatically included in your header by your theme.
```html
TODO: {{opengraph shorcode here}}
```
The options are sourced [using the context stack], first from your page frontmatter, and then from your site config. Thor will attempt to infer as much information as possible, to keep you from needing to set them explictly.
The following keys are available:
- `og.title`
- `og.type`
- `og.image`
- `og.url`
- `og.description`
- `og.locale`
- `og.site_name`
- `og.is_article`*
- `og.published_time`*
- `og.modified_time`*
- `og.section`
Please read [the protocol docs](https://ogp.me/#metadata) for the meaning and use of these keys.
TODO: * keys are not a 1:1 mapping
TODO: default Template support
TODO: Template override support
TODO: how to overwrite defaults (in page frontmatter / thor.json)
## Modules
## Zen
TODO: Modules
## Themes
TODO: Themes
<hgroup>
<h2 id="zen">Zen</h2>
<p>The guding principals behind Thor.</p>
</hgroup>
> The guding principals behind Thor.
Simpler is Better
site/content/faq.md
-68
View File
@@ -1,68 +0,0 @@
{
"title": "FAQ",
"date": "2026-08-30T10:17:00-04:00",
"toc": true
}
<!--
TODO: Make sure issue link is legit
-->
## context stack depth exceeded (possible recursive section/partial)
To maintain high performance and encourage maintainable sites, the context stack can never be more than 16 items deep. What does this mean?
Every time you enter a section or a partial, the closest referenced item gets pushed onto the stack. if you then enter another partial, or reference a nested object inside of that, you'll add another item.
This is easier to see with examples:
```mustache
{{! base.html }}
<html>
<head>
<title>
{{ title }}
{{! stack size = 3 throughout the file }}
</title>
</head>
<body>
{{> now}}
<nav>
<ul>
{{#pages}}
<li>{{name}} {{! stack size = 4 within #pages}}</li>
{{/pages}}
</ul>
</nav>
{{> footer}}
</body>
</html>
```
```mustache
{{! partials/footer.html }}
{{! stack size = 4 throughout the file (when called from base.html) }}
<footer>
{{#params}}
<p>&copy; {{author.name}} {{> now}}</p>
{{! stack size = 5 inside #params }}
{{/params}}
{{/site}}
</footer>
```
```mustache
{{! partials/now.html }}
{{! stack size =
4 when called from base.html
6 when called from #params inside of footer.html
}}
{{ now | format }}
```
The stack size starts at 3 because the stack is preloaded with `[site, page, context]`, allowing you to reference values like `title` or `params` from `page` or `site` without having to use the fully qualified name `{{site.title}}`.
> [!NOTE]
> If your sites needs a context depth higher than 16, please reach out and open an issue[^issue] with a **clear** and **concrete** example of what you're trying to do.
site/content/guide.md
+1 -1
View File
@@ -17,7 +17,7 @@ At the heart of any good SSG is the content files. Thor supports markdown (`.md`
Let's make a home page - copy the following text and place it into `./index.md`
```markdown
```md
# Hello, World!
Welcome to your new site.
site/thor.json
+3 -23
View File
@@ -1,30 +1,10 @@
{
"date": {
"timezone": "America/New_York",
"format": "January 2, 2006"
},
"markdown_extensions": {
"footnotes": true,
"highlight": true
},
"menus": {
"main": [
{
"name": "GitHub",
"url": "https://github.com/sbrow/thor",
"weight": 20
}
]
// "footnotes": true
},
"params": {
"author": {
"name": "Spencer Brower"
},
"stylesheets": [
"https://cdn.jsdelivr.net/npm/@picocss/pico@2/css/pico.blue.min.css",
"/css/main.css"
]
},
"grammars": "~/.config/helix/runtime/grammars/",
"queries": "/nix/store/n9da8d007ygbgsx983jr3ar3wb1fsh6q-helix-25.07.1/lib/runtime/queries"
}
}
}
site_test.odin
+8
View File
@@ -5,6 +5,7 @@ package main
import "core:encoding/json"
import "core:flags"
import "core:fmt"
import "core:log"
import "core:os"
import "core:testing"
import "core:time/datetime"
@@ -83,6 +84,7 @@ test_load_config_file_invalid_json :: proc(t: ^testing.T) {
cfg: Config_File
ok := false
{
context.logger = log.nil_logger()
ok = load_config_file(&cfg, path, context.temp_allocator)
}
testing.expect(t, !ok)
@@ -104,6 +106,7 @@ test_load_config_file_partial :: proc(t: ^testing.T) {
@(test)
test_init_site_defaults_no_config :: proc(t: ^testing.T) {
context.logger = log.nil_logger()
site: Site
_flags: Flags
@@ -123,6 +126,7 @@ test_init_site_defaults_no_config :: proc(t: ^testing.T) {
@(test)
test_init_site_config_dir_relative :: proc(t: ^testing.T) {
context.logger = log.nil_logger()
site: Site
_flags: Flags
@@ -138,6 +142,7 @@ test_init_site_config_dir_relative :: proc(t: ^testing.T) {
@(test)
test_init_site_flag_overrides_default :: proc(t: ^testing.T) {
context.logger = log.nil_logger()
site: Site
_flags: Flags
@@ -151,6 +156,7 @@ test_init_site_flag_overrides_default :: proc(t: ^testing.T) {
@(test)
test_init_site_full_pipeline :: proc(t: ^testing.T) {
context.logger = log.nil_logger()
path := write_temp_config(
"pipeline",
@@ -172,6 +178,7 @@ test_init_site_full_pipeline :: proc(t: ^testing.T) {
@(test)
test_init_site_md_enable_disable :: proc(t: ^testing.T) {
context.logger = log.nil_logger()
site: Site
_flags: Flags
@@ -192,6 +199,7 @@ test_init_site_md_enable_disable :: proc(t: ^testing.T) {
@(test)
test_init_site_config_paths :: proc(t: ^testing.T) {
context.logger = log.nil_logger()
path := write_temp_config(
"paths",
treesitter/treesitter.odin
+128 -182
View File
@@ -7,7 +7,6 @@ import "core:mem"
import "core:os"
import "core:strings"
import "core:sync"
import si "core:sys/info"
import "core:thread"
grammar_dir: string
@@ -121,15 +120,11 @@ foreign css_grammar {
tree_sitter_css :: proc() -> Language ---
}
// A Grammar bundles only the immutable, shareable tree-sitter objects for a
// language: `language` (from tree_sitter_x() or a dlopen'd .so) and `query` (the
// compiled highlight query). Both are immutable after construction and safe to
// read from any number of threads. The stateful, single-thread-use objects —
// parser and query cursor — are deliberately NOT stored here; the caller creates
// and owns those (see `open_parser`). See GRAMMAR_CACHE_THREADING.md.
Grammar :: struct {
Grammar_Cache :: struct {
language: Language,
parser: Parser,
query: Query,
cursor: Query_Cursor,
query_failed: bool,
}
@@ -137,98 +132,17 @@ Get_Language_Proc :: #type proc() -> Language
SPALL :: #config(SPALL, false)
// registry is the process-lifetime cache of immutable Grammars, keyed by language
// name. It is the library's ONLY shared mutable state; `mu` serializes all access.
// A nil value memoizes "tried and unavailable" so misses don't re-dlopen. Parsers
// live in the caller, so the registry holds nothing single-thread-use.
Grammar_Registry :: struct {
mu: sync.Mutex,
grammars: map[string]^Grammar,
grammar_store: Grammar_Store
cache_mutex: sync.Mutex
Grammar_Store :: struct {
cache: map[string]^Grammar_Cache,
allocator: mem.Allocator,
}
registry: Grammar_Registry
// init_persistent is optional — grammar() lazily initializes the registry on first
// use — but callers (main) may call it to bind the cache early. The allocator is
// pinned to the OS heap, NOT context.allocator, so the cache survives any transient
// allocator (per-build arena, per-test tracking) that happens to be live.
init_persistent :: proc() {
sync.mutex_lock(&registry.mu)
defer sync.mutex_unlock(&registry.mu)
ensure_registry()
}
// ensure_registry binds the registry to the heap on first use. Caller must hold mu.
@(private)
ensure_registry :: proc() {
if registry.grammars == nil {
registry.allocator = os.heap_allocator()
registry.grammars = make(map[string]^Grammar, registry.allocator)
}
}
// grammar returns the immutable Grammar for `lang`, loading it on first use.
// Thread-safe: concurrent callers serialize on `mu`. Returns (nil, false) when the
// grammar cannot be loaded. The returned pointer is immutable — callers may cache
// and read it (its `language`/`query`) without any further locking.
grammar :: proc(lang: string) -> (^Grammar, bool) {
sync.mutex_lock(&registry.mu)
defer sync.mutex_unlock(&registry.mu)
ensure_registry()
if g, seen := registry.grammars[lang]; seen {
return g, g != nil
}
g := build_grammar(lang)
registry.grammars[lang] = g
return g, g != nil
}
// build_grammar loads a language and compiles its highlight query. It touches no
// shared mutable state — only the set-once `registry.allocator` (for the heap
// `new`), read-only globals, and the per-thread temp allocator — so it is safe to
// run OFF the lock and concurrently, provided `ensure_registry` has already run
// (see preload_grammars). Returns nil if the language itself cannot be loaded; a
// grammar whose query is absent/failed is still returned (parsing works,
// highlighting does not).
@(private)
build_grammar :: proc(lang: string) -> ^Grammar {
language, ok := load_language(lang)
if !ok {
return nil
}
g := new(Grammar, registry.allocator)
g.language = language
if query, qok := compile_query(lang, language); qok {
g.query = query
} else {
g.query_failed = true
}
return g
}
// open_parser creates a fresh parser bound to the grammar's language. The CALLER
// owns the returned parser and must free it with parser_delete. This is the seam
// for parser lifetime: today callers open one and reuse it across a batch of
// parses on a single thread (render opens an html+css pair per render_site;
// assets opens one per copy_assets_dir), which is safe because nothing is shared.
// A future pooling or thread-local strategy can be dropped in here without
// touching any call site. Returns nil on failure.
open_parser :: proc(g: ^Grammar) -> Parser {
if g == nil {
return nil
}
parser := parser_new()
if parser == nil {
log.errorf("treesitter: cannot create parser")
return nil
}
if !parser_set_language(parser, g.language) {
log.errorf("treesitter: ABI mismatch setting parser language")
parser_delete(parser)
return nil
}
return parser
grammar_store.allocator = context.allocator
grammar_store.cache = make(map[string]^Grammar_Cache, grammar_store.allocator)
}
when SPALL {
@@ -239,19 +153,6 @@ when SPALL {
_thread_init = init
_thread_cleanup = cleanup
}
// Pool-shaped adapters: the thread pool calls these once per worker thread at
// startup/shutdown, so each worker's parses land in their own SPALL thread.
spall_pool_thread_init :: proc(t: ^thread.Thread, data: rawptr) {
if _thread_init != nil {
_thread_init()
}
}
spall_pool_thread_fini :: proc(t: ^thread.Thread, data: rawptr) {
if _thread_cleanup != nil {
_thread_cleanup()
}
}
}
builtin_language :: proc(lang: string) -> (language: Language, ok: bool) {
@@ -269,7 +170,7 @@ builtin_language :: proc(lang: string) -> (language: Language, ok: bool) {
// load_query returns the highlight query source for a language. Builtin
// languages (html/css) are baked into the binary via `#load`; all others are
// read from the runtime `query_dir`. `path` is the on-disk location for
// diagnostics ("(builtin)" for embedded queries). Mirrors `load_language`.
// diagnostics ("(builtin)" for embedded queries). Mirrors `ensure_parser`.
load_query :: proc(lang: string) -> (src: string, path: string, ok: bool) {
switch lang {
case "html":
@@ -318,7 +219,44 @@ load_language :: proc(lang: string) -> (language: Language, ok: bool) {
return
}
compile_query :: proc(lang: string, language: Language) -> (query: Query, ok: bool) {
ensure_parser :: proc(lang: string) -> ^Grammar_Cache {
if cached, ok := grammar_store.cache[lang]; ok {
return cached
}
grammar_store.cache[lang] = nil
language, ok := load_language(lang)
if !ok {
return nil
}
parser := parser_new()
if parser == nil {
log.errorf("treesitter: cannot create parser for %s", lang)
return nil
}
if !parser_set_language(parser, language) {
log.errorf("treesitter: ABI mismatch for %s grammar", lang)
parser_delete(parser)
return nil
}
gc := new(Grammar_Cache, grammar_store.allocator)
gc.language = language
gc.parser = parser
grammar_store.cache[lang] = gc
return gc
}
compile_query :: proc(
lang: string,
language: Language,
) -> (
query: Query,
cursor: Query_Cursor,
ok: bool,
) {
query_src, query_path, qok := load_query(lang)
if !qok {
return
@@ -380,104 +318,113 @@ compile_query :: proc(lang: string, language: Language) -> (query: Query, ok: bo
return
}
cursor = query_cursor_new()
ok = true
return
}
// PRELOAD_MAX_WORKERS caps the pool when the CPU count is unknown or absurd.
PRELOAD_MAX_WORKERS :: 4
// Preload_Task is the per-language work item handed to a pool worker.
Preload_Task :: struct {
lang: string,
load_grammar :: proc(lang: string) -> ^Grammar_Cache {
gc := ensure_parser(lang)
if gc == nil {
return nil
}
if gc.query != nil {
return gc
}
if gc.query_failed {
return nil
}
// preload_worker builds one grammar off the lock (dlopen + query compile — the
// expensive, parallelizable part), then takes `mu` only to publish the finished
// pointer. Publishing nil memoizes a failed load, matching grammar()/build_grammar.
@(private)
preload_worker :: proc(task: thread.Task) {
pt := cast(^Preload_Task)task.data
g := build_grammar(pt.lang)
sync.mutex_lock(&registry.mu)
registry.grammars[pt.lang] = g
sync.mutex_unlock(&registry.mu)
query, cursor, ok := compile_query(lang, gc.language)
if !ok {
gc.query_failed = true
return nil
}
gc.query = query
gc.cursor = cursor
return gc
}
preload_grammar :: proc(lang: string) -> ^Grammar_Cache {
language, ok := load_language(lang)
if !ok {
return nil
}
parser := parser_new()
if parser == nil {
log.errorf("treesitter: cannot create parser for %s", lang)
return nil
}
if !parser_set_language(parser, language) {
log.errorf("treesitter: ABI mismatch for %s grammar", lang)
parser_delete(parser)
return nil
}
gc := new(Grammar_Cache, grammar_store.allocator)
gc.language = language
gc.parser = parser
query, cursor, qok := compile_query(lang, language)
if !qok {
gc.query_failed = true
return gc
}
gc.query = query
gc.cursor = cursor
return gc
}
// preload_grammars warms the registry for the given languages ahead of the render
// loop, so the first page that needs one doesn't pay the load cost inline. The
// expensive per-language work (dlopen + compile_query) runs concurrently on a
// bounded thread pool. Builtins are always available lazily and are skipped.
//
// INVARIANT: preload runs to completion before any concurrent lazy grammar() use
// (render is single-threaded and runs afterward), so the registry is never read
// while workers publish. Workers still take `mu` for the map insert, keeping the
// "registry is only ever mutated under mu" discipline uniform with grammar().
preload_grammars :: proc(languages: []string) {
// Snapshot (main thread, under mu): pin the registry allocator before any
// worker reads it, and collect the languages still needing a load. Dedupes
// against builtins and already-cached grammars (watch-mode reuse).
sync.mutex_lock(&registry.mu)
ensure_registry()
if len(languages) == 0 {
return
}
// Filter out already-loaded languages (watch mode reuse)
to_load := make([dynamic]string, 0, len(languages), context.temp_allocator)
for lang in languages {
if _, bok := builtin_language(lang); bok {
if cached, ok := grammar_store.cache[lang]; ok && cached != nil {
continue
}
if _, seen := registry.grammars[lang]; seen {
if _, bok := builtin_language(lang); bok {
continue
}
append(&to_load, lang)
}
sync.mutex_unlock(&registry.mu)
if len(to_load) == 0 {
return
}
// Bind the pool to the logical core count (load is dlopen/CPU-bound), but
// never more workers than tasks, never fewer than one.
worker_count: int = ---
if _, logical, ok := si.cpu_core_count(); ok {
assert(logical > 0)
worker_count = min(logical, len(to_load))
} else {
worker_count = min(PRELOAD_MAX_WORKERS, len(to_load))
threads := make([]^thread.Thread, len(to_load), context.temp_allocator)
for i in 0 ..< len(to_load) {
threads[i] = thread.create_and_start_with_poly_data(to_load[i], grammar_worker)
}
for t in threads {
thread.join(t)
thread.destroy(t)
}
}
// Serial fast path: a single worker gains nothing from the pool machinery.
if worker_count == 1 {
for lang in to_load {
g := build_grammar(lang)
sync.mutex_lock(&registry.mu)
registry.grammars[lang] = g
sync.mutex_unlock(&registry.mu)
}
return
}
// The pool appends to its own `tasks_done` from worker threads, so its
// bookkeeping allocator MUST be thread-safe — use the heap, not the (possibly
// arena) context allocator that is live during a build.
pool: thread.Pool
init_proc: thread.Thread_Init_Proc = nil
fini_proc: thread.Thread_Init_Proc = nil
grammar_worker :: proc(lang: string) {
when SPALL {
init_proc = spall_pool_thread_init
fini_proc = spall_pool_thread_fini
if _thread_init != nil {
_thread_init()
}
defer if _thread_cleanup != nil {
_thread_cleanup()
}
}
thread.pool_init(&pool, os.heap_allocator(), worker_count, init_proc, nil, fini_proc, nil)
defer thread.pool_destroy(&pool)
tasks := make([]Preload_Task, len(to_load), context.temp_allocator)
thread.pool_start(&pool)
for lang, i in to_load {
tasks[i].lang = lang
// Per-task context allocator = heap, so any stray context.allocator use
// inside a worker is thread-safe.
thread.pool_add_task(&pool, os.heap_allocator(), preload_worker, &tasks[i], i)
gc := preload_grammar(lang)
if gc != nil {
sync.mutex_lock(&cache_mutex)
grammar_store.cache[lang] = gc
sync.mutex_unlock(&cache_mutex)
}
thread.pool_finish(&pool) // barrier: all workers joined before we return
}
extract_query_token :: proc(src: []byte, offset: u32) -> string {
@@ -512,4 +459,3 @@ helix_version_from_path :: proc(path: string) -> string {
if end <= start do return ""
return path[start:end]
}