From c913424d4101b7ea75128bff08c5859370f6b535 Mon Sep 17 00:00:00 2001 From: Spencer Brower <6729162+sbrow@users.noreply.github.com> Date: Sat, 25 Jul 2026 14:02:49 -0400 Subject: [PATCH] perf: Parallelized grammar loading. --- AGENTS.md | 12 +- LOAD_GRAMMARS_IDEAS.md | 88 --------------- PARALLELIZATION.md | 115 ------------------- TODOS.md | 6 + content.odin | 125 +++++++++++++++++---- main.odin | 15 +++ treesitter/treesitter.odin | 222 ++++++++++++++++++++++++++++--------- 7 files changed, 302 insertions(+), 281 deletions(-) delete mode 100644 LOAD_GRAMMARS_IDEAS.md delete mode 100644 PARALLELIZATION.md diff --git a/AGENTS.md b/AGENTS.md index 17e58c3..140bc96 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,7 +20,7 @@ thor/ ├── treesitter/ # FFI types + grammar management (standalone package) ├── markdown/ # Content transformation pipeline (imports ../treesitter) ├── mustache/ # Template engine with lambdas + pipe filters + diagnostics -├── content.odin # Page struct, scan_content, load_page +├── content.odin # Page struct, Pending_File, scan_content_files, collect_languages, load_page ├── render.odin # Template rendering, data structs, RSS, sitemap ├── site.odin # Config (Flags, Config_File, Site), init_site ├── minify.odin # HTML/CSS minification (imports treesitter) @@ -42,7 +42,7 @@ thor/ |---|---| | `main.odin` | Entry point. Sets `context.logger`, calls `init_site`, `build_vfs`, wires `treesitter.grammar_dir`/`query_dir` from config, `site_load_content`, `render_site`. Optional Spall profiling via `SPALL` config flag. | | `site.odin` | `Flags` (CLI), `Config_File` (thor.json, includes `og: Open_Graph`), `Site` (runtime state + arena + VFS + pages + modules + `og`). `Feature` enum. 5-step `init_site`. Imports `md "markdown"` for `Extension` enum. | -| `content.odin` | `Page` struct (includes `lastmod`, `og`), `scan_content` (section-aware walk that handles leaf bundles), `load_page`, `infer_layout`. Calls `md.process()` for the markdown pipeline. | +| `content.odin` | `Page` struct (includes `lastmod`, `og`), `Pending_File` struct, `scan_content_files` (section-aware walk that handles leaf bundles), `collect_languages` (pre-scan for code fence languages), `load_page`, `infer_layout`. Calls `md.process()` for the markdown pipeline. | | `render.odin` | Template rendering: `render_site`, `render_page_html`, `render_home_html`, `render_section`. Data structs (`Base_Data`, `Page_Data`, `Home_Data`, `Section_Data`). VFS-based template loading with fallback chain (`get_template`). | | `minify.odin` | HTML/CSS minification via tree-sitter. Imports `ts "treesitter"`. | | `feed.odin` | RSS feed + sitemap XML. Uses `page.url` for canonical URLs. | @@ -57,7 +57,7 @@ thor/ | Package | Files | Responsibility | |---|---|---| -| `treesitter/` | `treesitter.odin` | FFI types (`Parser`, `Node`, `Query`, etc.), `@(link_prefix="ts_")` foreign bindings, grammar management (`ensure_parser`, `load_grammar`, `grammar_cache`), statically-linked HTML/CSS grammars | +| `treesitter/` | `treesitter.odin` | FFI types (`Parser`, `Node`, `Query`, etc.), `@(link_prefix="ts_")` foreign bindings, grammar management (`Grammar_Store` with persistent allocator, `load_language`/`compile_query` building blocks, `ensure_parser`/`load_grammar` lazy loading, `preload_grammar`/`preload_grammars` for parallel loading with `sync.Mutex` cache protection), statically-linked HTML/CSS grammars | | `markdown/` | `markdown.odin` | `Extension` enum, `DEFAULT_EXTENSIONS`, `process(body, ext, file_path)` — full pipeline, `parse_extension_list`, `apply_extension_config` | | | `footnotes.odin` | `strip_definitions` (pre-cmark), `inject_notes` (post-cmark) | | | `alerts.odin` | `inject_alerts` — GitHub alert blocks (`> [!NOTE]`) → styled blockquotes with semantic class names (`alert-note` etc.) | @@ -74,7 +74,7 @@ Icon SVGs live as HTML partials in `layouts/partials/icons/` (home, github, rss, ``` thor.json → find_config → init_site (5-step) → build_vfs (defaults/layouts → modules → site/layouts, site/assets) - → site_load_content (scan_content + url computation) + → site_load_content (scan_content_files + collect_languages + preload_grammars + load_page + url computation) → render_site → load_partials + get_template (VFS + fallback chain) → render_page_html / render_home_html / render_section @@ -170,7 +170,7 @@ Three access patterns: - `vfs_get_entry(vfs, path) -> (VFS_Entry, []byte, bool)` — entry + data (for callers that need `fs_path` for diagnostics) - `vfs_entry_data(entry) -> ([]byte, bool)` — data from an entry already in hand (avoids redundant map lookup when iterating `vfs.files`) -Content is **not yet in the VFS** — `scan_content` still uses direct filesystem reads. (See `TODOS.md`.) +Content is **not yet in the VFS** — `scan_content_files` still uses direct filesystem reads. (See `TODOS.md`.) ## Open Graph @@ -279,7 +279,7 @@ Currently implemented: `group_by ` (list → list-of-groups) and `format` Build-time highlighting via Tree-sitter C FFI. No client-side JavaScript. - **HTML and CSS grammars** statically linked via Nix (`mkGrammarStaticLib` in `thor/flake.nix`). Always available, no `dlopen`. -- **Other grammars** (bash, odin, nu, etc.) loaded via `dlopen` from `.so` files. +- **Other grammars** (bash, odin, nu, etc.) loaded via `dlopen` from `.so` files. Pre-scanned from content code fences and loaded in parallel via `preload_grammars` (one thread per language, `sync.Mutex` on `Grammar_Store.cache`). `Grammar_Store.allocator` is the OS heap (set by `init_persistent` before arena override) so grammars persist across watch-mode rebuilds. - Grammar and query paths configured via `thor.json` (`grammars`, `queries`). Flow: `thor.json` → `Config_File` → `Site` → `main.odin` sets `treesitter.grammar_dir`/`treesitter.query_dir`. Tilde (`~/`) expanded by `expand_path` in `site.odin`. Paths logged at startup. - Grammar loading split: `ensure_parser` (parser only, used by minify) vs `load_grammar` (parser + query, used by highlight). - Capture names mapped to CSS classes: `keyword` → `.hl-keyword`, etc. diff --git a/LOAD_GRAMMARS_IDEAS.md b/LOAD_GRAMMARS_IDEAS.md deleted file mode 100644 index 684f61d..0000000 --- a/LOAD_GRAMMARS_IDEAS.md +++ /dev/null @@ -1,88 +0,0 @@ -# load_grammar Optimization Ideas - -## Current bottleneck - -`load_grammar` is called once per unique language on first use. Each call costs ~30-40ms: -- `dlopen` — loads `.so` from disk (dominant) -- `read_entire_file_from_path` — reads `.scm` from disk -- `query_new` — compiles query -- `query_cursor_new` — allocates cursor (cheap) - -With ~7 languages, first-load cost is ~210-280ms. Subsequent calls hit cache (~0ms). - -## Code-level optimizations - -### Avoid double-cloning query source - -Currently: `read_entire_file_from_path` → `[]byte` → `string(raw)` → `strings.clone_to_cstring(query_src)`. Two copies of the entire `.scm` file. - -Fix: read into a buffer with a trailing null byte, use directly as cstring. One read, zero copies. - -### Reuse the grammar path string - -`fmt.tprintf("%s/%s.so", grammar_dir, lang)` is computed in `ensure_parser` AND again in error diagnostics (line 284). Compute once, store in `Grammar_Cache`. - -### Avoid `fmt.tprintf` for symbol name - -`"tree_sitter_" + lang` can be simple concatenation instead of format string parsing. - -### Extract error diagnostics to a separate proc - -The query error diagnostics (lines 250-300) make up more than half of `load_grammar`. Extract to `report_query_error(lang, query_src, err_offset, err_type, query_path)`. Makes `load_grammar` focused on the happy path. - -## Architectural optimizations - -### Pre-scan content for languages, then batch-load - -Before rendering, scan all markdown for `language-X` code blocks. Collect the unique set. Load all grammars in one pass. - -Benefits: -- Makes cost visible ("loading grammars for: bash, odin, nu...") -- Enables parallel loading -- Moves cost to a predictable point in the build - -### Parallel grammar loading - -`dlopen` is thread-safe (POSIX). Load N grammars on N threads simultaneously. - -- Sequential: ~210ms (7 languages × ~30ms each) -- Parallel (4 cores): ~60ms -- Parallel (7 threads): ~30ms - -Biggest single win for first-load time. Odin's `core:thread` or `core:sync` can manage the thread pool. - -### Static-link more grammars - -Extend the `mkGrammarStaticLib` pattern (already used for HTML/CSS in `flake.nix`) to bash, odin, nu, etc. - -- Zero `dlopen` — grammars baked into binary -- Zero `read_entire_file_from_path` — queries embedded via `#load` or `#directory` -- Zero `query_new` compilation — could pre-compile or use compile-time embedding -- Downside: binary size grows (~500KB per language), adding languages requires recompiling - -### Embed queries at compile time - -Even without static-linking grammars, the `.scm` query files could be embedded via `#load` or `#directory`. - -- Eliminates `read_entire_file_from_path` per language -- Smaller win than static linking but simpler -- Queries are small text files (~2-10KB each) - -### Cache compiled queries across runs - -`query_new` compiles the `.scm` source into an internal representation. If this could be serialized and cached to disk (like a `.thorcache`), subsequent builds could skip compilation. - -- Tree-sitter's query format isn't documented as serializable — would need investigation -- Even if not serializable, caching the raw `.scm` source in memory avoids re-reading from disk in watch mode (already handled by `grammar_cache` persistence) - -### Pre-link common grammars in the flake - -The flake could compile grammar `.so` files as Nix build inputs and pass their paths to thor at runtime via the `grammars` config field. This doesn't eliminate `dlopen` but ensures the files are always available in the Nix store. - -## Priority ranking - -1. **Parallel grammar loading** — immediate ~4-7x speedup on first load, no binary changes -2. **Static-link common grammars** — eliminates the problem entirely, but requires flake work -3. **Pre-scan + batch-load** — enables parallel loading and better UX logging -4. **Avoid double-clone of query source** — quick code cleanup, small win -5. **Embed queries at compile time** — eliminates file reads, moderate effort diff --git a/PARALLELIZATION.md b/PARALLELIZATION.md deleted file mode 100644 index 7033186..0000000 --- a/PARALLELIZATION.md +++ /dev/null @@ -1,115 +0,0 @@ -# Parallelization Plan - -## Current bottleneck - -`compile_query` (which contains `query_new`) is ~100% of the grammar loading cost. With ~7 languages, sequential loading costs ~200-280ms. The rest of the build is ~12ms. - -## Thread safety concerns - -Current code is single-threaded and assumes it: -- `grammar_cache` map — concurrent writes unsafe -- `context.temp_allocator` — shared, not thread-safe -- `Site.pages` dynamic array — concurrent appends unsafe -- Site arena — bump pointer, not atomic - -Any parallelization must address these. - -## Dependency graph - -``` -init_site → build_vfs → scan_content (discover files) - → grammar loading (must finish before any highlight_code) - → load_page × N (markdown pipeline + highlight_code) - → render_site (template rendering × N + minify × N) - → RSS + sitemap (needs all pages loaded) - → asset copying (independent) -``` - -Key barriers: -- Grammar loading must finish before any highlighting -- Pages must be loaded before rendering -- All pages must be rendered before RSS/sitemap -- Minify + write are independent per page after render - -## Three approaches - -### Option 1: Per-subsystem parallelism - -``` -Phase 1: Parallel grammar loading (7 threads) -Phase 2: Parallel content loading (N pages on M threads) -Phase 3: Parallel template rendering (N pages on M threads) -Phase 4: Parallel minification (N pages on M threads) -Phase 5: Sequential: RSS, sitemap, assets -``` - -Pros: -- Simple barriers — each phase completes before the next starts -- Easy to reason about - -Cons: -- Thread pool setup/teardown per phase (or reuse a pool with barriers) -- Memory pressure: all pages loaded before any rendered -- Load imbalance within phases (complex pages vs simple) - -### Option 2: Per-page parallelism - -``` -Pre-load all grammars (sequential or parallel) -Then for each page (in parallel): - load_page → md.process → render → minify → write -Then sequential: RSS, sitemap, assets -``` - -Pros: -- Natural work unit — each page flows through the full pipeline independently -- No barriers between phases - -Cons: -- Complex pages clog workers while simple pages finish fast -- Grammar loading must happen first (barrier) -- Needs thread-safe shared state (grammar cache, template cache, allocator) - -### Option 3: Generic worker queue - -``` -Single thread pool with work stealing. -Jobs: grammar_load(lang), load_page(file), render_page(page), minify(html), write_output(path) -Dependencies tracked via futures or callbacks. -``` - -Pros: -- Most flexible — handles all work types -- Best load balancing (work stealing across types) -- No wasted thread setup between phases - -Cons: -- Most complex to implement -- Need dependency tracking (can't render before load completes) -- Careful shared-state management needed -- Memory management with arena allocator (thread safety) - -## Recommended starting point - -**Pre-load grammars in parallel, keep everything else sequential.** - -``` -Phase 0: Pre-scan content for unique languages (fast, sequential) -Phase 1: Parallel grammar loading (one thread per language, ~30ms) - → each thread writes to a pre-assigned slot (no map contention) - → each thread uses its own temp allocator - → main thread merges results into grammar_cache after join -Phase 2: Sequential build as today (~12ms) -``` - -Expected: ~42ms total instead of ~210ms. Minimal architecture change — no thread-safe allocators needed for the rest of the pipeline. - -This can later evolve toward option 3 (generic queue) if page count grows or per-page processing becomes a bottleneck. - -## Implementation notes - -- Odin's `core:thread` or `core:sync` can manage the thread pool -- Each grammar-loading thread needs its own `context.temp_allocator` for path strings -- The `grammar_cache` map write happens on the main thread after all threads join -- `dlopen` is thread-safe (POSIX guarantee) -- `query_new` is likely thread-safe (independent computation per language, no shared state in tree-sitter) diff --git a/TODOS.md b/TODOS.md index c03eb2e..6ea43b2 100644 --- a/TODOS.md +++ b/TODOS.md @@ -23,6 +23,12 @@ - [ ] generate summary before syntax highlighting. - [ ] generate summary before markdown to html conversion. - [ ] mount_recursive is pretty significant +- [ ] thread pool for grammar loading is unbounded. +- [ ] load grammars async. +- [ ] during `load_page`: + - pass each code block to the treesitter queue + - continue working on the page, + - `await` the highlighted code. ## Memory Management diff --git a/content.odin b/content.odin index 1c756db..2d315c4 100644 --- a/content.odin +++ b/content.odin @@ -1,6 +1,7 @@ package main import md "markdown" +import ts "treesitter" import "core:fmt" import "core:log" @@ -26,21 +27,46 @@ Page :: struct { _is_index: bool `private`, } +Pending_File :: struct { + path: string, + section: string, + slug: string, + is_index: bool, +} + // site_load_content reads the content directory and populates site.pages. // Drafts are excluded unless .Drafts is enabled. site_load_content :: proc(site: ^Site) { site.pages = make([dynamic]Page, 0, 8, site_allocator(site)) - scan_content(site, site.content_dir, "") + + // Phase 0: Enumerate content files + pending := make([dynamic]Pending_File, 0, 16, context.temp_allocator) + scan_content_files(site.content_dir, "", &pending) + + // Phase 1: Pre-scan for code fence languages + // Phase 2: Parallel grammar preload + if .Highlight in site.markdown_extensions { + languages := collect_languages(pending[:]) + ts.preload_grammars(languages) + } + + // Phase 3: Load pages (grammars already cached) + for file in pending { + page, ok := load_page(file.path, file.section, file.slug, file.is_index, site.markdown_extensions) + if ok && (!page.draft || .Drafts in site.features) { + append(&site.pages, page) + } + } for &page in site.pages { page.url = fmt.tprintf("%s%s", site.base_url, page.permalink) } } -// scan_content walks the content directory. At the root level (section=""), -// directories are treated as sections. Within a section, directories are -// treated as leaf bundles (directory with an index file). -scan_content :: proc(site: ^Site, dir: string, section: string) { +// scan_content_files walks the content directory and collects Pending_File +// entries. At the root level (section=""), directories are treated as +// sections. Within a section, directories are treated as leaf bundles. +scan_content_files :: proc(dir: string, section: string, pending: ^[dynamic]Pending_File) { entries, err := os.read_all_directory_by_path(dir, context.allocator) if err != nil { log.warnf("cannot read %s: %v", dir, err) @@ -59,30 +85,28 @@ scan_content :: proc(site: ^Site, dir: string, section: string) { is_idx := filename == "index" slug := is_idx ? "" : filename - page, ok := load_page(entry.fullpath, section, slug, is_idx, site.markdown_extensions) - if ok && (!page.draft || .Drafts in site.features) { - append(&site.pages, page) - } + append(pending, Pending_File { + path = strings.clone(entry.fullpath, context.temp_allocator), + section = section, + slug = slug, + is_index = is_idx, + }) case .Directory: if section == "" { - scan_content(site, entry.fullpath, entry.name) + scan_content_files(entry.fullpath, entry.name, pending) } else { index_path := fmt.tprintf("%s/index.html", entry.fullpath) if !os.exists(index_path) { index_path = fmt.tprintf("%s/index.md", entry.fullpath) } if os.exists(index_path) { - page, ok := load_page( - index_path, - section, - entry.name, - false, - site.markdown_extensions, - ) - if ok && (!page.draft || .Drafts in site.features) { - append(&site.pages, page) - } + append(pending, Pending_File { + path = strings.clone(index_path, context.temp_allocator), + section = section, + slug = entry.name, + is_index = false, + }) } } case .Undetermined, .Symlink, .Named_Pipe, .Socket, .Block_Device, .Character_Device: @@ -90,6 +114,66 @@ scan_content :: proc(site: ^Site, dir: string, section: string) { } } +// collect_languages scans .md files for code fence language identifiers +// (```lang or ~~~lang) and returns the unique set. +collect_languages :: proc(files: []Pending_File) -> []string { + set := make(map[string]bool, context.temp_allocator) + + for file in files { + if !strings.has_suffix(file.path, ".md") { + continue + } + data, err := os.read_entire_file_from_path(file.path, context.temp_allocator) + if err != nil { + continue + } + content := string(data) + + pos := 0 + for pos < len(content) { + newline := strings.index_byte(content[pos:], '\n') + line_end := pos + newline if newline >= 0 else len(content) + line := content[pos:line_end] + + i := 0 + for i < len(line) && (line[i] == ' ' || line[i] == '\t') { + i += 1 + } + + if i + 3 <= len(line) && (line[i] == '`' && line[i+1] == '`' && line[i+2] == '`') || + (i + 3 <= len(line) && line[i] == '~' && line[i+1] == '~' && line[i+2] == '~') { + fence_char := line[i] + j := i + 3 + for j < len(line) && line[j] == fence_char { + j += 1 + } + for j < len(line) && (line[j] == ' ' || line[j] == '\t') { + j += 1 + } + lang_start := j + for j < len(line) { + c := line[j] + if c == ' ' || c == '\t' || c == '\r' || c == '\n' { + break + } + j += 1 + } + if j > lang_start { + set[line[lang_start:j]] = true + } + } + + pos = line_end + 1 + } + } + + result := make([dynamic]string, 0, len(set), context.temp_allocator) + for lang in set { + append(&result, lang) + } + return result[:] +} + infer_layout :: proc(section: string, is_index: bool) -> string { if section == "" && is_index { return "home" @@ -172,4 +256,3 @@ strip_extension :: proc(name: string) -> string { } return name[:dot] } - diff --git a/main.odin b/main.odin index 81b5ccb..4ae3a27 100644 --- a/main.odin +++ b/main.odin @@ -15,6 +15,15 @@ when SPALL { spall_ctx: spall.Context @(thread_local) spall_buffer: spall.Buffer + + init_spall_for_thread :: proc() { + backing := make([]u8, spall.BUFFER_DEFAULT_SIZE, context.temp_allocator) + spall_buffer = spall.buffer_create(backing, u32(sync.current_thread_id())) + } + + cleanup_spall_for_thread :: proc() { + spall.buffer_destroy(&spall_ctx, &spall_buffer) + } } main :: proc() { @@ -40,6 +49,12 @@ main :: proc() { context.logger = console_logger defer log.destroy_console_logger(console_logger) + treesitter.init_persistent() + + when SPALL { + treesitter.set_thread_callbacks(init_spall_for_thread, cleanup_spall_for_thread) + } + for { defer free_all(context.temp_allocator) tick := time.tick_now() diff --git a/treesitter/treesitter.odin b/treesitter/treesitter.odin index 719c0fc..95a6d8a 100644 --- a/treesitter/treesitter.odin +++ b/treesitter/treesitter.odin @@ -3,8 +3,11 @@ package treesitter import "core:c" import "core:fmt" import "core:log" +import "core:mem" import "core:os" import "core:strings" +import "core:sync" +import "core:thread" grammar_dir: string query_dir: string @@ -127,7 +130,30 @@ Grammar_Cache :: struct { Get_Language_Proc :: #type proc() -> Language -grammar_cache: map[string]^Grammar_Cache +SPALL :: #config(SPALL, false) + +grammar_store: Grammar_Store +cache_mutex: sync.Mutex + +Grammar_Store :: struct { + cache: map[string]^Grammar_Cache, + allocator: mem.Allocator, +} + +init_persistent :: proc() { + grammar_store.allocator = context.allocator + grammar_store.cache = make(map[string]^Grammar_Cache, grammar_store.allocator) +} + +when SPALL { + _thread_init: proc() = nil + _thread_cleanup: proc() = nil + + set_thread_callbacks :: proc(init: proc() = nil, cleanup: proc() = nil) { + _thread_init = init + _thread_cleanup = cleanup + } +} builtin_language :: proc(lang: string) -> (language: Language, ok: bool) { switch lang { @@ -165,41 +191,44 @@ load_query :: proc(lang: string) -> (src: string, path: string, ok: bool) { return string(raw), path, true } -ensure_parser :: proc(lang: string) -> ^Grammar_Cache { - if grammar_cache == nil { - grammar_cache = make(map[string]^Grammar_Cache) +load_language :: proc(lang: string) -> (language: Language, ok: bool) { + if builtin, bok := builtin_language(lang); bok { + language = builtin + ok = true + return } - if cached, ok := grammar_cache[lang]; ok { + if grammar_dir == "" { + log.warnf("treesitter: no grammar path set, skipping %s", lang) + return + } + so_path := fmt.caprintf("%s/%s.so", grammar_dir, lang, allocator = context.temp_allocator) + handle := dlopen(so_path, RTLD_LAZY) + if handle == nil { + log.warnf("treesitter: cannot load grammar %s (%s)", lang, so_path) + return + } + sym_name := fmt.caprintf("tree_sitter_%s", lang, allocator = context.temp_allocator) + sym := dlsym(handle, sym_name) + if sym == nil { + log.errorf("treesitter: cannot find symbol %s in %s", sym_name, so_path) + return + } + get_language := transmute(Get_Language_Proc)(sym) + language = get_language() + ok = true + return +} + +ensure_parser :: proc(lang: string) -> ^Grammar_Cache { + if cached, ok := grammar_store.cache[lang]; ok { return cached } - grammar_cache[lang] = nil + grammar_store.cache[lang] = nil - language: Language - - if builtin, ok := builtin_language(lang); ok { - language = builtin - } else { - if grammar_dir == "" { - log.warnf("treesitter: no grammar path set, skipping %s", lang) - return nil - } - - so_path := fmt.caprintf("%s/%s.so", grammar_dir, lang, allocator = context.temp_allocator) - handle := dlopen(so_path, RTLD_LAZY) - if handle == nil { - log.warnf("treesitter: cannot load grammar %s (%s)", lang, so_path) - return nil - } - - sym_name := fmt.caprintf("tree_sitter_%s", lang, allocator = context.temp_allocator) - sym := dlsym(handle, sym_name) - if sym == nil { - log.errorf("treesitter: cannot find symbol %s in %s", sym_name, so_path) - return nil - } - get_language := transmute(Get_Language_Proc)(sym) - language = get_language() + language, ok := load_language(lang) + if !ok { + return nil } parser := parser_new() @@ -213,35 +242,23 @@ ensure_parser :: proc(lang: string) -> ^Grammar_Cache { return nil } - gc := new(Grammar_Cache) + gc := new(Grammar_Cache, grammar_store.allocator) gc.language = language gc.parser = parser - grammar_cache[lang] = gc + grammar_store.cache[lang] = gc return gc } -load_grammar :: proc(lang: string) -> ^Grammar_Cache { - gc := ensure_parser(lang) - if gc == nil { - return nil - } - if gc.query != nil { - return gc - } - if gc.query_failed { - return nil - } - - query_src, query_path, ok := load_query(lang) - if !ok { - gc.query_failed = true - return nil +compile_query :: proc(lang: string, language: Language) -> (query: Query, cursor: Query_Cursor, ok: bool) { + query_src, query_path, qok := load_query(lang) + if !qok { + return } query_c := strings.clone_to_cstring(query_src, context.temp_allocator) err_offset: u32 err_type: Query_Error - query := query_new(gc.language, query_c, u32(len(query_src)), &err_offset, &err_type) + query = query_new(language, query_c, u32(len(query_src)), &err_offset, &err_type) if query == nil { tok := extract_query_token(transmute([]byte)query_src, err_offset) cause := fmt.tprintf("query error at byte %d (type %v)", err_offset, err_type) @@ -290,15 +307,118 @@ load_grammar :: proc(lang: string) -> ^Grammar_Cache { } } + return + } + + cursor = query_cursor_new() + ok = true + return +} + +load_grammar :: proc(lang: string) -> ^Grammar_Cache { + gc := ensure_parser(lang) + if gc == nil { + return nil + } + if gc.query != nil { + return gc + } + if gc.query_failed { + return nil + } + + query, cursor, ok := compile_query(lang, gc.language) + if !ok { gc.query_failed = true return nil } gc.query = query - gc.cursor = query_cursor_new() + gc.cursor = cursor return gc } +preload_grammar :: proc(lang: string) -> ^Grammar_Cache { + language, ok := load_language(lang) + if !ok { + return nil + } + + parser := parser_new() + if parser == nil { + log.errorf("treesitter: cannot create parser for %s", lang) + return nil + } + if !parser_set_language(parser, language) { + log.errorf("treesitter: ABI mismatch for %s grammar", lang) + parser_delete(parser) + return nil + } + + gc := new(Grammar_Cache, grammar_store.allocator) + gc.language = language + gc.parser = parser + + query, cursor, qok := compile_query(lang, language) + if !qok { + gc.query_failed = true + return gc + } + + gc.query = query + gc.cursor = cursor + return gc +} + +preload_grammars :: proc(languages: []string) { + if len(languages) == 0 { + return + } + + // Filter out already-loaded languages (watch mode reuse) + to_load := make([dynamic]string, 0, len(languages), context.temp_allocator) + for lang in languages { + if cached, ok := grammar_store.cache[lang]; ok && cached != nil { + continue + } + if _, bok := builtin_language(lang); bok { + continue + } + append(&to_load, lang) + } + + if len(to_load) == 0 { + return + } + + threads := make([]^thread.Thread, len(to_load), context.temp_allocator) + for i in 0 ..< len(to_load) { + threads[i] = thread.create_and_start_with_poly_data(to_load[i], grammar_worker) + } + for t in threads { + thread.join(t) + thread.destroy(t) + } +} + +grammar_worker :: proc(lang: string) { + when SPALL { + if _thread_init != nil { + _thread_init() + } + defer if _thread_cleanup != nil { + _thread_cleanup() + } + } + + gc := preload_grammar(lang) + if gc != nil { + sync.mutex_lock(&cache_mutex) + grammar_store.cache[lang] = gc + sync.mutex_unlock(&cache_mutex) + } +} + extract_query_token :: proc(src: []byte, offset: u32) -> string { end := offset for int(end) < len(src) {