thor/mustache/tokenizer.odin

package mustache

import "core:strings"

Token_Kind :: enum {
	Text,
	Variable,
	Unescaped,
	Section_Open,
	Inverted_Open,
	Section_Close,
	Comment,
	Partial,
	Parent,
	Block_Open,
}

Token :: struct {
	kind:       Token_Kind,
	value:      string,
	is_dynamic: bool,
	pos:        int,
}

tokenize :: proc(
	src: string,
	allocator := context.allocator,
) -> (
	tokens: [dynamic]Token,
	err: Render_Error,
) {
	tokens = make([dynamic]Token, 0, 8, allocator)

	i := 0
	text_start := 0

	for i < len(src) {
		if src[i] == '{' && i + 1 < len(src) && src[i + 1] == '{' {
			if i > text_start {
				append(&tokens, Token{kind = .Text, value = src[text_start:i], pos = text_start})
			}

			tag_pos := i

			if i + 2 < len(src) && src[i + 2] == '{' {
				content_start := i + 3
				idx := strings.index(src[content_start:], "}}}")
				if idx < 0 {
					return tokens, Syntax_Error {
						msg = "unclosed triple mustache '{{{'",
						pos = tag_pos,
					}
				}
				close := content_start + idx
				key := strings.trim_space(src[content_start:close])
				append(&tokens, Token{kind = .Unescaped, value = key, pos = tag_pos})
				i = close + 3
				text_start = i
			} else {
				content_start := i + 2
				sigil: byte = 0
				if content_start < len(src) {
					sigil = src[content_start]
				}

				kind: Token_Kind
				key_start := content_start

				switch sigil {
				case '&':
					kind = .Unescaped; key_start = content_start + 1
				case '#':
					kind = .Section_Open; key_start = content_start + 1
				case '^':
					kind = .Inverted_Open; key_start = content_start + 1
				case '/':
					kind = .Section_Close; key_start = content_start + 1
				case '!':
					kind = .Comment; key_start = content_start + 1
				case '>':
					kind = .Partial; key_start = content_start + 1
				case '<':
					kind = .Parent; key_start = content_start + 1
				case '$':
					kind = .Block_Open; key_start = content_start + 1
				case:
					kind = .Variable
				}

				close_idx := strings.index(src[key_start:], "}}")
				if close_idx < 0 {
					return tokens, Syntax_Error{msg = "unclosed tag '{{'", pos = tag_pos}
				}
				close := key_start + close_idx

				content := src[key_start:close]

				if kind == .Comment {
					append(&tokens, Token{kind = .Comment, value = content, pos = tag_pos})
				} else if kind == .Partial {
					trimmed := strings.trim_space(content)
					is_dyn := false
					if len(trimmed) > 0 && trimmed[0] == '*' {
						is_dyn = true
						trimmed = strings.trim_space(trimmed[1:])
					}
					append(
						&tokens,
						Token {
							kind = .Partial,
							value = trimmed,
							is_dynamic = is_dyn,
							pos = tag_pos,
						},
					)
				} else {
					append(
						&tokens,
						Token{kind = kind, value = strings.trim_space(content), pos = tag_pos},
					)
				}

				i = close + 2
				text_start = i
			}
		} else {
			i += 1
		}
	}

	if i > text_start {
		append(&tokens, Token{kind = .Text, value = src[text_start:i], pos = text_start})
	}

	trim_standalone_whitespace(&tokens)
	return tokens, nil
}

// ---------------------------------------------------------------------------
// Standalone whitespace handling
// ---------------------------------------------------------------------------

trim_standalone_whitespace :: proc(tokens: ^[dynamic]Token) {
	for i := 0; i < len(tokens); i += 1 {
		if !should_trim_whitespace(tokens[i].kind) do continue

		left_ok, left_idx := check_left(tokens[:], i)
		right_ok, right_idx := check_right(tokens[:], i)

		if !left_ok || !right_ok do continue

		if left_idx >= 0 {
			text := tokens[left_idx].value
			nl := strings.last_index_byte(text, '\n')
			tokens[left_idx].value = text[:nl+1] if nl >= 0 else ""
		}

		if right_idx >= 0 {
			text := tokens[right_idx].value
			nl := strings.index_byte(text, '\n')
			tokens[right_idx].value = text[nl+1:] if nl >= 0 else ""
		}
	}
}

check_left :: proc(tokens: []Token, i: int) -> (ok: bool, text_idx: int) {
	j := i - 1
	for j >= 0 {
		if tokens[j].kind == .Text {
			text := tokens[j].value
			if len(text) == 0 {
				j -= 1
				continue
			}
			nl := strings.last_index_byte(text, '\n')
			if nl >= 0 {
				return strings.trim_space(text[nl+1:]) == "", j
			}
			if j == 0 {
				return strings.trim_space(text) == "", j
			}
			return false, -1
		} else if should_trim_whitespace(tokens[j].kind) {
			j -= 1
		} else {
			return false, -1
		}
	}
	return true, -1
}

check_right :: proc(tokens: []Token, i: int) -> (ok: bool, text_idx: int) {
	j := i + 1
	for j < len(tokens) {
		if tokens[j].kind == .Text {
			text := tokens[j].value
			if len(text) == 0 {
				j += 1
				continue
			}
			nl := strings.index_byte(text, '\n')
			if nl >= 0 {
				return strings.trim_space(text[:nl]) == "", j
			}
			if j == len(tokens) - 1 {
				return strings.trim_space(text) == "", j
			}
			return false, -1
		} else if should_trim_whitespace(tokens[j].kind) {
			j += 1
		} else {
			return false, -1
		}
	}
	return true, -1
}

should_trim_whitespace :: proc(kind: Token_Kind) -> bool {
	switch kind {
	case .Section_Open, .Inverted_Open, .Section_Close, .Comment, .Partial, .Parent, .Block_Open:
		return true
	case .Text, .Variable, .Unescaped:
		return false
	}
	return false
}