Source
pub fn tokenSlice(tree: Ast, token_index: TokenIndex) []const u8 {
const token_tag = tree.tokenTag(token_index);
// Many tokens can be determined entirely by their tag.
if (token_tag.lexeme()) |lexeme| {
return lexeme;
}
// For some tokens, re-tokenization is needed to find the end.
var tokenizer: std.zig.Tokenizer = .{
.buffer = tree.source,
.index = tree.tokenStart(token_index),
};
const token = tokenizer.next();
assert(token.tag == token_tag);
return tree.source[token.loc.start..token.loc.end];
}