Source
pub fn next(self: *Tokenizer) ?Token {
var start = self.index;
var must_resolve = false;
while (self.index < self.bytes.len) {
const char = self.bytes[self.index];
switch (self.state) {
.lhs => switch (char) {
'\t', '\n', '\r', ' ' => {
// silently ignore whitespace
self.index += 1;
},
else => {
start = self.index;
self.state = .target;
},
},
.target => switch (char) {
'\n', '\r' => {
return errorIllegalChar(.invalid_target, self.index, char);
},
'$' => {
self.state = .target_dollar_sign;
self.index += 1;
},
'\\' => {
self.state = .target_reverse_solidus;
self.index += 1;
},
':' => {
self.state = .target_colon;
self.index += 1;
},
'\t', ' ' => {
self.state = .target_space;
const bytes = self.bytes[start..self.index];
std.debug.assert(bytes.len != 0);
self.index += 1;
return finishTarget(must_resolve, bytes);
},
else => {
self.index += 1;
},
},
.target_reverse_solidus => switch (char) {
'\t', '\n', '\r' => {
return errorIllegalChar(.bad_target_escape, self.index, char);
},
' ', '#', '\\' => {
must_resolve = true;
self.state = .target;
self.index += 1;
},
'$' => {
self.state = .target_dollar_sign;
self.index += 1;
},
else => {
self.state = .target;
self.index += 1;
},
},
.target_dollar_sign => switch (char) {
'$' => {
must_resolve = true;
self.state = .target;
self.index += 1;
},
else => {
return errorIllegalChar(.expected_dollar_sign, self.index, char);
},
},
.target_colon => switch (char) {
'\n', '\r' => {
const bytes = self.bytes[start .. self.index - 1];
if (bytes.len != 0) {
self.state = .lhs;
return finishTarget(must_resolve, bytes);
}
// silently ignore null target
self.state = .lhs;
},
'/', '\\' => {
self.state = .target_colon_reverse_solidus;
self.index += 1;
},
else => {
const bytes = self.bytes[start .. self.index - 1];
if (bytes.len != 0) {
self.state = .rhs;
return finishTarget(must_resolve, bytes);
}
// silently ignore null target
self.state = .lhs;
},
},
.target_colon_reverse_solidus => switch (char) {
'\n', '\r' => {
const bytes = self.bytes[start .. self.index - 2];
if (bytes.len != 0) {
self.state = .lhs;
return finishTarget(must_resolve, bytes);
}
// silently ignore null target
self.state = .lhs;
},
else => {
self.state = .target;
},
},
.target_space => switch (char) {
'\t', ' ' => {
// silently ignore additional horizontal whitespace
self.index += 1;
},
':' => {
self.state = .rhs;
self.index += 1;
},
else => {
return errorIllegalChar(.expected_colon, self.index, char);
},
},
.rhs => switch (char) {
'\t', ' ' => {
// silently ignore horizontal whitespace
self.index += 1;
},
'\n', '\r' => {
self.state = .lhs;
},
'\\' => {
self.state = .rhs_continuation;
self.index += 1;
},
'"' => {
self.state = .prereq_quote;
self.index += 1;
start = self.index;
},
else => {
start = self.index;
self.state = .prereq;
},
},
.rhs_continuation => switch (char) {
'\n' => {
self.state = .rhs;
self.index += 1;
},
'\r' => {
self.state = .rhs_continuation_linefeed;
self.index += 1;
},
else => {
return errorIllegalChar(.continuation_eol, self.index, char);
},
},
.rhs_continuation_linefeed => switch (char) {
'\n' => {
self.state = .rhs;
self.index += 1;
},
else => {
return errorIllegalChar(.continuation_eol, self.index, char);
},
},
.prereq_quote => switch (char) {
'"' => {
self.index += 1;
self.state = .rhs;
return finishPrereq(must_resolve, self.bytes[start .. self.index - 1]);
},
else => {
self.index += 1;
},
},
.prereq => switch (char) {
'\t', ' ' => {
self.state = .rhs;
return finishPrereq(must_resolve, self.bytes[start..self.index]);
},
'\n', '\r' => {
self.state = .lhs;
return finishPrereq(must_resolve, self.bytes[start..self.index]);
},
'\\' => {
self.state = .prereq_continuation;
self.index += 1;
},
else => {
self.index += 1;
},
},
.prereq_continuation => switch (char) {
'\n' => {
self.index += 1;
self.state = .rhs;
return finishPrereq(must_resolve, self.bytes[start .. self.index - 2]);
},
'\r' => {
self.state = .prereq_continuation_linefeed;
self.index += 1;
},
'\\' => {
// The previous \ wasn't a continuation, but this one might be.
self.index += 1;
},
' ' => {
// not continuation, but escaped space must be resolved
must_resolve = true;
self.state = .prereq;
self.index += 1;
},
else => {
// not continuation
self.state = .prereq;
self.index += 1;
},
},
.prereq_continuation_linefeed => switch (char) {
'\n' => {
self.index += 1;
self.state = .rhs;
return finishPrereq(must_resolve, self.bytes[start .. self.index - 3]);
},
else => {
return errorIllegalChar(.continuation_eol, self.index, char);
},
},
}
} else {
switch (self.state) {
.lhs,
.rhs,
.rhs_continuation,
.rhs_continuation_linefeed,
=> return null,
.target => {
return errorPosition(.incomplete_target, start, self.bytes[start..]);
},
.target_reverse_solidus,
.target_dollar_sign,
=> {
const idx = self.index - 1;
return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]);
},
.target_colon => {
const bytes = self.bytes[start .. self.index - 1];
if (bytes.len != 0) {
self.index += 1;
self.state = .rhs;
return finishTarget(must_resolve, bytes);
}
// silently ignore null target
self.state = .lhs;
return null;
},
.target_colon_reverse_solidus => {
const bytes = self.bytes[start .. self.index - 2];
if (bytes.len != 0) {
self.index += 1;
self.state = .rhs;
return finishTarget(must_resolve, bytes);
}
// silently ignore null target
self.state = .lhs;
return null;
},
.target_space => {
const idx = self.index - 1;
return errorIllegalChar(.expected_colon, idx, self.bytes[idx]);
},
.prereq_quote => {
return errorPosition(.incomplete_quoted_prerequisite, start, self.bytes[start..]);
},
.prereq => {
self.state = .lhs;
return finishPrereq(must_resolve, self.bytes[start..]);
},
.prereq_continuation => {
self.state = .lhs;
return finishPrereq(must_resolve, self.bytes[start .. self.index - 1]);
},
.prereq_continuation_linefeed => {
self.state = .lhs;
return finishPrereq(must_resolve, self.bytes[start .. self.index - 2]);
},
}
}
unreachable;
}