|
|
|
@ -103,7 +103,7 @@ struct TokenizerPriv |
|
|
|
|
std::string errorMsg{ "" }; |
|
|
|
|
|
|
|
|
|
DataIndex idx{ data }; |
|
|
|
|
TokenType curType{ TokenType::Undefined }; |
|
|
|
|
STokenType curType{ STokenType::Undefined }; |
|
|
|
|
|
|
|
|
|
std::vector<std::unique_ptr<Scope>> scopes{}; |
|
|
|
|
std::vector<std::string> globals{}; |
|
|
|
@ -162,25 +162,25 @@ bool TokenizerPriv::determineType(bool blankExpressionRelated) |
|
|
|
|
const char c = data[idx()]; |
|
|
|
|
|
|
|
|
|
if (c == '/' && blankExpressionRelated) |
|
|
|
|
curType = TokenType::Command; |
|
|
|
|
curType = STokenType::Command; |
|
|
|
|
|
|
|
|
|
else if (c == '&' && blankExpressionRelated) |
|
|
|
|
curType = TokenType::Reference; |
|
|
|
|
curType = STokenType::Reference; |
|
|
|
|
|
|
|
|
|
else if (keywordChar(c, false)) |
|
|
|
|
curType = TokenType::Keyword; |
|
|
|
|
curType = STokenType::Keyword; |
|
|
|
|
|
|
|
|
|
else if (c >= '0' && c <= '9') |
|
|
|
|
curType = TokenType::NumberLiteral; // RealLiteral will be detected during parsing of this type
|
|
|
|
|
curType = STokenType::NumberLiteral; // RealLiteral will be detected during parsing of this type
|
|
|
|
|
|
|
|
|
|
else if (c == '"') |
|
|
|
|
curType = TokenType::StringLiteral; |
|
|
|
|
curType = STokenType::StringLiteral; |
|
|
|
|
|
|
|
|
|
else if (c == '{') |
|
|
|
|
curType = TokenType::Scope; |
|
|
|
|
curType = STokenType::Scope; |
|
|
|
|
|
|
|
|
|
else if (c == '}') |
|
|
|
|
curType = TokenType::EndScope; |
|
|
|
|
curType = STokenType::EndScope; |
|
|
|
|
|
|
|
|
|
else if (c == '(') { |
|
|
|
|
if (idx > 0) { |
|
|
|
@ -190,57 +190,57 @@ bool TokenizerPriv::determineType(bool blankExpressionRelated) |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|
if (keywordChar(prev[0], false)) |
|
|
|
|
curType = TokenType::Function; |
|
|
|
|
curType = STokenType::Function; |
|
|
|
|
else |
|
|
|
|
curType = TokenType::Expression; |
|
|
|
|
curType = STokenType::Expression; |
|
|
|
|
} |
|
|
|
|
else |
|
|
|
|
curType = TokenType::Expression; |
|
|
|
|
curType = STokenType::Expression; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
else if (c == ')' || c == ']') |
|
|
|
|
curType = TokenType::EndExpression; |
|
|
|
|
curType = STokenType::EndExpression; |
|
|
|
|
|
|
|
|
|
else if (c == '+' || c == '-' || c == '*' || c == '/' || c == '!' |
|
|
|
|
|| c == '&' || c == '|' || c == '<' || c == '>' || c == '=') |
|
|
|
|
curType = TokenType::Operator; |
|
|
|
|
curType = STokenType::Operator; |
|
|
|
|
|
|
|
|
|
else if (c == ',') |
|
|
|
|
curType = TokenType::Comma; |
|
|
|
|
curType = STokenType::Comma; |
|
|
|
|
|
|
|
|
|
else if (c == ';') |
|
|
|
|
curType = TokenType::SemiColon; |
|
|
|
|
curType = STokenType::SemiColon; |
|
|
|
|
|
|
|
|
|
else { |
|
|
|
|
curType = TokenType::Undefined; |
|
|
|
|
curType = STokenType::Undefined; |
|
|
|
|
errorMsg = "Stray character: "; |
|
|
|
|
errorMsg.push_back(c); |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/* Peek next keyword for specials like 'if', 'else', etc. */ |
|
|
|
|
if (curType == TokenType::Keyword) { |
|
|
|
|
if (curType == STokenType::Keyword) { |
|
|
|
|
auto peek = peekNextKeyword(idx); |
|
|
|
|
|
|
|
|
|
if (peek.first == TKN::If) |
|
|
|
|
curType = TokenType::CondIf; |
|
|
|
|
curType = STokenType::CondIf; |
|
|
|
|
else if (peek.first == TKN::Else) |
|
|
|
|
curType = TokenType::CondElse; |
|
|
|
|
curType = STokenType::CondElse; |
|
|
|
|
else if (peek.first == TKN::While) |
|
|
|
|
curType = TokenType::CondWhile; |
|
|
|
|
curType = STokenType::CondWhile; |
|
|
|
|
else if (peek.first == TKN::Dispatch) |
|
|
|
|
curType = TokenType::Dispatch; |
|
|
|
|
curType = STokenType::Dispatch; |
|
|
|
|
else if (peek.first == TKN::False || peek.first == TKN::True) { |
|
|
|
|
curType = TokenType::BoolLiteral; |
|
|
|
|
curType = STokenType::BoolLiteral; |
|
|
|
|
buf = peek.first; |
|
|
|
|
} |
|
|
|
|
else if (data[peek.second()] == '[') { |
|
|
|
|
curType = TokenType::MapKeyword; |
|
|
|
|
curType = STokenType::MapKeyword; |
|
|
|
|
buf = peek.first; |
|
|
|
|
++peek.second; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (curType != TokenType::Keyword) |
|
|
|
|
if (curType != STokenType::Keyword) |
|
|
|
|
idx = peek.second; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -324,7 +324,7 @@ bool TokenizerPriv::number() |
|
|
|
|
buf += decimalSeparator(); |
|
|
|
|
decimalsep = true; |
|
|
|
|
// Side effect:
|
|
|
|
|
curType = TokenType::RealLiteral; |
|
|
|
|
curType = STokenType::RealLiteral; |
|
|
|
|
} |
|
|
|
|
else |
|
|
|
|
break; |
|
|
|
@ -343,13 +343,13 @@ void TokenizerPriv::comment() |
|
|
|
|
|
|
|
|
|
bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
{ |
|
|
|
|
const TokenType& type = curType; |
|
|
|
|
const STokenType& type = curType; |
|
|
|
|
|
|
|
|
|
while (idx < data.length()) { |
|
|
|
|
if (!determineType(root.count() == 0)) return false; |
|
|
|
|
|
|
|
|
|
switch (type) { |
|
|
|
|
case TokenType::Command: |
|
|
|
|
case STokenType::Command: |
|
|
|
|
{ |
|
|
|
|
if (!command()) return false; |
|
|
|
|
if (buf.empty()) { |
|
|
|
@ -362,7 +362,7 @@ bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
return true; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
case TokenType::Keyword: // TODO return as own token type
|
|
|
|
|
case STokenType::Keyword: // TODO return as own token type
|
|
|
|
|
if (!keyword()) return false; |
|
|
|
|
if (buf == "return") { |
|
|
|
|
{ |
|
|
|
@ -379,7 +379,7 @@ bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
|
|
|
|
|
else if (buf == "continue") { |
|
|
|
|
const auto lc = idx.lineCol(); |
|
|
|
|
root.addToken("", TokenType::Continue, lc.first, lc.second); |
|
|
|
|
root.addToken("", STokenType::Continue, lc.first, lc.second); |
|
|
|
|
buf.clear(); |
|
|
|
|
if (!whitespace()) return false; |
|
|
|
|
if (data[idx()] != ';') |
|
|
|
@ -389,7 +389,7 @@ bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
|
|
|
|
|
else if (buf == "break") { |
|
|
|
|
const auto lc = idx.lineCol(); |
|
|
|
|
root.addToken("", TokenType::Break, lc.first, lc.second); |
|
|
|
|
root.addToken("", STokenType::Break, lc.first, lc.second); |
|
|
|
|
buf.clear(); |
|
|
|
|
if (!whitespace()) return false; |
|
|
|
|
if (data[idx()] != ';') |
|
|
|
@ -398,7 +398,7 @@ bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
} |
|
|
|
|
break; |
|
|
|
|
|
|
|
|
|
case TokenType::Reference: |
|
|
|
|
case STokenType::Reference: |
|
|
|
|
++idx; |
|
|
|
|
buf.clear(); |
|
|
|
|
if (!keyword()) return false; |
|
|
|
@ -413,7 +413,7 @@ bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
} |
|
|
|
|
break; |
|
|
|
|
|
|
|
|
|
case TokenType::MapKeyword: |
|
|
|
|
case STokenType::MapKeyword: |
|
|
|
|
{ |
|
|
|
|
const auto lc = idx.lineCol(); |
|
|
|
|
MapToken& token = root.addMapToken(buf, lc.first, lc.second); |
|
|
|
@ -422,7 +422,7 @@ bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
} |
|
|
|
|
continue; |
|
|
|
|
|
|
|
|
|
case TokenType::Operator: |
|
|
|
|
case STokenType::Operator: |
|
|
|
|
if (!oper()) return false; |
|
|
|
|
if (buf == "=") { |
|
|
|
|
++idx; |
|
|
|
@ -440,21 +440,21 @@ bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
|
|
|
|
|
/* Check if this is subtraction or unary minus */ |
|
|
|
|
if (buf == "-") { |
|
|
|
|
if (root.count() == 0 || (root.back().type() == TokenType::Operator)) |
|
|
|
|
if (root.count() == 0 || (root.back().type() == STokenType::Operator)) |
|
|
|
|
continue; |
|
|
|
|
} |
|
|
|
|
break; |
|
|
|
|
|
|
|
|
|
case TokenType::StringLiteral: |
|
|
|
|
case STokenType::StringLiteral: |
|
|
|
|
if (!string()) return false; |
|
|
|
|
break; |
|
|
|
|
|
|
|
|
|
case TokenType::NumberLiteral: |
|
|
|
|
case STokenType::NumberLiteral: |
|
|
|
|
// May turn the type into a FloatLiteral
|
|
|
|
|
if (!number()) return false; |
|
|
|
|
break; |
|
|
|
|
|
|
|
|
|
case TokenType::BoolLiteral: |
|
|
|
|
case STokenType::BoolLiteral: |
|
|
|
|
{ |
|
|
|
|
const auto lc = idx.lineCol(); |
|
|
|
|
// buf is peeked and stored via determineType().
|
|
|
|
@ -463,7 +463,7 @@ bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
continue; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
case TokenType::Expression: |
|
|
|
|
case STokenType::Expression: |
|
|
|
|
{ |
|
|
|
|
++idx; |
|
|
|
|
const auto lc = idx.lineCol(); |
|
|
|
@ -474,20 +474,20 @@ bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
continue; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
case TokenType::EndScope: [[fallthrough]]; |
|
|
|
|
case TokenType::EndExpression: [[fallthrough]]; |
|
|
|
|
case TokenType::Comma: |
|
|
|
|
case STokenType::EndScope: [[fallthrough]]; |
|
|
|
|
case STokenType::EndExpression: [[fallthrough]]; |
|
|
|
|
case STokenType::Comma: |
|
|
|
|
++idx; |
|
|
|
|
[[fallthrough]]; |
|
|
|
|
|
|
|
|
|
case TokenType::SemiColon: |
|
|
|
|
case STokenType::SemiColon: |
|
|
|
|
return true; |
|
|
|
|
|
|
|
|
|
case TokenType::Comment: |
|
|
|
|
case STokenType::Comment: |
|
|
|
|
comment(); |
|
|
|
|
continue; |
|
|
|
|
|
|
|
|
|
case TokenType::Function: |
|
|
|
|
case STokenType::Function: |
|
|
|
|
{ |
|
|
|
|
std::string fname; |
|
|
|
|
{ |
|
|
|
@ -505,7 +505,7 @@ bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
continue; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
case TokenType::CondIf: |
|
|
|
|
case STokenType::CondIf: |
|
|
|
|
{ |
|
|
|
|
if (!whitespace()) return false; |
|
|
|
|
++idx; |
|
|
|
@ -521,7 +521,7 @@ bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
} |
|
|
|
|
return true; |
|
|
|
|
|
|
|
|
|
case TokenType::CondWhile: |
|
|
|
|
case STokenType::CondWhile: |
|
|
|
|
{ |
|
|
|
|
if (!whitespace()) return false; |
|
|
|
|
++idx; |
|
|
|
@ -536,7 +536,7 @@ bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
} |
|
|
|
|
return true; |
|
|
|
|
|
|
|
|
|
case TokenType::CondElse: |
|
|
|
|
case STokenType::CondElse: |
|
|
|
|
{ |
|
|
|
|
const auto lc = idx.lineCol(); |
|
|
|
|
ConditionalScopeToken& scopeTk = root.addElseScope(lc.first, lc.second); |
|
|
|
@ -545,7 +545,7 @@ bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
} |
|
|
|
|
return true; |
|
|
|
|
|
|
|
|
|
case TokenType::Dispatch: |
|
|
|
|
case STokenType::Dispatch: |
|
|
|
|
{ |
|
|
|
|
buf.clear(); |
|
|
|
|
if (!whitespace()) return false; |
|
|
|
@ -568,7 +568,7 @@ bool TokenizerPriv::processExpression(Expression& root) |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (curType != TokenType::SemiColon) { |
|
|
|
|
if (curType != STokenType::SemiColon) { |
|
|
|
|
const auto lc = idx.lineCol(); |
|
|
|
|
root.addToken(buf, type, lc.first, lc.second); |
|
|
|
|
buf.clear(); |
|
|
|
@ -600,10 +600,10 @@ bool TokenizerPriv::processStatementList(Scope& scope, bool parameterList) |
|
|
|
|
if (expr.count() > 0) |
|
|
|
|
scope.addStatement(std::move(expr)); |
|
|
|
|
|
|
|
|
|
if ((parameterList && curType == TokenType::EndExpression) || curType == TokenType::EndScope) |
|
|
|
|
if ((parameterList && curType == STokenType::EndExpression) || curType == STokenType::EndScope) |
|
|
|
|
return true; |
|
|
|
|
|
|
|
|
|
if (curType == TokenType::SemiColon) { |
|
|
|
|
if (curType == STokenType::SemiColon) { |
|
|
|
|
++idx; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -690,7 +690,7 @@ bool TokenizerPriv::processDialog(Dialog& dialog) |
|
|
|
|
buf.clear(); |
|
|
|
|
if (!determineType(false)) return false; |
|
|
|
|
|
|
|
|
|
if (curType != TokenType::Keyword) { |
|
|
|
|
if (curType != STokenType::Keyword) { |
|
|
|
|
errorMsg = "Expected a keyword"; |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
@ -711,7 +711,7 @@ bool TokenizerPriv::processDialog(Dialog& dialog) |
|
|
|
|
else if (buf == "hook") { |
|
|
|
|
buf.clear(); |
|
|
|
|
if (!determineType(false)) return false; |
|
|
|
|
if (curType != TokenType::Keyword) { |
|
|
|
|
if (curType != STokenType::Keyword) { |
|
|
|
|
errorMsg = "Expected a keyword"; |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
@ -755,22 +755,22 @@ bool TokenizerPriv::processDialogAttr(T& dialog_or_widget) |
|
|
|
|
|
|
|
|
|
ValueHolder value; |
|
|
|
|
switch (curType) { |
|
|
|
|
case TokenType::StringLiteral: |
|
|
|
|
case STokenType::StringLiteral: |
|
|
|
|
buf.clear(); |
|
|
|
|
if (!string()) return false; |
|
|
|
|
value = buf; |
|
|
|
|
break; |
|
|
|
|
|
|
|
|
|
case TokenType::NumberLiteral: |
|
|
|
|
case STokenType::NumberLiteral: |
|
|
|
|
buf.clear(); |
|
|
|
|
if (!number()) return false; // side-effect: may change current type to a real number.
|
|
|
|
|
if (curType == TokenType::RealLiteral) |
|
|
|
|
if (curType == STokenType::RealLiteral) |
|
|
|
|
value = std::stod(buf); |
|
|
|
|
else |
|
|
|
|
value = std::stoi(buf); |
|
|
|
|
break; |
|
|
|
|
|
|
|
|
|
case TokenType::BoolLiteral: |
|
|
|
|
case STokenType::BoolLiteral: |
|
|
|
|
value = buf == "true" ? true : false; |
|
|
|
|
break; |
|
|
|
|
|
|
|
|
@ -799,7 +799,7 @@ bool TokenizerPriv::processDialogWidget(Dialog& dialog, DialogWidget::Type wtype |
|
|
|
|
if (!whitespace()) return false; |
|
|
|
|
if (!determineType(false)) return false; |
|
|
|
|
|
|
|
|
|
if (curType != TokenType::Keyword) { |
|
|
|
|
if (curType != STokenType::Keyword) { |
|
|
|
|
errorMsg = "Expected a keyword when declaring a widget in dialog '" + dialog.name() + "'"; |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
@ -825,7 +825,7 @@ bool TokenizerPriv::processDialogWidget(Dialog& dialog, DialogWidget::Type wtype |
|
|
|
|
buf.clear(); |
|
|
|
|
if (!determineType(false)) return false; |
|
|
|
|
|
|
|
|
|
if (curType != TokenType::Keyword) { |
|
|
|
|
if (curType != STokenType::Keyword) { |
|
|
|
|
errorMsg = "Expected a keyword"; |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
@ -834,7 +834,7 @@ bool TokenizerPriv::processDialogWidget(Dialog& dialog, DialogWidget::Type wtype |
|
|
|
|
else if (buf == "hook") { |
|
|
|
|
buf.clear(); |
|
|
|
|
if (!determineType(false)) return false; |
|
|
|
|
if (curType != TokenType::Keyword) { |
|
|
|
|
if (curType != STokenType::Keyword) { |
|
|
|
|
errorMsg = "Expected a keyword"; |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|