idealirc/Script/Tokenizer.cpp

#include "Tokenizer.h"
#include <iostream>
#include <locale>

namespace TKN
{
constexpr auto* Function = "function";
constexpr auto* Command = "command";
constexpr auto* If = "if";
constexpr auto* Else = "else";
constexpr auto* While = "while";
constexpr auto* Hook = "hook";
constexpr auto* Dispatch = "dispatch";
constexpr auto* True = "true";
constexpr auto* False = "false";
constexpr auto* Global = "global";
constexpr auto* Menu = "menu";
constexpr auto* Dialog = "dialog";
}

namespace {
inline char decimalSeparator()
{
	return std::use_facet<std::numpunct<char>>(std::cout.getloc()).decimal_point();
}
}

class DataIndex
{
	const std::string& m_data;
	std::size_t m_idx{ 0 };
	int m_line{ 1 };
	int m_col{ 1 };

public:
	explicit DataIndex(const std::string& data)
		: m_data(data)
	{}

	DataIndex(const DataIndex& other)
		: m_data(other.m_data)
		, m_idx(other.m_idx)
		, m_line(other.m_line)
		, m_col(other.m_col)
	{}

	DataIndex& operator=(const DataIndex& other)
	{
		m_idx = other.m_idx;
		m_line = other.m_line;
		m_col = other.m_col;
		return *this;
	}

	void operator++()
	{
		m_idx += 1;
		if (m_data[m_idx] == '\n') {
			++m_line;
			m_col = 0;
		}
		++m_col;
	}

	bool operator<(std::size_t other)
	{
		return m_idx < other;
	}

	bool operator>(std::size_t other)
	{
		return m_idx > other;
	}

	bool operator==(std::size_t other)
	{
		return m_idx == other;
	}

	std::size_t operator()()
	{
		return m_idx;
	}

	std::pair<int,int> lineCol() const
	{
		return std::make_pair(m_line, m_col);
	}
};

struct TokenizerPriv
{
	const std::string& data;

	std::string buf{ "" };
	std::string errorMsg{ "" };

	DataIndex idx{ data };
	TokenType curType{ TokenType::Undefined };

	std::vector<std::unique_ptr<Scope>> scopes{};
	std::vector<std::string> globals{};
	std::vector<Menu> menus{};
	std::vector<Dialog> dialogs{};

	bool whitespace(bool eofIsError = true);
	bool determineType(bool blankExpressionRelated);

	bool processExpression(Expression& root);
	bool processStatementList(Scope& scope, bool parameterList = false);
	bool processSubmenu(MenuEntry& submenu);
	bool processDialog(Dialog& dialog);
	template<typename T> bool processDialogAttr(T& dialog_or_widget);
	bool processDialogWidgetTab(Dialog& dialog);
	bool processDialogWidget(Dialog& dialog, DialogWidget::Type wtype);

    bool keywordChar(char c, bool allowNumbers = true);
	std::pair<std::string, DataIndex> peekNextKeyword(DataIndex idx);
	std::string peekPreviousKeyword(DataIndex idx);

	bool command();
	bool keyword();
	bool oper();
	bool string();
	bool number();
	void comment();
};

bool TokenizerPriv::whitespace(bool eofIsError)
{
	for (; idx < data.length(); ++idx) {
		char c = data[idx()];
        if (c == '#') {
            comment();
            if (idx == data.length())
                break;
			c = data[idx()];
        }

		if (!std::isspace(c))
			return true;
	}

	if (eofIsError) {
		errorMsg = "Unexpected end-of-file";
		return false;
	}
	else
		return true;
}

bool TokenizerPriv::determineType(bool blankExpressionRelated)
{
    if (!whitespace()) return false;
	const char c = data[idx()];

	if (c == '/' && blankExpressionRelated)
		curType = TokenType::Command;

	else if (c == '&' && blankExpressionRelated)
		curType = TokenType::Reference;

	else if (keywordChar(c, false))
		curType = TokenType::Keyword;

	else if (c >= '0' && c <= '9')
		curType = TokenType::NumberLiteral; // RealLiteral will be detected during parsing of this type

	else if (c == '"')
		curType = TokenType::StringLiteral;

	else if (c == '{')
		curType = TokenType::Scope;

	else if (c == '}')
		curType = TokenType::EndScope;

	else if (c == '(') {
		if (idx > 0) {
			std::string prev = peekPreviousKeyword(idx);
			if (prev.empty()) {
				errorMsg = "Syntax error";
				return false;
			}
			if (keywordChar(prev[0], false))
				curType = TokenType::Function;
			else
				curType = TokenType::Expression;
		}
		else
			curType = TokenType::Expression;
	}

	else if (c == ')' || c == ']')
		curType = TokenType::EndExpression;

	else if (c == '+' || c == '-' || c == '*' || c == '/' || c == '!'
			 || c == '&' || c == '|' || c == '<' || c == '>' || c == '=')
		curType = TokenType::Operator;

	else if (c == ',')
		curType = TokenType::Comma;

	else if (c == ';')
		curType = TokenType::SemiColon;

	else {
		curType = TokenType::Undefined;
		errorMsg = "Stray character: ";
		errorMsg.push_back(c);
		return false;
	}

    /* Peek next keyword for specials like 'if', 'else', etc. */
    if (curType == TokenType::Keyword) {
		auto peek = peekNextKeyword(idx);

		if (peek.first == TKN::If)
            curType = TokenType::CondIf;
		else if (peek.first == TKN::Else)
            curType = TokenType::CondElse;
		else if (peek.first == TKN::While)
            curType = TokenType::CondWhile;
		else if (peek.first == TKN::Dispatch)
			curType = TokenType::Dispatch;
		else if (peek.first == TKN::False || peek.first == TKN::True) {
			curType = TokenType::BoolLiteral;
			buf = peek.first;
		}
		else if (data[peek.second()] == '[') {
			curType = TokenType::MapKeyword;
			buf = peek.first;
			++peek.second;
		}

        if (curType != TokenType::Keyword)
            idx = peek.second;
    }

	return true;
}

bool TokenizerPriv::command()
{
	++idx;

	for (; idx < data.length(); ++idx) {
		const char c = data[idx()];
		if (c == '\n') break;
		if (c == '\r') continue;
		buf += c;
	}
	return true;
}

bool TokenizerPriv::keyword()
{
	for (; idx < data.length(); ++idx) {
		const char c = data[idx()];
        if (keywordChar(c)) {
			buf += c;
		}
		else
			break;
	}
	return true;
}

bool TokenizerPriv::oper()
{
	for (; idx < data.length(); ++idx) {
		const char c = data[idx()];
		if (c == '+' || c == '-' || c == '*' || c == '/' || c == '!'
				|| c == '&' || c == '|' || c == '<' || c == '>' || c == '=')
		{
			buf += c;
		}
		else
			break;
	}
	return true;
}

bool TokenizerPriv::string()
{
	for (++idx; idx < data.length(); ++idx) {
		const char c = data[idx()];
		if (c == '"') {
			++idx;
			break;
		}
		if (c == '\\') {
			++idx;
			if (idx() + 1 == data.length()) {
				errorMsg = "Escape on end-of-file";
				return false;
			}
		}
		buf += c;
	}
	return true;
}

bool TokenizerPriv::number()
{
	bool decimalsep = false;
	for (; idx < data.length(); ++idx) {
		const char c = data[idx()];
		if (c == '.' && decimalsep) {
			errorMsg = "Multiple decimal separators";
			return false;
		}
		if (c >= '0' && c <= '9') {
			buf += c;
		}
		else if (c == '.') {
			buf += decimalSeparator();
			decimalsep = true;
			// Side effect:
			curType = TokenType::RealLiteral;
		}
		else
			break;
	}

	return true;
}

void TokenizerPriv::comment()
{
	for (; idx < data.length(); ++idx) {
		if (data[idx()] == '\n')
			break;
	}
}

bool TokenizerPriv::processExpression(Expression& root)
{
	const TokenType& type = curType;

	while (idx < data.length()) {
		if (!determineType(root.count() == 0)) return false;

		switch (type) {
		case TokenType::Command:
		{
			if (!command()) return false;
			if (buf.empty()) {
				errorMsg = "Empty command statement";
				return false;
			}
			const auto lc = idx.lineCol();
			root.addToken(buf, type, lc.first, lc.second);
            buf.clear();
            return true;
		}

		case TokenType::Keyword: // TODO return as own token type
			if (!keyword()) return false;
			if (buf == "return") {
				{
					const auto lc = idx.lineCol();
					root.addToken(buf, type, lc.first, lc.second);
				}
				buf.clear();
				const auto lc = idx.lineCol();
				Expression& expr = root.addExpression(lc.first, lc.second);
				if (!processExpression(expr)) return false;
                if (expr.count() == 0)
                    root.eraseBack();
			}

			else if (buf == "continue") {
				const auto lc = idx.lineCol();
				root.addToken("", TokenType::Continue, lc.first, lc.second);
				buf.clear();
				if (!whitespace()) return false;
				if (data[idx()] != ';')
					errorMsg = "Expected empty statement after 'continue'.";
				continue;
			}

			else if (buf == "break") {
				const auto lc = idx.lineCol();
				root.addToken("", TokenType::Break, lc.first, lc.second);
				buf.clear();
				if (!whitespace()) return false;
				if (data[idx()] != ';')
					errorMsg = "Expected empty statement after 'break'.";
				continue;
			}
		break;

		case TokenType::Reference:
			++idx;
			buf.clear();
			if (!keyword()) return false;
			if (!whitespace()) return false;
			if (data[idx()] == '[') { // Reference to array element
				++idx;
				const auto lc = idx.lineCol();
				MapToken& token = root.addMapToken(buf, lc.first, lc.second, /*isRef*/ true);
				buf.clear();
				if (!processExpression(token.subscript())) return false;
				continue;
			}
			break;

		case TokenType::MapKeyword:
		{
			const auto lc = idx.lineCol();
			MapToken& token = root.addMapToken(buf, lc.first, lc.second);
			buf.clear();
			if (!processExpression(token.subscript())) return false;
		}
		continue;

		case TokenType::Operator:
			if (!oper()) return false;
            if (buf == "=") {
				++idx;
				{
					const auto lc = idx.lineCol();
					root.addToken(buf, type, lc.first, lc.second);
				}
				buf.clear();
				const auto lc = idx.lineCol();
				Expression& expr = root.addExpression(lc.first, lc.second);
				if (!processExpression(expr)) return false;
				if (expr.count() == 0)
                    root.eraseBack();
			}

			/* Check if this is subtraction or unary minus */
			if (buf == "-") {
				if (root.count() == 0 || (root.back().type() == TokenType::Operator))
					continue;
			}
			break;

		case TokenType::StringLiteral:
			if (!string()) return false;
			break;

		case TokenType::NumberLiteral:
			// May turn the type into a FloatLiteral
			if (!number()) return false;
			break;

		case TokenType::BoolLiteral:
		{
			const auto lc = idx.lineCol();
			// buf is peeked and stored via determineType().
			root.addBool(buf == "true", lc.first, lc.second);
			buf.clear();
			continue;
		}

		case TokenType::Expression:
		{
			++idx;
			const auto lc = idx.lineCol();
			Expression& expr = root.addExpression(lc.first, lc.second);
			if (!processExpression(expr)) return false;
            if (expr.count() == 0)
                root.eraseBack();
			continue;
		}

		case TokenType::EndScope: [[fallthrough]];
		case TokenType::EndExpression: [[fallthrough]];
        case TokenType::Comma:
			++idx;
			 [[fallthrough]];

		case TokenType::SemiColon:
			return true;

		case TokenType::Comment:
			comment();
			continue;

		case TokenType::Function:
		{
			std::string fname;
			{
				Token& fnametk = static_cast<Token&>(root.back());
				fname = fnametk.token();
				root.eraseBack();
			}

			const auto lc = idx.lineCol();
			FunctionToken& fun = root.addFunction(fname, lc.first, lc.second);

			++idx;
			if (!processStatementList(fun.parameters(), true)) return false;

			continue;
		}

		case TokenType::CondIf:
		{
			if (!whitespace()) return false;
			++idx;
			buf.clear();

			const auto lc1 = idx.lineCol();
			Expression condition(lc1.first, lc1.second);
			if (!processExpression(condition)) return false;

			const auto lc2 = idx.lineCol();
			ConditionalScopeToken& scopeTk = root.addIfScope(lc2.first, lc2.second, std::move(condition));
			if (!processStatementList(scopeTk.scope())) return false;
		}
		return true;

		case TokenType::CondWhile:
		{
			if (!whitespace()) return false;
            ++idx;
			buf.clear();
			const auto lc1 = idx.lineCol();
			Expression condition(lc1.first, lc1.second);
			if (!processExpression(condition)) return false;

			const auto lc2 = idx.lineCol();
			ConditionalScopeToken& scopeTk = root.addWhileScope(lc2.first, lc2.second, std::move(condition));
			if (!processStatementList(scopeTk.scope())) return false;
		}
		return true;

		case TokenType::CondElse:
		{
			const auto lc = idx.lineCol();
			ConditionalScopeToken& scopeTk = root.addElseScope(lc.first, lc.second);
			buf.clear();
			if (!processStatementList(scopeTk.scope())) return false;
		}
		return true;

		case TokenType::Dispatch:
		{
			buf.clear();
			if (!whitespace()) return false;
			if (!keyword()) return false;
			const auto lc = idx.lineCol();
			DispatchToken& dispTk = root.addDispatchToken(buf, lc.first, lc.second);
			if (!whitespace()) return false;
			if (data[idx()] == '(') {
				++idx;
				buf.clear();
				if (!processStatementList(dispTk.parameters(), true)) return false;
				++idx;
			}
			buf.clear();
			continue;
		}

		default:
			errorMsg = "Invalid token";
			return false;
		}

		if (curType != TokenType::SemiColon) {
			const auto lc = idx.lineCol();
			root.addToken(buf, type, lc.first, lc.second);
			buf.clear();
		}
	}

	return true;
}

bool TokenizerPriv::processStatementList(Scope& scope, bool parameterList)
{
	if (!whitespace()) return false;
	bool multipleStatements = parameterList;
	if (data[idx()] == '{') {
		multipleStatements = true;
		++idx;
	}

	while (idx < data.length()) {
		if (!whitespace()) return false;
		if (data[idx()] == '}') {
			++idx;
			return true;
		}

		const auto lc = idx.lineCol();
		Expression expr(lc.first, lc.second);
		if (!processExpression(expr)) return false;
        if (expr.count() > 0)
			scope.addStatement(std::move(expr));

		if ((parameterList && curType == TokenType::EndExpression) || curType == TokenType::EndScope)
            return true;

		if (curType == TokenType::SemiColon) {
			++idx;
		}

		if (!multipleStatements)
			return true;
	}

	errorMsg = "Unexpected end-of-file";
	return false;
}

bool TokenizerPriv::processSubmenu(MenuEntry& submenu)
{
	if (data[idx()] != '{') {
		errorMsg = "Expected a '{' for defining the menu";
		return false;
	}
	++idx;
	while (idx < data.length() - 2) { // -2 so we avoid any reading past data[]
		if (!whitespace()) return false;
		if (data[idx()] == '}') { // End submenu
			++idx;
			return true;
		}

		if (data[idx()] == 'd') {
			auto kwp = peekNextKeyword(idx);
			if (kwp.first != "divide") {
				errorMsg = "Expected either a submenu, an item or divide";
				return false;
			}
			idx = kwp.second;
			submenu.addDivide();
			continue;
		}

		if (data[idx()] != '"') {
			errorMsg = "Expected a string constant";
			return false;
		}

		if (!string()) return false;
		const std::string label = buf;
		buf.clear();

		if (!whitespace()) return false;
		if (data[idx()] == '=' && data[idx() + 1] == '>') { // Menu item
			if (!oper()) return false;
			buf.clear();
			if (!whitespace()) return false;
			const auto lc = idx.lineCol();
			Scope scope(lc.first, lc.second);
			if (!processStatementList(scope)) return false;
			submenu.addItem(label, std::move(scope));
		}
		else if (data[idx()] == '{') { // Submenu
			MenuEntry& entry = submenu.addSubMenu(label);
			if (!processSubmenu(entry)) return false;
		}
		else {
			errorMsg = "Expected either a submenu, an item or divide";
			return false;
		}
	}

	errorMsg = "Unexpected end-of-file";
	return false;
}

bool TokenizerPriv::processDialog(Dialog& dialog)
{
	if (data[idx()] != '{') {
		errorMsg = "Expected a '{' to define dialog attributes";
		return false;
	}
	++idx;

	while (idx < data.length() - 2) { // -2 so we avoid any reading past data[]
		if (!whitespace()) return false;
		if (data[idx()] == '}') { // End dialog
			++idx;
			return true;
		}
		buf.clear();
		if (!determineType(false)) return false;

		if (curType != TokenType::Keyword) {
			errorMsg = "Expected a keyword";
			return false;
		}

		if (!keyword()) return false;

		auto maybeWidgetType = DialogWidget::strToType(buf);
		if (maybeWidgetType) {
			switch (*maybeWidgetType) {
			case DialogWidget::Type::Tab:
				if (!processDialogWidgetTab(dialog)) return false;
				break;
			default:
				if (!processDialogWidget(dialog, *maybeWidgetType)) return false;
			}
		}

		else if (buf == "hook") {
			buf.clear();
			if (!determineType(false)) return false;
			if (curType != TokenType::Keyword) {
				errorMsg = "Expected a keyword";
				return false;
			}
			if (!keyword()) return false;
			std::string identifier = buf;
			buf.clear();
			if (!whitespace()) return false;
			if (!oper()) return false;
			if (buf != "=>") {
				errorMsg = "Expected operator '=>' to define a scope for the hook '" + identifier + "'";
				return false;
			}
			if (!whitespace()) return false;
			Scope scope(idx.lineCol());
			buf.clear();
			if (!processStatementList(scope)) return false;
			dialog.addHook(identifier, std::move(scope));
		}

		else // Not a widget type: declare as an attribute.
			if (!processDialogAttr(dialog)) return false;
	}

	errorMsg = "Unexpected end-of-file";
	return false;
}

template<typename T>
bool TokenizerPriv::processDialogAttr(T& dialog_or_widget)
{
	if (!whitespace()) return false;
	std::string identifier = buf;
	buf.clear();
	if (!oper()) return false;
	if (buf != "=") {
		errorMsg = "Expected a '=' to define the attribute '" + identifier + "'";
		return false;
	}
	if (!whitespace()) return false;
	if (!determineType(false)) return false;

	ValueHolder value;
	switch (curType) {
	case TokenType::StringLiteral:
		buf.clear();
		if (!string()) return false;
		value = buf;
		break;

	case TokenType::NumberLiteral:
		buf.clear();
		if (!number()) return false; // side-effect: may change current type to a real number.
		if (curType == TokenType::RealLiteral)
			value = std::stod(buf);
		else
			value = std::stoi(buf);
		break;

	case TokenType::BoolLiteral:
		value = buf == "true" ? true : false;
		break;

	default:
		errorMsg = "Expected literal value for defining the attribute '" + identifier + "'";
		return false;
	}
	dialog_or_widget.setDefaultAttr(identifier, value);

	if (!whitespace()) return false;
	if (data[idx()] != ';') {
		errorMsg = "Expected a ';' to declare the end of attribute '" + identifier + "'";
		return false;
	}
	++idx;
	return true;
}

bool TokenizerPriv::processDialogWidgetTab(Dialog& dialog)
{
    return false; // TODO this function
}

bool TokenizerPriv::processDialogWidget(Dialog& dialog, DialogWidget::Type wtype)
{
	if (!whitespace()) return false;
	if (!determineType(false)) return false;

	if (curType != TokenType::Keyword) {
		errorMsg = "Expected a keyword when declaring a widget in dialog '" + dialog.name() + "'";
		return false;
	}

	buf.clear();
	if (!keyword()) return false;
	std::string widgetIdentifier = buf;
	if (!whitespace()) return false;
	if (data[idx()] != '{') {
		errorMsg = "Expected a '{' to define widget attributes for widget '" + widgetIdentifier + "' in dialog '" + dialog.name() + "'";
		return false;
	}
	++idx;

	auto& widget = dialog.addWidget(widgetIdentifier, wtype);

	while (idx < data.length() - 2) { // -2 so we avoid any reading past data[]
		if (!whitespace()) return false;
		if (data[idx()] == '}') { // End widget attributes
			++idx;
			return true;
		}
		buf.clear();
		if (!determineType(false)) return false;

		if (curType != TokenType::Keyword) {
			errorMsg = "Expected a keyword";
			return false;
		}

		if (!keyword()) return false;
		else if (buf == "hook") {
			buf.clear();
			if (!determineType(false)) return false;
			if (curType != TokenType::Keyword) {
				errorMsg = "Expected a keyword";
				return false;
			}
			if (!keyword()) return false;
			std::string hookIdentifier = buf;
			buf.clear();
			if (!whitespace()) return false;
			if (!oper()) return false;
			if (buf != "=>") {
				errorMsg = "Expected operator '=>' to define a scope for the hook '" + hookIdentifier + "'";
				return false;
			}
			if (!whitespace()) return false;
			Scope scope(idx.lineCol());
			buf.clear();
			if (!processStatementList(scope)) return false;
			widget.addHook(hookIdentifier, std::move(scope));
		}
		else if (!processDialogAttr(widget)) return false;
	}

	errorMsg = "Unexpected end-of-file";
	return false;
}

bool TokenizerPriv::keywordChar(char c, bool allowNumbers)
{
    if (allowNumbers)
        return ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_');
    else
        return ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_');
}

std::pair<std::string, DataIndex> TokenizerPriv::peekNextKeyword(DataIndex idx)
{
    std::string ret;
    for (; idx < data.length(); ++idx) {
		const char c = data[idx()];
        if (keywordChar(c))
            ret += c;
        else
            break;
    }
	return { ret, idx };
}

std::string TokenizerPriv::peekPreviousKeyword(DataIndex idx)
{
	std::string ret;
	long i = static_cast<long>(idx()) - 1;
	for (; i >= 0; --i) {
		const char c = data[i];
		if (!keywordChar(c))
			break;
	}

	if (i < 0)
		i = 0;
	else
		++i;

	// Here, 'i' will be the first character of the token, and 'idx' the end (exclusive, as STL end).
	return data.substr(i, data.length() - idx() - 1);
}

Tokenizer::Tokenizer(const std::string& input)
	: m_data(input)
{
	std::cout.imbue(std::locale(""));
}

Tokenizer::Tokenizer(Tokenizer&& other)
	: mp(std::move(other.mp))
	, m_data(std::move(other.m_data))
{}

Tokenizer::~Tokenizer() = default;

bool Tokenizer::process()
{
	mp.reset(new TokenizerPriv{ m_data });

	while (mp->idx() < m_data.length()) {
		mp->buf.clear();
		if (!nextKeyword()) return false; // Get "block" type

		if (mp->buf == TKN::Function) {
			if (!processImperativeBlock(ImperativeBlockType::Function))
				return false;
		}

		else if (mp->buf == TKN::Command) {
			if (!processImperativeBlock(ImperativeBlockType::Command))
				return false;
		}

		else if (mp->buf == TKN::Hook) {
			if (!processImperativeBlock(ImperativeBlockType::Hook))
				return false;
		}

		else if (mp->buf == TKN::Global) {
			if (!processGlobals())
				return false;
		}

		else if (mp->buf == TKN::Menu) {
			if (!processMenu())
				return false;
		}

		else if (mp->buf == TKN::Dialog) {
			if (!processDialog())
				return false;
		}

		else if (!mp->buf.empty()) {
			mp->errorMsg = "Undefined keyword: " + mp->buf;
			return false;
		}

		else if (mp->idx() == m_data.length() - 1) {
			mp->errorMsg = "Undefined error (end of buffer)";
			return false;
		}

		// TODO
		//else {
		//	mp->errorMsg = "Undefined error (empty buffer)";
		//	return false;
		//}
	}
	return true;
}

bool Tokenizer::processImperativeBlock(ImperativeBlockType blockType)
{
	if (!nextKeyword()) return false; // Get function name

	if (mp->buf.empty()) {
		mp->errorMsg = "Expected identifier";
		return false;
	}
	std::string identifier = mp->buf;

	if (!mp->whitespace()) return false;
	if (m_data[mp->idx()] != '(') {
		mp->errorMsg = "Expected parameter declaration";
		return false;
	}

	++mp->idx;
	{
		const auto lc1 = mp->idx.lineCol();
		Scope params(lc1.first, lc1.second);
        mp->buf.clear();
		if (!mp->processStatementList(params, true)) return false; // Read parameters
        if (!mp->whitespace()) return false;
		const auto lc2 = mp->idx.lineCol();
		mp->scopes.emplace_back(new ImperativeScope(lc2.first, lc2.second, blockType, identifier, std::move(params)));
	}

	Scope& scope = *(mp->scopes.back().get());
	if (!mp->processStatementList(scope)) return false;

	return true;
}

bool Tokenizer::processGlobals()
{
	mp->buf.clear();
	while (mp->idx < m_data.length()) {
		if (!mp->whitespace()) return false;
		if (!mp->keyword()) return false;
		if (!mp->whitespace()) return false;

		if (m_data[mp->idx()] == ';') {
			if (!mp->buf.empty())
				mp->globals.emplace_back(mp->buf);
			++mp->idx;
			mp->buf.clear();
			return true;
		}

		if (m_data[mp->idx()] == ',') {
			++mp->idx;
		}
		else {
			mp->errorMsg = "Expected a comma or semicolon after identifier";
			return false;
		}

		if (!mp->buf.empty()) {
			mp->globals.emplace_back(mp->buf);
			mp->buf.clear();
		}
		else {
			mp->errorMsg = "Expected an identifier";
			return false;
		}
	}
	mp->errorMsg = "Unexpected end-of-file";
	return false;
}

bool Tokenizer::processMenu()
{
	mp->buf.clear();
	if (!mp->whitespace()) return false;
	if (!mp->keyword()) return false;
	std::string menuName = mp->buf;
	mp->buf.clear();
	mp->menus.emplace_back(menuName);
	Menu& menu = mp->menus.back();
	if (!mp->whitespace()) return false;
	return mp->processSubmenu(menu.content());
}

bool Tokenizer::processDialog()
{
	mp->buf.clear();
	if (!mp->whitespace()) return false;
	if (!mp->keyword()) return false;
	std::string dialogName = mp->buf;
	mp->buf.clear();
	mp->dialogs.emplace_back(dialogName);
	Dialog& dialog = mp->dialogs.back();
	if (!mp->whitespace()) return false;
	return mp->processDialog(dialog);
}

const std::string& Tokenizer::lastError()
{
	return mp->errorMsg;
}

std::pair<int, int> Tokenizer::currentLineCol() const
{
	return mp->idx.lineCol();
}

bool Tokenizer::nextKeyword()
{
	mp->buf.clear();
	if (!mp->whitespace(/*eofIsError*/ false)) return true;
	if (!mp->keyword()) return false;
	return true;
}

Tokenizer::Iterator Tokenizer::begin() const
{
	return mp->scopes.begin();
}

Tokenizer::Iterator Tokenizer::end() const
{
	return mp->scopes.end();
}

const std::vector<std::string>& Tokenizer::globals()
{
	return mp->globals;
}

const std::vector<Menu>& Tokenizer::menus() const
{
	return mp->menus;
}

const std::vector<Dialog>& Tokenizer::dialogs() const
{
	return mp->dialogs;
}