std::vector<TToken>
TTokenizer::tokenize(const std::string& s)
{
- TPushLexer lexer(*this);
+ TPushLexer lexer(logger, *this);
tokens.clear();
for (std::string::const_iterator p = s.begin();
p != s.end();
p++)
lexer.push(*p);
- //lexer.push('\n');
+
+ lexer.flush();
std::vector<TToken> res;
res.reserve(tokens.size());