4 #include "TTokenizer.hh"
5 #include "TPushLexer.hh"
8 TTokenizer::tokenize(const std::string& s)
10 TPushLexer lexer(logger, *this);
13 for (std::string::const_iterator p = s.begin();
20 std::vector<TToken> res;
21 res.reserve(tokens.size());
22 copy(tokens.begin(), tokens.end(), back_inserter(res));
28 TTokenizer::push(const TToken& token)
30 tokens.push_back(token);