4 #include "TTokenizer.hh"
5 #include "TPushLexer.hh"
8 TTokenizer::tokenize(const std::string& s)
10 TPushLexer lexer(*this);
13 for (std::string::const_iterator p = s.begin();
19 std::vector<TToken> res;
20 res.reserve(tokens.size());
21 copy(tokens.begin(), tokens.end(), back_inserter(res));
27 TTokenizer::push(const TToken& token)
29 tokens.push_back(token);