1 /* Copyright (C) 2002-2003, Luca Padovani <luca.padovani@cs.unibo.it>,
2 * 2003, Paolo Marinelli <pmarinel@cs.unibo.it>.
4 * This file is part of EdiTeX, an editor of mathematical
5 * expressions based on TeX syntax
7 * EdiTeX is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 2
10 * of the License, or (at your option) any later version.
12 * EdiTeX is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with EdiTeX; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
21 * For details, see the EdiTeX World-Wide-Web page,
22 * http://helm.cs.unibo.it/editex, or send a mail to
23 * <luca.padovani@cs.unibo.it>
28 #include "TTokenizer.hh"
29 #include "TPushLexer.hh"
32 TTokenizer::tokenize(const std::string& s)
34 TPushLexer lexer(logger, *this);
37 for (std::string::const_iterator p = s.begin();
44 std::vector<TToken> res;
45 res.reserve(tokens.size());
46 copy(tokens.begin(), tokens.end(), back_inserter(res));
58 TTokenizer::push(const TToken& token)
60 tokens.push_back(token);
64 TTokenizer::drop(bool alt)