+++ /dev/null
-/* This file is part of EdiTeX, an editor of mathematical
- * expressions based on TeX syntax.
- *
- * Copyright (C) 2002-2003 Luca Padovani <lpadovan@cs.unibo.it>,
- * 2003 Paolo Marinelli <pmarinel@cs.unibo.it>.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- *
- * For more information, please visit the project's home page
- * http://helm.cs.unibo.it/editex/
- * or send an email to <lpadovan@cs.unibo.it>
- */
-
-#include <algorithm>
-#include <cassert>
-
-#include "TTokenizer.hh"
-#include "TPushLexer.hh"
-
-std::vector<TToken>
-TTokenizer::tokenize(const std::string& s)
-{
- TPushLexer lexer(logger, *this);
-
- tokens.clear();
- for (std::string::const_iterator p = s.begin();
- p != s.end();
- p++)
- lexer.push(*p);
-
- lexer.flush();
-
- std::vector<TToken> res;
- res.reserve(tokens.size());
- copy(tokens.begin(), tokens.end(), back_inserter(res));
-
- return res;
-}
-
-void
-TTokenizer::reset()
-{
- assert(0);
-}
-
-void
-TTokenizer::push(const TToken& token)
-{
- tokens.push_back(token);
-}
-
-std::string
-TTokenizer::drop(bool alt)
-{
- assert(0);
- return "";
-}