Initial revision

[helm.git] / helm / DEVEL / mathml_editor / src / TTokenizer.cc

diff --git a/helm/DEVEL/mathml_editor/src/TTokenizer.cc b/helm/DEVEL/mathml_editor/src/TTokenizer.cc
new file mode 100644 (file)
index 0000000..cf74c1d
--- /dev/null
+++ b/helm/DEVEL/mathml_editor/src/TTokenizer.cc
@@ -0,0 +1,31 @@
+
+#include <algorithm>
+
+#include "TTokenizer.hh"
+#include "TPushLexer.hh"
+
+std::vector<TToken>
+TTokenizer::tokenize(const std::string& s)
+{
+  TPushLexer lexer(*this);
+
+  tokens.clear();
+  for (std::string::const_iterator p = s.begin();
+       p != s.end();
+       p++)
+    lexer.push(*p);
+  //lexer.push('\n');
+
+  std::vector<TToken> res;
+  res.reserve(tokens.size());
+  copy(tokens.begin(), tokens.end(), back_inserter(res));
+
+  return res;
+}
+
+void
+TTokenizer::push(const TToken& token)
+{
+  tokens.push_back(token);
+}
+