X-Git-Url: http://matita.cs.unibo.it/gitweb/?p=helm.git;a=blobdiff_plain;f=helm%2FDEVEL%2Fmathml_editor%2Fsrc%2FTTokenizer.cc;fp=helm%2FDEVEL%2Fmathml_editor%2Fsrc%2FTTokenizer.cc;h=cf74c1d4f2237e52601990fb4d6ee9e6c01e928d;hp=0000000000000000000000000000000000000000;hb=89262281b6e83bd2321150f81f1a0583645eb0c8;hpb=b1fb6b8e1767d775bc452303629e95941d142bea

diff --git a/helm/DEVEL/mathml_editor/src/TTokenizer.cc b/helm/DEVEL/mathml_editor/src/TTokenizer.cc
new file mode 100644
index 000000000..cf74c1d4f
--- /dev/null
+++ b/helm/DEVEL/mathml_editor/src/TTokenizer.cc
@@ -0,0 +1,31 @@
+
+#include <algorithm>
+
+#include "TTokenizer.hh"
+#include "TPushLexer.hh"
+
+std::vector<TToken>
+TTokenizer::tokenize(const std::string& s)
+{
+  TPushLexer lexer(*this);
+
+  tokens.clear();
+  for (std::string::const_iterator p = s.begin();
+       p != s.end();
+       p++)
+    lexer.push(*p);
+  //lexer.push('\n');
+
+  std::vector<TToken> res;
+  res.reserve(tokens.size());
+  copy(tokens.begin(), tokens.end(), back_inserter(res));
+
+  return res;
+}
+
+void
+TTokenizer::push(const TToken& token)
+{
+  tokens.push_back(token);
+}
+