let regexp tex_token = '\\' ident
+let regexp delim_begin = "\\["
+let regexp delim_end = "\\]"
+
let regexp keyword = '"' ident '"'
let regexp implicit = '?'
| ident -> return lexbuf ("IDENT", Ulexing.utf8_lexeme lexbuf)
| number -> return lexbuf ("NUMBER", Ulexing.utf8_lexeme lexbuf)
| keyword -> return lexbuf (keyword lexbuf)
+ | delim_begin -> return lexbuf ("DELIM", Ulexing.utf8_lexeme lexbuf)
+ | delim_end -> return lexbuf ("DELIM", Ulexing.utf8_lexeme lexbuf)
| tex_token -> return lexbuf (expand_macro lexbuf)
| uri -> return lexbuf ("URI", Ulexing.utf8_lexeme lexbuf)
| eof -> return lexbuf ("EOI", "")