+ | qstring ->
+ return lexbuf ("QSTRING", remove_quotes (Ulexing.utf8_lexeme lexbuf))
+ | csymbol ->
+ return lexbuf ("CSYMBOL", remove_left_quote (Ulexing.utf8_lexeme lexbuf))
+ | "${" -> read_unparsed_group "UNPARSED_META" lexbuf
+ | "@{" -> read_unparsed_group "UNPARSED_AST" lexbuf
+ | '(' -> return lexbuf ("LPAREN", "")
+ | ')' -> return lexbuf ("RPAREN", "")
+ | meta_ident ->
+ return lexbuf ("UNPARSED_META",
+ remove_left_quote (Ulexing.utf8_lexeme lexbuf))
+ | meta_anonymous -> return lexbuf ("UNPARSED_META", "anonymous")
+ | note ->
+ let comment =
+ Ulexing.utf8_sub_lexeme lexbuf 2 (Ulexing.lexeme_length lexbuf - 4)
+ in
+ return lexbuf ("NOTE", comment)
+ | begincomment -> return lexbuf ("BEGINCOMMENT","")
+ | endcomment -> return lexbuf ("ENDCOMMENT","")
+ | eof -> return lexbuf ("EOI", "")
+ | _ -> return lexbuf ("SYMBOL", Ulexing.utf8_lexeme lexbuf)
+
+let rec level1_pattern_token = lexer
+ | xml_blank+ -> level1_pattern_token lexbuf
+ | number -> return lexbuf ("NUMBER", Ulexing.utf8_lexeme lexbuf)
+ | ident ->
+ let s = Ulexing.utf8_lexeme lexbuf in
+ begin
+ if List.mem s level1_keywords then
+ return lexbuf ("", s)
+ else
+ return lexbuf ("IDENT", s)
+ end
+ | tex_token -> return lexbuf (expand_macro lexbuf)
+ | qkeyword ->
+ return lexbuf ("QKEYWORD", remove_quotes (Ulexing.utf8_lexeme lexbuf))
+ | '(' -> return lexbuf ("LPAREN", "")
+ | ')' -> return lexbuf ("RPAREN", "")