First functions on substitutions for unification

[helm.git] / helm / software / components / content_pres / cicNotationLexer.ml

diff --git a/helm/software/components/content_pres/cicNotationLexer.ml b/helm/software/components/content_pres/cicNotationLexer.ml
index ac3b1142252dda604e88ccde5c86b32af757865b..17a5bd5bbdebc7f3a5d61876ece74478d9a34474 100644 (file)
--- a/helm/software/components/content_pres/cicNotationLexer.ml
+++ b/helm/software/components/content_pres/cicNotationLexer.ml
@@ -52,16 +52,21 @@ let regexp we_proved = "we" utf8_blank+ "proved"
 let regexp we_have = "we" utf8_blank+ "have"
 let regexp let_rec = "let" utf8_blank+ "rec" 
 let regexp let_corec = "let" utf8_blank+  "corec"
+let regexp nlet_rec = "nlet" utf8_blank+ "rec" 
+let regexp nlet_corec = "nlet" utf8_blank+  "corec"
 let regexp ident_decoration = '\'' | '?' | '`'
 let regexp ident_cont = ident_letter | xml_digit | '_'
-let regexp ident = ident_letter ident_cont* ident_decoration*
+let regexp ident_start = ident_letter 
+let regexp ident = ident_letter ident_cont* ident_decoration* 
+let regexp variable_ident = '_' '_' number
+let regexp pident = '_' ident
 
 let regexp tex_token = '\\' ident
 
 let regexp delim_begin = "\\["
 let regexp delim_end = "\\]"
 
-let regexp qkeyword = "'" ident "'"
+let regexp qkeyword = "'" ( ident | pident ) "'"
 
 let regexp implicit = '?'
 let regexp placeholder = '%'
@@ -205,7 +210,9 @@ let expand_macro lexbuf =
   in
   try
     ("SYMBOL", Utf8Macro.expand macro)
-  with Utf8Macro.Macro_not_found _ -> "SYMBOL", Ulexing.utf8_lexeme lexbuf
+  with Utf8Macro.Macro_not_found _ -> 
+(* FG: unexpanded TeX macros are terminated by a space for rendering *)     
+     "SYMBOL", (Ulexing.utf8_lexeme lexbuf ^ " ")
 
 let remove_quotes s = String.sub s 1 (String.length s - 2)
 let remove_left_quote s = String.sub s 1 (String.length s - 1)
@@ -230,17 +237,22 @@ let read_unparsed_group token_name lexbuf =
   let end_cnum = level2_pattern_token_group 0 buffer lexbuf in
     return_with_loc (token_name, Buffer.contents buffer) begin_cnum end_cnum
 
+let handle_keywords lexbuf k name = 
+  let s = Ulexing.utf8_lexeme lexbuf in
+  if k s then
+           return lexbuf ("", s)
+         else
+           return lexbuf (name, s)
+;;
+
 let rec level2_meta_token =
   lexer
   | utf8_blank+ -> level2_meta_token lexbuf
   | ident ->
-      let s = Ulexing.utf8_lexeme lexbuf in
-       begin
-         if List.mem s level2_meta_keywords then
-           return lexbuf ("", s)
-         else
-           return lexbuf ("IDENT", s)
-       end
+      handle_keywords lexbuf (fun x -> List.mem x level2_meta_keywords) "IDENT"
+  | variable_ident -> return lexbuf ("IDENT", Ulexing.utf8_lexeme lexbuf)
+  | pident ->
+      handle_keywords lexbuf (fun x -> List.mem x level2_meta_keywords) "PIDENT"
   | "@{" -> read_unparsed_group "UNPARSED_AST" lexbuf
   | ast_ident ->
       return lexbuf ("UNPARSED_AST",
@@ -284,6 +296,8 @@ and level2_ast_token =
   lexer
   | let_rec -> return lexbuf ("LETREC","")
   | let_corec -> return lexbuf ("LETCOREC","")
+  | nlet_rec -> return lexbuf ("NLETREC","")
+  | nlet_corec -> return lexbuf ("NLETCOREC","")
   | we_proved -> return lexbuf ("WEPROVED","")
   | we_have -> return lexbuf ("WEHAVE","")
   | utf8_blank+ -> ligatures_token level2_ast_token lexbuf
@@ -292,12 +306,9 @@ and level2_ast_token =
       return lexbuf ("META", String.sub s 1 (String.length s - 1))
   | implicit -> return lexbuf ("IMPLICIT", "")
   | placeholder -> return lexbuf ("PLACEHOLDER", "")
-  | ident ->
-      let lexeme = Ulexing.utf8_lexeme lexbuf in
-      if Hashtbl.mem !level2_ast_keywords lexeme then
-        return lexbuf ("", lexeme)
-      else
-        return lexbuf ("IDENT", lexeme)
+  | ident -> handle_keywords lexbuf (Hashtbl.mem !level2_ast_keywords) "IDENT"
+  | variable_ident -> return lexbuf ("IDENT", Ulexing.utf8_lexeme lexbuf)
+  | pident -> handle_keywords lexbuf (Hashtbl.mem !level2_ast_keywords) "PIDENT"
   | number -> return lexbuf ("NUMBER", Ulexing.utf8_lexeme lexbuf)
   | tex_token -> return lexbuf (expand_macro lexbuf)
   | uri -> return lexbuf ("URI", Ulexing.utf8_lexeme lexbuf)
@@ -329,14 +340,9 @@ and level1_pattern_token =
   lexer
   | utf8_blank+ -> ligatures_token level1_pattern_token lexbuf
   | number -> return lexbuf ("NUMBER", Ulexing.utf8_lexeme lexbuf)
-  | ident ->
-      let s = Ulexing.utf8_lexeme lexbuf in
-       begin
-         if List.mem s level1_keywords then
-           return lexbuf ("", s)
-         else
-           return lexbuf ("IDENT", s)
-       end
+  | ident ->handle_keywords lexbuf (fun x -> List.mem x level1_keywords) "IDENT"
+  | variable_ident -> return lexbuf ("IDENT", Ulexing.utf8_lexeme lexbuf)
+  | pident->handle_keywords lexbuf (fun x->List.mem x level1_keywords) "PIDENT" 
   | color -> return lexbuf ("COLOR", Ulexing.utf8_lexeme lexbuf)
   | percentage -> 
       return lexbuf ("PERCENTAGE", Ulexing.utf8_lexeme lexbuf)