added new extractor based on MetadataDb.index_obj

diff --git a/helm/ocaml/metadata/Makefile b/helm/ocaml/metadata/Makefile
index 52fe27edd79b0e40ebab812bcf6b31c4723237f8..afacda168711889ade0709827818386069a680d0 100644 (file)
--- a/helm/ocaml/metadata/Makefile
+++ b/helm/ocaml/metadata/Makefile
@@ -15,23 +15,24 @@ EXTRA_OBJECTS_TO_CLEAN =
 
 include ../Makefile.common
 
-all: all_table_creator
-opt: opt_table_creator
+all: all_table_creator all_extractor
+opt: opt_table_creator opt_extractor
 
 all_table_creator:
        make -C table_creator/ all
 opt_table_creator:
        make -C table_creator/ opt
 
-clean: clean_table_creator
+all_extractor:
+       make -C extractor/ all
+opt_extractor:
+       make -C extractor/ opt
+
+clean: clean_table_creator clean_extractor
 
 clean_table_creator:
        make -C table_creator/ clean
        
-test: test.ml $(PACKAGE).cma
-       $(OCAMLFIND) ocamlc -thread -package mysql,helm-metadata -linkpkg -o $@ $<
-test.opt: test.ml $(PACKAGE).cmxa
-       $(OCAMLFIND) opt -thread -package mysql,helm-metadata -linkpkg -o $@ $<
-test_query: test_query.ml $(PACKAGE).cma
-       $(OCAMLFIND) ocamlc -thread -package mysql,helm-metadata -linkpkg -o $@ $<
-
+clean_extractor:
+       make -C extractor/ clean
+       

diff --git a/helm/ocaml/metadata/extractor/Makefile b/helm/ocaml/metadata/extractor/Makefile
new file mode 100644 (file)
index 0000000..1275aaa
--- /dev/null
+++ b/helm/ocaml/metadata/extractor/Makefile
@@ -0,0 +1,21 @@
+REQUIRES = mysql helm-metadata
+
+INTERFACE_FILES = 
+IMPLEMENTATION_FILES = $(INTERFACE_FILES:%.mli=%.ml) 
+EXTRA_OBJECTS_TO_INSTALL =
+EXTRA_OBJECTS_TO_CLEAN = \
+       extractor extractor.opt 
+
+all: extractor
+opt: extractor.opt
+
+extractor: extractor.ml
+       $(OCAMLFIND) ocamlc \
+               -thread -package mysql,helm-metadata -linkpkg -o $@ $<
+
+extractor.opt: extractor.ml
+       $(OCAMLFIND) ocamlopt \
+               -thread -package mysql,helm-metadata -linkpkg -o $@ $<
+
+include ../../Makefile.common
+include .depend

diff --git a/helm/ocaml/metadata/extractor/extractor.conf.xml b/helm/ocaml/metadata/extractor/extractor.conf.xml
new file mode 100644 (file)
index 0000000..8dbc9a9
--- /dev/null
+++ b/helm/ocaml/metadata/extractor/extractor.conf.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<helm_registry>
+  <section name="tmp">
+    <key name="dir">.tmp/</key>
+  </section>   
+  <section name="db">
+    <key name="host">localhost</key>
+    <key name="user">helm</key>
+    <key name="database">mowgli</key>
+  </section>
+  <section name="getter">
+    <key name="servers">
+      file:///projects/helm/library/coq_contribs
+    </key>
+    <key name="cache_dir">$(tmp.dir)/cache</key>
+    <key name="maps_dir">$(tmp.dir)/maps</key>
+    <key name="dtd_dir">/projects/helm/xml/dtd</key>
+  </section>
+</helm_registry>

diff --git a/helm/ocaml/metadata/extractor/extractor.ml b/helm/ocaml/metadata/extractor/extractor.ml
new file mode 100644 (file)
index 0000000..f3c39d3
--- /dev/null
+++ b/helm/ocaml/metadata/extractor/extractor.ml
@@ -0,0 +1,92 @@
+let _ = Helm_registry.load_from "extractor.conf.xml"
+let _ = Unix.system ("mkdir -p " ^ (Helm_registry.get "tmp.dir"))
+let _ = Http_getter.init () 
+(*let _ = Http_getter.update () *)
+
+let dbd =
+  Mysql.quick_connect 
+    ~host:(Helm_registry.get "db.host") 
+    ~user:(Helm_registry.get "db.user") 
+    ~database:(Helm_registry.get "db.database") ()
+
+let _ =
+  try
+    MetadataTypes.ownerize_tables Sys.argv.(1)
+  with Invalid_argument _ -> MetadataTypes.ownerize_tables "NEW"
+
+let uri_RE = Str.regexp ".*\\(ind\\|var\\|con\\)$"
+ 
+let create_all () =
+  let obj_tbl = MetadataTypes.obj_tbl () in
+  let sort_tbl = MetadataTypes.sort_tbl () in
+  let rel_tbl = MetadataTypes.rel_tbl () in
+  let name_tbl =  MetadataTypes.name_tbl () in
+  let count_tbl = MetadataTypes.count_tbl () in
+  let tbls = [ 
+    (obj_tbl,`RefObj) ; (sort_tbl,`RefSort) ; (rel_tbl,`RefRel) ;
+    (name_tbl,`ObjectName) ; (count_tbl,`Count) ] 
+  in
+  let statements = 
+    (SqlStatements.create_tables tbls) @ (SqlStatements.create_indexes tbls)
+  in
+  List.iter (fun statement -> 
+    try
+      ignore (Mysql.exec dbd statement)
+    with
+      exn -> 
+         let status = Mysql.status dbd in
+         match status with 
+         | Mysql.StatusError Mysql.Table_exists_error -> ()
+         | Mysql.StatusError _ -> raise exn
+         | _ -> ()
+      ) statements
+
+let drop_all () =
+  let obj_tbl = MetadataTypes.obj_tbl () in
+  let sort_tbl = MetadataTypes.sort_tbl () in
+  let rel_tbl = MetadataTypes.rel_tbl () in
+  let name_tbl =  MetadataTypes.name_tbl () in
+  let count_tbl = MetadataTypes.count_tbl () in
+  let tbls = [ 
+    (obj_tbl,`RefObj) ; (sort_tbl,`RefSort) ; (rel_tbl,`RefRel) ;
+    (name_tbl,`ObjectName) ; (count_tbl,`Count) ] 
+  in
+  let statements = 
+    (SqlStatements.drop_tables tbls) @ (SqlStatements.drop_indexes tbls)
+  in
+  List.iter (fun statement -> 
+    try
+      ignore (Mysql.exec dbd statement)
+    with Mysql.Error _ as exn ->
+      match Mysql.errno dbd with 
+      | Mysql.Bad_table_error 
+      | Mysql.No_such_index | Mysql.No_such_table -> () 
+      | _ -> raise exn
+    ) statements
+  
+
+let main () =
+  drop_all ();
+  create_all ();
+  let uris = Http_getter.getalluris () in
+  let uris = List.filter (fun u -> Str.string_match uri_RE u 0) uris in
+  let len = float_of_int (List.length uris) in
+  let i = ref 0 in
+  List.iter (fun u ->
+    incr i;
+    let perc = ((float_of_int !i)  /. len *. 100.0) in
+    let l = String.length u in
+    let short = 
+      if l < 60 then 
+        u ^ String.make (63 - l) ' ' 
+      else 
+        "..." ^  String.sub u (l - 60) 60
+    in
+    Printf.printf "\rIndexing (%3.1f%%): %s" perc short;
+    flush stdout;
+    let uri = UriManager.uri_of_string u in
+    MetadataDb.index_obj ~dbd ~uri)
+  uris
+;;
+
+main ()