added new extractor based on MetadataDb.index_obj

[helm.git] / helm / ocaml / metadata / extractor / extractor.ml

diff --git a/helm/ocaml/metadata/extractor/extractor.ml b/helm/ocaml/metadata/extractor/extractor.ml
new file mode 100644 (file)
index 0000000..f3c39d3
--- /dev/null
+++ b/helm/ocaml/metadata/extractor/extractor.ml
@@ -0,0 +1,92 @@
+let _ = Helm_registry.load_from "extractor.conf.xml"
+let _ = Unix.system ("mkdir -p " ^ (Helm_registry.get "tmp.dir"))
+let _ = Http_getter.init () 
+(*let _ = Http_getter.update () *)
+
+let dbd =
+  Mysql.quick_connect 
+    ~host:(Helm_registry.get "db.host") 
+    ~user:(Helm_registry.get "db.user") 
+    ~database:(Helm_registry.get "db.database") ()
+
+let _ =
+  try
+    MetadataTypes.ownerize_tables Sys.argv.(1)
+  with Invalid_argument _ -> MetadataTypes.ownerize_tables "NEW"
+
+let uri_RE = Str.regexp ".*\\(ind\\|var\\|con\\)$"
+ 
+let create_all () =
+  let obj_tbl = MetadataTypes.obj_tbl () in
+  let sort_tbl = MetadataTypes.sort_tbl () in
+  let rel_tbl = MetadataTypes.rel_tbl () in
+  let name_tbl =  MetadataTypes.name_tbl () in
+  let count_tbl = MetadataTypes.count_tbl () in
+  let tbls = [ 
+    (obj_tbl,`RefObj) ; (sort_tbl,`RefSort) ; (rel_tbl,`RefRel) ;
+    (name_tbl,`ObjectName) ; (count_tbl,`Count) ] 
+  in
+  let statements = 
+    (SqlStatements.create_tables tbls) @ (SqlStatements.create_indexes tbls)
+  in
+  List.iter (fun statement -> 
+    try
+      ignore (Mysql.exec dbd statement)
+    with
+      exn -> 
+         let status = Mysql.status dbd in
+         match status with 
+         | Mysql.StatusError Mysql.Table_exists_error -> ()
+         | Mysql.StatusError _ -> raise exn
+         | _ -> ()
+      ) statements
+
+let drop_all () =
+  let obj_tbl = MetadataTypes.obj_tbl () in
+  let sort_tbl = MetadataTypes.sort_tbl () in
+  let rel_tbl = MetadataTypes.rel_tbl () in
+  let name_tbl =  MetadataTypes.name_tbl () in
+  let count_tbl = MetadataTypes.count_tbl () in
+  let tbls = [ 
+    (obj_tbl,`RefObj) ; (sort_tbl,`RefSort) ; (rel_tbl,`RefRel) ;
+    (name_tbl,`ObjectName) ; (count_tbl,`Count) ] 
+  in
+  let statements = 
+    (SqlStatements.drop_tables tbls) @ (SqlStatements.drop_indexes tbls)
+  in
+  List.iter (fun statement -> 
+    try
+      ignore (Mysql.exec dbd statement)
+    with Mysql.Error _ as exn ->
+      match Mysql.errno dbd with 
+      | Mysql.Bad_table_error 
+      | Mysql.No_such_index | Mysql.No_such_table -> () 
+      | _ -> raise exn
+    ) statements
+  
+
+let main () =
+  drop_all ();
+  create_all ();
+  let uris = Http_getter.getalluris () in
+  let uris = List.filter (fun u -> Str.string_match uri_RE u 0) uris in
+  let len = float_of_int (List.length uris) in
+  let i = ref 0 in
+  List.iter (fun u ->
+    incr i;
+    let perc = ((float_of_int !i)  /. len *. 100.0) in
+    let l = String.length u in
+    let short = 
+      if l < 60 then 
+        u ^ String.make (63 - l) ' ' 
+      else 
+        "..." ^  String.sub u (l - 60) 60
+    in
+    Printf.printf "\rIndexing (%3.1f%%): %s" perc short;
+    flush stdout;
+    let uri = UriManager.uri_of_string u in
+    MetadataDb.index_obj ~dbd ~uri)
+  uris
+;;
+
+main ()