From 53a481cfc7546504db6005b6f29fe30c3d9eabc7 Mon Sep 17 00:00:00 2001 From: Enrico Tassi Date: Tue, 3 May 2005 17:12:49 +0000 Subject: [PATCH] added new extractor based on MetadataDb.index_obj --- helm/ocaml/metadata/Makefile | 21 +++-- helm/ocaml/metadata/extractor/Makefile | 21 +++++ .../metadata/extractor/extractor.conf.xml | 19 ++++ helm/ocaml/metadata/extractor/extractor.ml | 92 +++++++++++++++++++ 4 files changed, 143 insertions(+), 10 deletions(-) create mode 100644 helm/ocaml/metadata/extractor/Makefile create mode 100644 helm/ocaml/metadata/extractor/extractor.conf.xml create mode 100644 helm/ocaml/metadata/extractor/extractor.ml diff --git a/helm/ocaml/metadata/Makefile b/helm/ocaml/metadata/Makefile index 52fe27edd..afacda168 100644 --- a/helm/ocaml/metadata/Makefile +++ b/helm/ocaml/metadata/Makefile @@ -15,23 +15,24 @@ EXTRA_OBJECTS_TO_CLEAN = include ../Makefile.common -all: all_table_creator -opt: opt_table_creator +all: all_table_creator all_extractor +opt: opt_table_creator opt_extractor all_table_creator: make -C table_creator/ all opt_table_creator: make -C table_creator/ opt -clean: clean_table_creator +all_extractor: + make -C extractor/ all +opt_extractor: + make -C extractor/ opt + +clean: clean_table_creator clean_extractor clean_table_creator: make -C table_creator/ clean -test: test.ml $(PACKAGE).cma - $(OCAMLFIND) ocamlc -thread -package mysql,helm-metadata -linkpkg -o $@ $< -test.opt: test.ml $(PACKAGE).cmxa - $(OCAMLFIND) opt -thread -package mysql,helm-metadata -linkpkg -o $@ $< -test_query: test_query.ml $(PACKAGE).cma - $(OCAMLFIND) ocamlc -thread -package mysql,helm-metadata -linkpkg -o $@ $< - +clean_extractor: + make -C extractor/ clean + diff --git a/helm/ocaml/metadata/extractor/Makefile b/helm/ocaml/metadata/extractor/Makefile new file mode 100644 index 000000000..1275aaab8 --- /dev/null +++ b/helm/ocaml/metadata/extractor/Makefile @@ -0,0 +1,21 @@ +REQUIRES = mysql helm-metadata + +INTERFACE_FILES = +IMPLEMENTATION_FILES = $(INTERFACE_FILES:%.mli=%.ml) +EXTRA_OBJECTS_TO_INSTALL = +EXTRA_OBJECTS_TO_CLEAN = \ + extractor extractor.opt + +all: extractor +opt: extractor.opt + +extractor: extractor.ml + $(OCAMLFIND) ocamlc \ + -thread -package mysql,helm-metadata -linkpkg -o $@ $< + +extractor.opt: extractor.ml + $(OCAMLFIND) ocamlopt \ + -thread -package mysql,helm-metadata -linkpkg -o $@ $< + +include ../../Makefile.common +include .depend diff --git a/helm/ocaml/metadata/extractor/extractor.conf.xml b/helm/ocaml/metadata/extractor/extractor.conf.xml new file mode 100644 index 000000000..8dbc9a935 --- /dev/null +++ b/helm/ocaml/metadata/extractor/extractor.conf.xml @@ -0,0 +1,19 @@ + + +
+ .tmp/ +
+
+ localhost + helm + mowgli +
+
+ + file:///projects/helm/library/coq_contribs + + $(tmp.dir)/cache + $(tmp.dir)/maps + /projects/helm/xml/dtd +
+ diff --git a/helm/ocaml/metadata/extractor/extractor.ml b/helm/ocaml/metadata/extractor/extractor.ml new file mode 100644 index 000000000..f3c39d382 --- /dev/null +++ b/helm/ocaml/metadata/extractor/extractor.ml @@ -0,0 +1,92 @@ +let _ = Helm_registry.load_from "extractor.conf.xml" +let _ = Unix.system ("mkdir -p " ^ (Helm_registry.get "tmp.dir")) +let _ = Http_getter.init () +(*let _ = Http_getter.update () *) + +let dbd = + Mysql.quick_connect + ~host:(Helm_registry.get "db.host") + ~user:(Helm_registry.get "db.user") + ~database:(Helm_registry.get "db.database") () + +let _ = + try + MetadataTypes.ownerize_tables Sys.argv.(1) + with Invalid_argument _ -> MetadataTypes.ownerize_tables "NEW" + +let uri_RE = Str.regexp ".*\\(ind\\|var\\|con\\)$" + +let create_all () = + let obj_tbl = MetadataTypes.obj_tbl () in + let sort_tbl = MetadataTypes.sort_tbl () in + let rel_tbl = MetadataTypes.rel_tbl () in + let name_tbl = MetadataTypes.name_tbl () in + let count_tbl = MetadataTypes.count_tbl () in + let tbls = [ + (obj_tbl,`RefObj) ; (sort_tbl,`RefSort) ; (rel_tbl,`RefRel) ; + (name_tbl,`ObjectName) ; (count_tbl,`Count) ] + in + let statements = + (SqlStatements.create_tables tbls) @ (SqlStatements.create_indexes tbls) + in + List.iter (fun statement -> + try + ignore (Mysql.exec dbd statement) + with + exn -> + let status = Mysql.status dbd in + match status with + | Mysql.StatusError Mysql.Table_exists_error -> () + | Mysql.StatusError _ -> raise exn + | _ -> () + ) statements + +let drop_all () = + let obj_tbl = MetadataTypes.obj_tbl () in + let sort_tbl = MetadataTypes.sort_tbl () in + let rel_tbl = MetadataTypes.rel_tbl () in + let name_tbl = MetadataTypes.name_tbl () in + let count_tbl = MetadataTypes.count_tbl () in + let tbls = [ + (obj_tbl,`RefObj) ; (sort_tbl,`RefSort) ; (rel_tbl,`RefRel) ; + (name_tbl,`ObjectName) ; (count_tbl,`Count) ] + in + let statements = + (SqlStatements.drop_tables tbls) @ (SqlStatements.drop_indexes tbls) + in + List.iter (fun statement -> + try + ignore (Mysql.exec dbd statement) + with Mysql.Error _ as exn -> + match Mysql.errno dbd with + | Mysql.Bad_table_error + | Mysql.No_such_index | Mysql.No_such_table -> () + | _ -> raise exn + ) statements + + +let main () = + drop_all (); + create_all (); + let uris = Http_getter.getalluris () in + let uris = List.filter (fun u -> Str.string_match uri_RE u 0) uris in + let len = float_of_int (List.length uris) in + let i = ref 0 in + List.iter (fun u -> + incr i; + let perc = ((float_of_int !i) /. len *. 100.0) in + let l = String.length u in + let short = + if l < 60 then + u ^ String.make (63 - l) ' ' + else + "..." ^ String.sub u (l - 60) 60 + in + Printf.printf "\rIndexing (%3.1f%%): %s" perc short; + flush stdout; + let uri = UriManager.uri_of_string u in + MetadataDb.index_obj ~dbd ~uri) + uris +;; + +main () -- 2.39.2