1 (* Copyright (C) 2004-2005, HELM Team.
3 * This file is part of HELM, an Hypertextual, Electronic
4 * Library of Mathematics, developed at the Computer Science
5 * Department, University of Bologna, Italy.
7 * HELM is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 2
10 * of the License, or (at your option) any later version.
12 * HELM is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with HELM; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place - Suite 330, Boston,
22 * For details, see the HELM World-Wide-Web page,
23 * http://helm.cs.unibo.it/
31 open Http_getter_types
34 exception Resource_not_found of string * string (** method, uri *)
36 let index_fname = "INDEX"
38 let trailing_slash_RE = Pcre.regexp "/$"
39 let relative_RE_raw = "(^[^/]+(/[^/]+)*/?$)"
40 let relative_RE = Pcre.regexp relative_RE_raw
41 let file_scheme_RE_raw = "(^file://)"
42 let extended_file_scheme_RE = Pcre.regexp "(^file:/+)"
43 let file_scheme_RE = Pcre.regexp (relative_RE_raw ^ "|" ^ file_scheme_RE_raw)
44 let http_scheme_RE = Pcre.regexp "^http://"
45 let newline_RE = Pcre.regexp "\\n"
46 let cic_scheme_sep_RE = Pcre.regexp ":/"
48 let gz_suffix_len = String.length gz_suffix
50 let path_of_file_url url =
51 assert (Pcre.pmatch ~rex:file_scheme_RE url);
52 if Pcre.pmatch ~rex:relative_RE url then
54 else (* absolute path, add heading "/" if missing *)
55 "/" ^ (Pcre.replace ~rex:extended_file_scheme_RE url)
57 (** associative list regular expressions -> url prefixes
58 * sorted with longest prefixes first *)
59 let prefix_map = lazy (
62 (fun (uri_prefix, url_prefix) ->
63 let uri_prefix = normalize_dir uri_prefix in
64 let url_prefix = normalize_dir url_prefix in
65 let regexp = Pcre.regexp ("^(" ^ Pcre.quote uri_prefix ^ ")") in
66 (regexp, String.length uri_prefix, uri_prefix, url_prefix))
67 (Lazy.force Http_getter_env.prefixes)
69 let decreasing_length (_, len1, _, _) (_, len2, _, _) = compare len2 len1 in
71 (fun (regexp, len, uri_prefix, url_prefix) ->
72 (regexp, strip_trailing_slash uri_prefix, url_prefix))
73 (List.fast_sort decreasing_length map_w_length))
75 let resolve_prefix uri =
77 List.filter (fun (rex, _, _) -> Pcre.pmatch ~rex uri)
78 (Lazy.force prefix_map)
81 | (rex, _, url_prefix) :: _ -> Pcre.replace_first ~rex ~templ:url_prefix uri
82 | [] -> raise (Unresolvable_URI uri)
84 let resolve_prefixes uri =
86 List.filter (fun (rex, _, _) -> Pcre.pmatch ~rex uri)
87 (Lazy.force prefix_map)
89 if matches = [] then raise (Unresolvable_URI uri);
91 (fun (rex, _, url_prefix) -> Pcre.replace_first ~rex ~templ:url_prefix uri)
94 let exists_http _ url =
95 Http_getter_wget.exists (url ^ gz_suffix) || Http_getter_wget.exists url
97 let exists_file _ fname =
98 Sys.file_exists (fname ^ gz_suffix) || Sys.file_exists fname
100 let resolve_http _ url =
102 List.find Http_getter_wget.exists [ url ^ gz_suffix; url ]
103 with Not_found -> raise Not_found'
105 let resolve_file _ fname =
107 List.find Sys.file_exists [ fname ^ gz_suffix; fname ]
108 with Not_found -> raise Not_found'
110 let strip_gz_suffix fname =
111 if extension fname = gz_suffix then
112 String.sub fname 0 (String.length fname - gz_suffix_len)
116 let remove_duplicates l =
117 Http_getter_misc.list_uniq (List.fast_sort Pervasives.compare l)
119 let ls_file_single _ path_prefix =
120 let is_dir fname = (Unix.stat fname).Unix.st_kind = Unix.S_DIR in
121 let is_useless dir = try dir.[0] = '.' with _ -> false in
122 let entries = ref [] in
124 let dir_handle = Unix.opendir path_prefix in
127 let entry = Unix.readdir dir_handle in
128 if is_useless entry then
130 else if is_dir (path_prefix ^ "/" ^ entry) then
131 entries := normalize_dir entry :: !entries
133 entries := strip_gz_suffix entry :: !entries
135 with End_of_file -> Unix.closedir dir_handle);
136 remove_duplicates !entries
137 with Unix.Unix_error (_, "opendir", _) -> []
139 let ls_http_single _ url_prefix =
141 let index = Http_getter_wget.get (normalize_dir url_prefix ^ index_fname) in
142 Pcre.split ~rex:newline_RE index
143 with Http_client_error _ -> raise Not_found'
145 let get_file _ path =
146 if Sys.file_exists (path ^ gz_suffix) then
148 else if Sys.file_exists path then
153 let get_http uri url =
155 match Pcre.split ~rex:cic_scheme_sep_RE uri with
156 | [scheme; path] -> scheme, path
160 sprintf "%s%s/%s" (Lazy.force Http_getter_env.cache_dir) scheme path
162 if Sys.file_exists (cache_name ^ gz_suffix) then
163 cache_name ^ gz_suffix
164 else if Sys.file_exists cache_name then
166 else begin (* fill cache *)
167 Http_getter_misc.mkdir ~parents:true (Filename.dirname cache_name);
169 Http_getter_wget.get_and_save (url ^ gz_suffix) (cache_name ^ gz_suffix);
170 cache_name ^ gz_suffix
171 with Http_client_error _ ->
173 Http_getter_wget.get_and_save url cache_name;
175 with Http_client_error _ ->
179 let remove_file _ path =
180 if Sys.file_exists (path ^ gz_suffix) then Sys.remove (path ^ gz_suffix);
181 if Sys.file_exists path then Sys.remove path
183 let remove_http _ _ =
184 prerr_endline "Http_getter_storage.remove: not implemented for HTTP scheme";
187 type 'a storage_method = {
189 file: string -> string -> 'a; (* unresolved uri, resolved uri *)
190 http: string -> string -> 'a; (* unresolved uri, resolved uri *)
193 let normalize_root uri = (* add trailing slash to roots *)
195 if uri.[String.length uri - 1] = ':' then uri ^ "/"
197 with Invalid_argument _ -> uri
199 let invoke_method storage_method uri url =
201 if Pcre.pmatch ~rex:file_scheme_RE url then
202 storage_method.file uri (path_of_file_url url)
203 else if Pcre.pmatch ~rex:http_scheme_RE url then
204 storage_method.http uri url
206 raise (Unsupported_scheme url)
207 with Not_found' -> raise (Resource_not_found (storage_method.name, uri))
209 let dispatch_single storage_method uri =
210 assert (extension uri <> gz_suffix);
211 let uri = normalize_root uri in
212 let url = resolve_prefix uri in
213 invoke_method storage_method uri url
215 let dispatch_multi storage_method uri =
216 let urls = resolve_prefixes uri in
217 let rec aux = function
218 | [] -> raise (Resource_not_found (storage_method.name, uri))
221 invoke_method storage_method uri url
222 with Resource_not_found _ -> aux tl)
227 dispatch_single { name = "exists"; file = exists_file; http = exists_http }
230 dispatch_single { name = "resolve"; file = resolve_file; http = resolve_http }
233 dispatch_single { name = "ls"; file = ls_file_single; http = ls_http_single }
236 dispatch_single { name = "remove"; file = remove_file; http = remove_http }
238 let filename ?(find = false) =
240 dispatch_multi { name = "filename"; file = get_file; http = get_http }
242 dispatch_single { name = "filename"; file = get_file; http = get_http }
244 (* ls_single performs ls only below a single prefix, but prefixes which have
245 * common prefix (sorry) with a given one may need to be considered as well
246 * for example: when doing "ls cic:/" we would like to see the "cic:/matita"
249 (* prerr_endline ("Http_getter_storage.ls " ^ uri_prefix); *)
250 let direct_results = ls_single uri_prefix in
252 (fun results (_, uri_prefix', _) ->
253 if Filename.dirname uri_prefix' = strip_trailing_slash uri_prefix then
254 (Filename.basename uri_prefix' ^ "/") :: results
258 (Lazy.force prefix_map)
262 (sprintf "rm -rf %s/" (Lazy.force Http_getter_env.cache_dir)))