- debug_print ("wget " ^ String.concat " " flags);
- Shell.call
- ~stdout:Shell.to_dev_null ~stderr:Shell.to_dev_null
- [Shell.cmd "wget" flags]
- in
- if Pcre.pmatch ~rex:file_scheme_RE url then begin (* file:// URL *)
- let src_fname = Pcre.replace ~rex:file_scheme_RE url in
- match output with
- | Some dst_fname -> cp src_fname dst_fname
- | None ->
- let dst_fname = Filename.basename src_fname in
- if src_fname <> dst_fname then
- cp src_fname dst_fname
- else (* src and dst are the same: do nothing *)
- ()
- end else (* other URL, pass it to wget *)
- use_wget ()
-
-let bufsiz = 16384
-let gzip ?(keep = false) fname =
- let (ic, oc) = (open_in fname, Gzip.open_out (fname ^ ".gz")) in
+ (subs.(1),
+ (if subs.(2) = "" then 80 else int_of_string subs.(3)),
+ (if subs.(4) = "" then "/" else subs.(4)))
+ with exc ->
+ failwith
+ (sprintf "Can't parse url: %s (exception: %s)"
+ url (Printexc.to_string exc))
+let init_socket addr port =
+ let inet_addr = (Unix.gethostbyname addr).Unix.h_addr_list.(0) in
+ let sockaddr = Unix.ADDR_INET (inet_addr, port) in
+ let suck = Unix.socket Unix.PF_INET Unix.SOCK_STREAM 0 in
+ Unix.connect suck sockaddr;
+ let outchan = Unix.out_channel_of_descr suck in
+ let inchan = Unix.in_channel_of_descr suck in
+ (inchan, outchan)
+let http_get_iter_buf ~callback url =
+ let (address, port, path) = parse_url url in
+ let buf = String.create tcp_bufsiz in
+ let (inchan, outchan) = init_socket address port in
+ output_string outchan (sprintf "GET %s\r\n" path);
+ flush outchan;
+ (try
+ while true do
+ match input inchan buf 0 tcp_bufsiz with
+ | 0 -> raise End_of_file
+ | bytes when bytes = tcp_bufsiz -> (* buffer full, no need to slice it *)
+ callback buf
+ | bytes when bytes < tcp_bufsiz -> (* buffer not full, slice it *)
+ callback (String.sub buf 0 bytes)
+ | _ -> (* ( bytes < 0 ) || ( bytes > tcp_bufsiz ) *)
+ assert false
+ done
+ with End_of_file -> ());
+ close_in inchan (* close also outchan, same fd *)
+
+let wget ?output url =
+ debug_print
+ (sprintf "wgetting %s (output: %s)" url
+ (match output with None -> "default" | Some f -> f));
+ match url with
+ | url when Pcre.pmatch ~rex:file_scheme_RE url -> (* file:// *)
+ (let src_fname = Pcre.replace ~rex:file_scheme_RE url in
+ match output with
+ | Some dst_fname -> cp src_fname dst_fname
+ | None ->
+ let dst_fname = Filename.basename src_fname in
+ if src_fname <> dst_fname then
+ cp src_fname dst_fname
+ else (* src and dst are the same: do nothing *)
+ ())
+ | url when Pcre.pmatch ~rex:http_scheme_RE url -> (* http:// *)
+ (let oc =
+ open_out (match output with Some f -> f | None -> Filename.basename url)
+ in
+ http_get_iter_buf ~callback:(fun data -> output_string oc data) url;
+ close_out oc)
+ | scheme -> (* unsupported scheme *)
+ failwith ("Http_getter_misc.wget: unsupported scheme: " ^ scheme)
+
+let gzip ?(keep = false) ?output fname =
+ let output = match output with None -> fname ^ ".gz" | Some fname -> fname in
+ debug_print (sprintf "gzipping %s (keep: %b, output: %s)" fname keep output);
+ let (ic, oc) = (open_in fname, Gzip.open_out output) in