X-Git-Url: http://git.archive.openwrt.org/?p=project%2Fuclient.git;a=blobdiff_plain;f=uclient-fetch.c;h=065742eeed3378167b234cb664efb19750af26e0;hp=ab29ceac2803e03dbf78cde41ee1904384596470;hb=f0754619b9ee216b0cb5fc7484f861e876781984;hpb=b9bb340542facf12789047182d2f05a11499dcdd diff --git a/uclient-fetch.c b/uclient-fetch.c index ab29cea..065742e 100644 --- a/uclient-fetch.c +++ b/uclient-fetch.c @@ -17,15 +17,21 @@ */ #define _GNU_SOURCE +#include +#include #include #include #include #include #include #include +#include +#include +#include #include +#include "progress.h" #include "uclient.h" #include "uclient-utils.h" @@ -41,65 +47,193 @@ static struct ustream_ssl_ctx *ssl_ctx; static const struct ustream_ssl_ops *ssl_ops; static int quiet = false; static bool verify = true; +static bool proxy = true; static bool default_certs = false; +static bool no_output; static const char *output_file; static int output_fd = -1; static int error_ret; -static int out_bytes; -static char *username; -static char *password; +static off_t out_offset; +static off_t out_bytes; +static off_t out_len; static char *auth_str; static char **urls; static int n_urls; +static int timeout; +static bool resume, cur_resume; +static struct progress pmt; +static struct uloop_timeout pmt_timer; + +static int init_request(struct uclient *cl); static void request_done(struct uclient *cl); -static int open_output_file(const char *path, bool create) +static void pmt_update(struct uloop_timeout *t) +{ + progress_update(&pmt, out_offset, out_bytes, out_len); + uloop_timeout_set(t, 1000); +} + +static const char * +get_proxy_url(char *url) +{ + char prefix[16]; + char *sep; + + if (!proxy) + return NULL; + + sep = strchr(url, ':'); + if (!sep) + return NULL; + + if (sep - url > 5) + return NULL; + + memcpy(prefix, url, sep - url); + strcpy(prefix + (sep - url), "_proxy"); + return getenv(prefix); +} + +static int open_output_file(const char *path, uint64_t resume_offset) { - char *filename; - int flags = O_WRONLY; + char *filename = NULL; + int flags; int ret; - if (create) - flags |= O_CREAT | O_EXCL; + if (cur_resume) + flags = O_RDWR; + else + flags = O_WRONLY | O_TRUNC; + + if (!cur_resume && !output_file) + flags |= O_EXCL; + + flags |= O_CREAT; if (output_file) { - if (!strcmp(output_file, "-")) - return STDOUT_FILENO; + if (!strcmp(output_file, "-")) { + if (!quiet) + fprintf(stderr, "Writing to stdout\n"); + + ret = STDOUT_FILENO; + goto done; + } + } else { + filename = uclient_get_url_filename(path, "index.html"); + output_file = filename; + } + + if (!quiet) + fprintf(stderr, "Writing to '%s'\n", output_file); + ret = open(output_file, flags, 0644); + if (ret < 0) + goto free; + if (resume_offset && + lseek(ret, resume_offset, SEEK_SET) < 0) { if (!quiet) - fprintf(stderr, "Writing to stdout\n"); + fprintf(stderr, "Failed to seek %"PRIu64" bytes in output file\n", resume_offset); + close(ret); + ret = -1; + goto free; + } - unlink(output_file); - return open(output_file, flags, 0644); + out_offset = resume_offset; + out_bytes += resume_offset; +done: + if (!quiet) { + progress_init(&pmt, output_file); + pmt_timer.cb = pmt_update; + pmt_timer.cb(&pmt_timer); } - filename = uclient_get_url_filename(path, "index.html"); - if (!quiet) - fprintf(stderr, "Writing to '%s'\n", filename); - ret = open(filename, flags, 0644); +free: free(filename); - return ret; } static void header_done_cb(struct uclient *cl) { + enum { + H_RANGE, + H_LEN, + __H_MAX + }; + static const struct blobmsg_policy policy[__H_MAX] = { + [H_RANGE] = { .name = "content-range", .type = BLOBMSG_TYPE_STRING }, + [H_LEN] = { .name = "content-length", .type = BLOBMSG_TYPE_STRING }, + }; + struct blob_attr *tb[__H_MAX]; + uint64_t resume_offset = 0, resume_end, resume_size; static int retries; - if (retries < 10 && uclient_http_redirect(cl)) { - if (!quiet) - fprintf(stderr, "Redirected to %s on %s\n", cl->url->location, cl->url->host); + if (retries < 10) { + int ret = uclient_http_redirect(cl); + if (ret < 0) { + if (!quiet) + fprintf(stderr, "Failed to redirect to %s on %s\n", cl->url->location, cl->url->host); + error_ret = 8; + request_done(cl); + return; + } + if (ret > 0) { + if (!quiet) + fprintf(stderr, "Redirected to %s on %s\n", cl->url->location, cl->url->host); - retries++; + retries++; + return; + } + } + + if (cl->status_code == 204 && cur_resume) { + /* Resume attempt failed, try normal download */ + cur_resume = false; + init_request(cl); return; } - retries = 0; + blobmsg_parse(policy, __H_MAX, tb, blob_data(cl->meta), blob_len(cl->meta)); + switch (cl->status_code) { + case 416: + if (!quiet) + fprintf(stderr, "File download already fully retrieved; nothing to do.\n"); + request_done(cl); + break; + case 206: + if (!cur_resume) { + if (!quiet) + fprintf(stderr, "Error: Partial content received, full content requested\n"); + error_ret = 8; + request_done(cl); + break; + } + + if (!tb[H_RANGE]) { + if (!quiet) + fprintf(stderr, "Content-Range header is missing\n"); + error_ret = 8; + break; + } + + if (sscanf(blobmsg_get_string(tb[H_RANGE]), + "bytes %"PRIu64"-%"PRIu64"/%"PRIu64, + &resume_offset, &resume_end, &resume_size) != 3) { + if (!quiet) + fprintf(stderr, "Content-Range header is invalid\n"); + error_ret = 8; + break; + } case 204: case 200: - output_fd = open_output_file(cl->url->location, true); + if (no_output) + break; + + if (tb[H_LEN]) + out_len = strtoul(blobmsg_get_string(tb[H_LEN]), NULL, 10); + + output_fd = open_output_file(cl->url->location, resume_offset); if (output_fd < 0) { if (!quiet) perror("Cannot open output file"); @@ -122,7 +256,7 @@ static void read_data_cb(struct uclient *cl) char buf[256]; int len; - if (output_fd < 0) + if (!no_output && output_fd < 0) return; while (1) { @@ -131,7 +265,8 @@ static void read_data_cb(struct uclient *cl) return; out_bytes += len; - write(output_fd, buf, len); + if (!no_output) + write(output_fd, buf, len); } } @@ -147,13 +282,41 @@ static void msg_connecting(struct uclient *cl) fprintf(stderr, "Connecting to %s:%d\n", addr, port); } +static void check_resume_offset(struct uclient *cl) +{ + char range_str[64]; + struct stat st; + char *file; + int ret; + + file = uclient_get_url_filename(cl->url->location, "index.html"); + if (!file) + return; + + ret = stat(file, &st); + free(file); + if (ret) + return; + + if (!st.st_size) + return; + + snprintf(range_str, sizeof(range_str), "bytes=%"PRIu64"-", (uint64_t) st.st_size); + uclient_http_set_header(cl, "Range", range_str); +} + static int init_request(struct uclient *cl) { int rc; + out_offset = 0; out_bytes = 0; + out_len = 0; uclient_http_set_ssl_ctx(cl, ssl_ops, ssl_ctx, verify); + if (timeout) + cl->timeout_msecs = timeout * 1000; + rc = uclient_connect(cl); if (rc) return rc; @@ -166,6 +329,8 @@ static int init_request(struct uclient *cl) uclient_http_reset_headers(cl); uclient_http_set_header(cl, "User-Agent", user_agent); + if (cur_resume) + check_resume_offset(cl); if (post_data) { uclient_http_set_header(cl, "Content-Type", "application/x-www-form-urlencoded"); @@ -181,9 +346,18 @@ static int init_request(struct uclient *cl) static void request_done(struct uclient *cl) { + const char *proxy_url; + if (n_urls) { - uclient_set_url(cl, *urls, auth_str); + proxy_url = get_proxy_url(*urls); + if (proxy_url) { + uclient_set_url(cl, proxy_url, NULL); + uclient_set_proxy_url(cl, *urls, auth_str); + } else { + uclient_set_url(cl, *urls, auth_str); + } n_urls--; + cur_resume = resume; error_ret = init_request(cl); if (error_ret == 0) return; @@ -200,12 +374,18 @@ static void request_done(struct uclient *cl) static void eof_cb(struct uclient *cl) { + if (!quiet) { + pmt_update(&pmt_timer); + uloop_timeout_cancel(&pmt_timer); + fprintf(stderr, "\n"); + } + if (!cl->data_eof) { if (!quiet) fprintf(stderr, "Connection reset prematurely\n"); error_ret = 4; } else if (!quiet) { - fprintf(stderr, "Download completed (%d bytes)\n", out_bytes); + fprintf(stderr, "Download completed (%"PRIu64" bytes)\n", (uint64_t) out_bytes); } request_done(cl); } @@ -260,16 +440,22 @@ static int usage(const char *progname) fprintf(stderr, "Usage: %s [options] \n" "Options:\n" - " -q: Turn off status messages\n" - " -O : Redirect output to file (use \"-\" for stdout)\n" + " -4 Use IPv4 only\n" + " -6 Use IPv6 only\n" + " -q Turn off status messages\n" + " -O Redirect output to file (use \"-\" for stdout)\n" + " -P Set directory for output files\n" " --user= HTTP authentication username\n" " --password= HTTP authentication password\n" " --user-agent|-U Set HTTP user agent\n" " --post-data=STRING use the POST method; send STRING as the data\n" + " --spider|-s Spider mode - only check file existence\n" + " --timeout=N|-T N Set connect/request timeout to N seconds\n" + " --proxy=on|off|-Y on|off Enable/disable env var configured proxy\n" "\n" "HTTPS options:\n" - " --ca-certificate=: Load CA certificates from file \n" - " --no-check-certificate: don't validate the server's certificate\n" + " --ca-certificate= Load CA certificates from file \n" + " --no-check-certificate don't validate the server's certificate\n" "\n", progname); return 1; } @@ -312,6 +498,11 @@ enum { L_PASSWORD, L_USER_AGENT, L_POST_DATA, + L_SPIDER, + L_TIMEOUT, + L_CONTINUE, + L_PROXY, + L_NO_PROXY, }; static const struct option longopts[] = { @@ -321,6 +512,11 @@ static const struct option longopts[] = { [L_PASSWORD] = { "password", required_argument }, [L_USER_AGENT] = { "user-agent", required_argument }, [L_POST_DATA] = { "post-data", required_argument }, + [L_SPIDER] = { "spider", no_argument }, + [L_TIMEOUT] = { "timeout", required_argument }, + [L_CONTINUE] = { "continue", no_argument }, + [L_PROXY] = { "proxy", required_argument }, + [L_NO_PROXY] = { "no-proxy", no_argument }, {} }; @@ -329,15 +525,20 @@ static const struct option longopts[] = { int main(int argc, char **argv) { const char *progname = argv[0]; + const char *proxy_url; + char *username = NULL; + char *password = NULL; struct uclient *cl; int longopt_idx = 0; bool has_cert = false; int i, ch; int rc; + int af = -1; + signal(SIGPIPE, SIG_IGN); init_ustream_ssl(); - while ((ch = getopt_long(argc, argv, "qO:U:", longopts, &longopt_idx)) != -1) { + while ((ch = getopt_long(argc, argv, "46cO:P:qsT:U:Y:", longopts, &longopt_idx)) != -1) { switch(ch) { case 0: switch (longopt_idx) { @@ -367,19 +568,61 @@ int main(int argc, char **argv) case L_POST_DATA: post_data = optarg; break; + case L_SPIDER: + no_output = true; + break; + case L_TIMEOUT: + timeout = atoi(optarg); + break; + case L_CONTINUE: + resume = true; + break; + case L_PROXY: + if (strcmp(optarg, "on") != 0) + proxy = false; + break; + case L_NO_PROXY: + proxy = false; + break; default: return usage(progname); } break; + case '4': + af = AF_INET; + break; + case '6': + af = AF_INET6; + break; + case 'c': + resume = true; + break; case 'U': user_agent = optarg; break; case 'O': output_file = optarg; break; + case 'P': + if (chdir(optarg)) { + if (!quiet) + perror("Change output directory"); + exit(1); + } + break; case 'q': quiet = true; break; + case 's': + no_output = true; + break; + case 'T': + timeout = atoi(optarg); + break; + case 'Y': + if (strcmp(optarg, "on") != 0) + proxy = false; + break; default: return usage(progname); } @@ -416,15 +659,25 @@ int main(int argc, char **argv) if (!quiet) fprintf(stderr, "Downloading '%s'\n", argv[0]); - cl = uclient_new(argv[0], auth_str, &cb); + proxy_url = get_proxy_url(argv[0]); + if (proxy_url) { + cl = uclient_new(proxy_url, auth_str, &cb); + if (cl) + uclient_set_proxy_url(cl, argv[0], NULL); + } else { + cl = uclient_new(argv[0], auth_str, &cb); + } if (!cl) { fprintf(stderr, "Failed to allocate uclient context\n"); return 1; } + if (af >= 0) + uclient_http_set_address_family(cl, af); if (ssl_ctx && default_certs) init_ca_cert(); + cur_resume = resume; rc = init_request(cl); if (!rc) { /* no error received, we can enter main loop */