From 7d024f80863d7bfb84a0ab526e24e4f367af09f4 Mon Sep 17 00:00:00 2001 From: Bram Matthys Date: Fri, 24 Nov 2023 14:34:54 +0100 Subject: [PATCH] URL API: add request->connect_timeout & request->transfer_timeout ... in case you want to do fine-tuning. Defaults to DOWNLOAD_CONNECT_TIMEOUT (15 seconds) and DOWNLOAD_TRANSFER_TIMEOUT (20 seconds). For example, the module manager uses a shorter timeout of 7 and 20. (that was already the case, but now it uses the generic api so it needed an option to set it to those values) --- include/h.h | 2 +- include/struct.h | 2 ++ src/misc.c | 8 +++++--- src/modulemanager.c | 6 +++--- src/url_curl.c | 4 ++-- src/url_unreal.c | 14 ++++++++++---- 6 files changed, 23 insertions(+), 13 deletions(-) diff --git a/include/h.h b/include/h.h index 13e576b43..9d3af1894 100644 --- a/include/h.h +++ b/include/h.h @@ -1475,4 +1475,4 @@ extern int valid_operclass_name(const char *str); extern void free_outgoingwebrequest(OutgoingWebRequest *r); extern OutgoingWebRequest *duplicate_outgoingwebrequest(OutgoingWebRequest *orig); extern void url_callback(OutgoingWebRequest *r, const char *file, const char *memory, int memory_len, const char *errorbuf, int cached, void *ptr); -extern const char *synchronous_http_request(const char *url, int max_redirects, int connect_timeout, int read_timeout); +extern const char *synchronous_http_request(const char *url, int max_redirects, int connect_timeout, int transfer_timeout); diff --git a/include/struct.h b/include/struct.h index f955fb15d..d136a0a93 100644 --- a/include/struct.h +++ b/include/struct.h @@ -1898,6 +1898,8 @@ struct OutgoingWebRequest time_t cachetime; int max_redirects; int keep_file; + int connect_timeout; /**< How many seconds to wait for the (TLS) connect to succeed */ + int transfer_timeout; /**< How many seconds the total transfer may take (connect+reading everything) */ // If you are adding allocated fields here: // 1) update duplicate_outgoingwebrequest() in src/misc.c // 2) and update url_free_handle_request_portion() there as well diff --git a/src/misc.c b/src/misc.c index aa5fcdb2e..1f750890e 100644 --- a/src/misc.c +++ b/src/misc.c @@ -3200,6 +3200,8 @@ OutgoingWebRequest *duplicate_outgoingwebrequest(OutgoingWebRequest *orig) e->cachetime = orig->cachetime; e->max_redirects = orig->max_redirects; e->keep_file = orig->keep_file; + e->connect_timeout = orig->connect_timeout; + e->transfer_timeout = orig->transfer_timeout; return e; } @@ -3283,7 +3285,7 @@ void synchronous_http_request_handle_response(OutgoingWebRequest *request, Outgo return; } -const char *synchronous_http_request(const char *url, int max_redirects, int connect_timeout, int read_timeout) +const char *synchronous_http_request(const char *url, int max_redirects, int connect_timeout, int transfer_timeout) { OutgoingWebRequest *request; @@ -3297,11 +3299,11 @@ const char *synchronous_http_request(const char *url, int max_redirects, int con request->max_redirects = max_redirects; request->store_in_file = 1; request->keep_file = 1; + request->connect_timeout = connect_timeout; + request->transfer_timeout = transfer_timeout; url_start_async(request); synchronous_http_request_in_progress = 1; - // FIXME: actually use 'connect_timeout' and 'read_timeout' ;) - while (synchronous_http_request_in_progress == 1) { gettimeofday(&timeofday_tv, NULL); diff --git a/src/modulemanager.c b/src/modulemanager.c index e93d86d66..0648e940f 100644 --- a/src/modulemanager.c +++ b/src/modulemanager.c @@ -8,7 +8,7 @@ #ifndef _WIN32 #define MODULEMANAGER_CONNECT_TIMEOUT 7 -#define MODULEMANAGER_READ_TIMEOUT 20 +#define MODULEMANAGER_TRANSFER_TIMEOUT 20 typedef struct ManagedModule ManagedModule; @@ -618,7 +618,7 @@ int mm_refresh_repository(void) } printf("Checking module repository %s...\n", line); numrepos++; - tmpfile = synchronous_http_request(line, 1, MODULEMANAGER_CONNECT_TIMEOUT, MODULEMANAGER_READ_TIMEOUT); + tmpfile = synchronous_http_request(line, 1, MODULEMANAGER_CONNECT_TIMEOUT, MODULEMANAGER_TRANSFER_TIMEOUT); if (tmpfile) { if (!mm_parse_repo_db(line, tmpfile)) @@ -960,7 +960,7 @@ void mm_install_module(ManagedModule *m) const char *tmpfile; printf("Downloading %s from %s...\n", m->name, m->source); - tmpfile = synchronous_http_request(m->source, 1, MODULEMANAGER_CONNECT_TIMEOUT, MODULEMANAGER_READ_TIMEOUT); + tmpfile = synchronous_http_request(m->source, 1, MODULEMANAGER_CONNECT_TIMEOUT, MODULEMANAGER_TRANSFER_TIMEOUT); if (!tmpfile) { fprintf(stderr, "Repository %s seems to list a module file that cannot be retrieved (%s).\n", m->repo_url, m->source); diff --git a/src/url_curl.c b/src/url_curl.c index e87392f48..d3dbffbe3 100644 --- a/src/url_curl.c +++ b/src/url_curl.c @@ -394,8 +394,8 @@ void url_start_async(OutgoingWebRequest *request) curl_easy_setopt(curl, CURLOPT_TIMEVALUE, handle->request->cachetime); } curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, DOWNLOAD_TRANSFER_TIMEOUT); - curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, DOWNLOAD_CONNECT_TIMEOUT); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, request->transfer_timeout ? request->transfer_timeout : DOWNLOAD_TRANSFER_TIMEOUT); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, request->connect_timeout ? request->connect_timeout : DOWNLOAD_CONNECT_TIMEOUT); #if LIBCURL_VERSION_NUM >= 0x070f01 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curl, CURLOPT_MAXREDIRS, handle->request->max_redirects); diff --git a/src/url_unreal.c b/src/url_unreal.c index 5bb633074..73875031e 100644 --- a/src/url_unreal.c +++ b/src/url_unreal.c @@ -158,6 +158,12 @@ void url_start_async(OutgoingWebRequest *request) if (!request->url || !request->http_method) abort(); + /* Set request defaults */ + if (request->connect_timeout == 0) + request->connect_timeout = DOWNLOAD_CONNECT_TIMEOUT; + if (request->transfer_timeout == 0) + request->transfer_timeout = DOWNLOAD_TRANSFER_TIMEOUT; + handle = safe_alloc(sizeof(Download)); handle->download_started = TStime(); handle->request = request; @@ -1158,14 +1164,14 @@ EVENT(url_socket_timeout) d_next = d->next; if (d->dns_refcnt) continue; /* can't touch this... */ - if (!d->connected && (TStime() - d->download_started > DOWNLOAD_CONNECT_TIMEOUT)) + if (!d->connected && (TStime() - d->download_started > d->request->connect_timeout)) { - https_cancel(d, "Connect or DNS timeout after %ld seconds", (long)DOWNLOAD_CONNECT_TIMEOUT); + https_cancel(d, "Connect or DNS timeout after %d seconds", d->request->connect_timeout); continue; } - if (d->connected && (TStime() - d->download_started > DOWNLOAD_TRANSFER_TIMEOUT)) + if (d->connected && (TStime() - d->download_started > d->request->transfer_timeout)) { - https_cancel(d, "Download timeout after %ld seconds", (long)DOWNLOAD_TRANSFER_TIMEOUT); + https_cancel(d, "Download timeout after %d seconds", d->request->transfer_timeout); continue; } }