URL API: add request->connect_timeout & request->transfer_timeout

... in case you want to do fine-tuning.

Defaults to DOWNLOAD_CONNECT_TIMEOUT (15 seconds) and
DOWNLOAD_TRANSFER_TIMEOUT (20 seconds).

For example, the module manager uses a shorter timeout of 7 and 20.
(that was already the case, but now it uses the generic api so
 it needed an option to set it to those values)

include/h.h

@@ -1475,4 +1475,4 @@ extern int valid_operclass_name(const char *str);
 extern void free_outgoingwebrequest(OutgoingWebRequest *r);
 extern OutgoingWebRequest *duplicate_outgoingwebrequest(OutgoingWebRequest *orig);
 extern void url_callback(OutgoingWebRequest *r, const char *file, const char *memory, int memory_len, const char *errorbuf, int cached, void *ptr);
-extern const char *synchronous_http_request(const char *url, int max_redirects, int connect_timeout, int read_timeout);
+extern const char *synchronous_http_request(const char *url, int max_redirects, int connect_timeout, int transfer_timeout);

include/struct.h

@@ -1898,6 +1898,8 @@ struct OutgoingWebRequest
 	time_t cachetime;
 	int max_redirects;
 	int keep_file;
+	int connect_timeout; /**< How many seconds to wait for the (TLS) connect to succeed */
+	int transfer_timeout; /**< How many seconds the total transfer may take (connect+reading everything) */
 	// If you are adding allocated fields here:
 	// 1) update duplicate_outgoingwebrequest() in src/misc.c
 	// 2) and update url_free_handle_request_portion() there as well

src/misc.c

@@ -3200,6 +3200,8 @@ OutgoingWebRequest *duplicate_outgoingwebrequest(OutgoingWebRequest *orig)
 	e->cachetime = orig->cachetime;
 	e->max_redirects = orig->max_redirects;
 	e->keep_file = orig->keep_file;
+	e->connect_timeout = orig->connect_timeout;
+	e->transfer_timeout = orig->transfer_timeout;
 	return e;
 }
 
@@ -3283,7 +3285,7 @@ void synchronous_http_request_handle_response(OutgoingWebRequest *request, Outgo
 	return;
 }
 
-const char *synchronous_http_request(const char *url, int max_redirects, int connect_timeout, int read_timeout)
+const char *synchronous_http_request(const char *url, int max_redirects, int connect_timeout, int transfer_timeout)
 {
 	OutgoingWebRequest *request;
 
@@ -3297,11 +3299,11 @@ const char *synchronous_http_request(const char *url, int max_redirects, int con
 	request->max_redirects = max_redirects;
 	request->store_in_file = 1;
 	request->keep_file = 1;
+	request->connect_timeout = connect_timeout;
+	request->transfer_timeout = transfer_timeout;
 	url_start_async(request);
 	synchronous_http_request_in_progress = 1;
 
-	// FIXME: actually use 'connect_timeout' and 'read_timeout' ;)
-
 	while (synchronous_http_request_in_progress == 1)
 	{
 		gettimeofday(&timeofday_tv, NULL);

src/modulemanager.c

@@ -8,7 +8,7 @@
 #ifndef _WIN32
 
 #define MODULEMANAGER_CONNECT_TIMEOUT	7
-#define MODULEMANAGER_READ_TIMEOUT	20
+#define MODULEMANAGER_TRANSFER_TIMEOUT	20
 
 typedef struct ManagedModule ManagedModule;
 
@@ -618,7 +618,7 @@ int mm_refresh_repository(void)
 		}
 		printf("Checking module repository %s...\n", line);
 		numrepos++;
-		tmpfile = synchronous_http_request(line, 1, MODULEMANAGER_CONNECT_TIMEOUT, MODULEMANAGER_READ_TIMEOUT);
+		tmpfile = synchronous_http_request(line, 1, MODULEMANAGER_CONNECT_TIMEOUT, MODULEMANAGER_TRANSFER_TIMEOUT);
 		if (tmpfile)
 		{
 			if (!mm_parse_repo_db(line, tmpfile))
@@ -960,7 +960,7 @@ void mm_install_module(ManagedModule *m)
 	const char *tmpfile;
 
 	printf("Downloading %s from %s...\n", m->name, m->source);
-	tmpfile = synchronous_http_request(m->source, 1, MODULEMANAGER_CONNECT_TIMEOUT, MODULEMANAGER_READ_TIMEOUT);
+	tmpfile = synchronous_http_request(m->source, 1, MODULEMANAGER_CONNECT_TIMEOUT, MODULEMANAGER_TRANSFER_TIMEOUT);
 	if (!tmpfile)
 	{
 		fprintf(stderr, "Repository %s seems to list a module file that cannot be retrieved (%s).\n", m->repo_url, m->source);

src/url_curl.c

@@ -394,8 +394,8 @@ void url_start_async(OutgoingWebRequest *request)
 		curl_easy_setopt(curl, CURLOPT_TIMEVALUE, handle->request->cachetime);
 	}
 	curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
-	curl_easy_setopt(curl, CURLOPT_TIMEOUT, DOWNLOAD_TRANSFER_TIMEOUT);
-	curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, DOWNLOAD_CONNECT_TIMEOUT);
+	curl_easy_setopt(curl, CURLOPT_TIMEOUT, request->transfer_timeout ? request->transfer_timeout : DOWNLOAD_TRANSFER_TIMEOUT);
+	curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, request->connect_timeout ? request->connect_timeout : DOWNLOAD_CONNECT_TIMEOUT);
 #if LIBCURL_VERSION_NUM >= 0x070f01
 	curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
 	curl_easy_setopt(curl, CURLOPT_MAXREDIRS, handle->request->max_redirects);

src/url_unreal.c

@@ -158,6 +158,12 @@ void url_start_async(OutgoingWebRequest *request)
 	if (!request->url || !request->http_method)
 		abort();
 
+	/* Set request defaults */
+	if (request->connect_timeout == 0)
+		request->connect_timeout = DOWNLOAD_CONNECT_TIMEOUT;
+	if (request->transfer_timeout == 0)
+		request->transfer_timeout = DOWNLOAD_TRANSFER_TIMEOUT;
+
 	handle = safe_alloc(sizeof(Download));
 	handle->download_started = TStime();
 	handle->request = request;
@@ -1158,14 +1164,14 @@ EVENT(url_socket_timeout)
 		d_next = d->next;
 		if (d->dns_refcnt)
 			continue; /* can't touch this... */
-		if (!d->connected && (TStime() - d->download_started > DOWNLOAD_CONNECT_TIMEOUT))
+		if (!d->connected && (TStime() - d->download_started > d->request->connect_timeout))
 		{
-			https_cancel(d, "Connect or DNS timeout after %ld seconds", (long)DOWNLOAD_CONNECT_TIMEOUT);
+			https_cancel(d, "Connect or DNS timeout after %d seconds", d->request->connect_timeout);
 			continue;
 		}
-		if (d->connected && (TStime() - d->download_started > DOWNLOAD_TRANSFER_TIMEOUT))
+		if (d->connected && (TStime() - d->download_started > d->request->transfer_timeout))
 		{
-			https_cancel(d, "Download timeout after %ld seconds", (long)DOWNLOAD_TRANSFER_TIMEOUT);
+			https_cancel(d, "Download timeout after %d seconds", d->request->transfer_timeout);
 			continue;
 		}
 	}