New URL API (not really a unrealircd module api tho) - work in progress.

No longer url_start_async(a,b,c,d,e,f,g,...) but usings structs so
simply url_start_async(tehstruct);
makes it easy to add fields later without forcing all modules to
change the prototype.

Work in progress....

include/h.h

@@ -1397,8 +1397,8 @@ extern int has_cached_version(const char *url);
 extern int url_is_valid(const char *);
 extern const char *displayurl(const char *url);
 extern char *url_getfilename(const char *url);
-extern void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects);
-extern void url_start_async(const char *url, HttpMethod http_method, const char *body, NameValuePrioList *request_headers, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects);
+extern void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, int maxredirects);
+extern void url_start_async(OutgoingWebRequest *request);
 extern void url_init(void);
 extern void url_cancel_handle_by_callback_data(void *ptr);
 extern EVENT(url_socket_timeout);
@@ -1471,3 +1471,6 @@ extern int highest_channel_member_count(Client *client);
 extern MODVAR long long central_spamfilter_last_download;
 extern int valid_operclass_character(char c);
 extern int valid_operclass_name(const char *str);
+#define safe_free_outgoingwebrequest(x)	do { if (x) { free_outgoingwebrequest(x); x = NULL; } } while(0)
+extern void free_outgoingwebrequest(OutgoingWebRequest *r);
+extern OutgoingWebRequest *duplicate_outgoingwebrequest(OutgoingWebRequest *orig);

include/struct.h

@@ -1881,7 +1881,27 @@ struct HTTPForwardedHeader
 	char ip[IPLEN+1];
 };
 
+typedef struct OutgoingWebRequest OutgoingWebRequest;
+/** An outgoing web request (eg remote includes download) */
+struct OutgoingWebRequest
+{
+	vFP callback;
+	void *callback_data;
+	char *url; /**< must be freed by url_do_transfers_async() */
+	char *actual_url; /**< if you actually want to use a different url, mostly for redirects (end-users: don't set this!) */
+	HttpMethod http_method;
+	char *body;
+	NameValuePrioList *headers;
+	int store_in_file;
+	time_t cachetime;
+	int max_redirects;
+	// If you are adding allocated fields here:
+	// 1) update duplicate_outgoingwebrequest() in src/misc.c
+	// 2) and update url_free_handle_request_portion() there as well
+};
+
 typedef struct WebRequest WebRequest;
+/** An incoming web request */
 struct WebRequest {
 	HttpMethod method; /**< GET/PUT/POST */
 	char *uri; /**< Requested resource, eg "/api" */

src/conf.c

@@ -11407,7 +11407,7 @@ int add_config_resource(const char *resource, int type, ConfigEntry *ce)
 				prev = cep;
 			}
 		}
-		download_file_async(rs->url, modtime, resource_download_complete, (void *)rs, NULL, DOWNLOAD_MAX_REDIRECTS);
+		download_file_async(rs->url, modtime, resource_download_complete, (void *)rs, DOWNLOAD_MAX_REDIRECTS);
 	}
 	return 1;
 }
@@ -11947,7 +11947,7 @@ void central_spamfilter_start_download(void)
 	safe_free_nvplist(nvp);
 
 	/* Start HTTPS request */
-	download_file_async(url, CENTRAL_SPAMFILTER_CACHE_TIME, central_spamfilter_download_complete, NULL, NULL, DOWNLOAD_MAX_REDIRECTS);
+	download_file_async(url, CENTRAL_SPAMFILTER_CACHE_TIME, central_spamfilter_download_complete, NULL, DOWNLOAD_MAX_REDIRECTS);
 }
 
 EVENT(central_spamfilter_download_evt)

src/misc.c

@@ -3174,3 +3174,66 @@ int valid_operclass_name(const char *str)
 
 	return 1;
 }
+
+/** Free an OutgoingWebRequest struct - note: use safe_free_outgoingwebrequest() instead (which calls us). */
+void free_outgoingwebrequest(OutgoingWebRequest *r)
+{
+	safe_free(r->url);
+	safe_free(r->actual_url);
+        safe_free(r->body);
+        safe_free_nvplist(r->headers);
+        safe_free(r);
+}
+
+/** Safely duplicate an OutgoingWebRequest struct (eg for https redirects) */
+OutgoingWebRequest *duplicate_outgoingwebrequest(OutgoingWebRequest *orig)
+{
+	OutgoingWebRequest *e = safe_alloc(sizeof(OutgoingWebRequest));
+	e->callback = orig->callback;
+	e->callback_data = orig->callback_data;
+	safe_strdup(e->url, orig->url);
+	safe_strdup(e->actual_url, orig->actual_url);
+	e->http_method = orig->http_method;
+	safe_strdup(e->body, orig->body);
+	e->headers = duplicate_nvplist(orig->headers);
+	e->store_in_file = orig->store_in_file;
+	e->cachetime = orig->cachetime;
+	e->max_redirects = orig->max_redirects;
+	return e;
+}
+
+/*
+ * Handles asynchronous downloading of a file (simple non-flexible version).
+ * NOTE: url_start_async() is the more advanced one.
+ *
+ * This function allows a download to be made transparently without
+ * the caller having any knowledge of how libcurl works. The specified
+ * callback function is called when the download completes, or the
+ * download fails. The callback function is defined as:
+ *
+ * void callback(const char *url, const char *filename, const char *memory_data, int memory_data_len, char *errorbuf, int cached, void *data);
+ *  - url will contain the original URL used to download the file.
+ *  - filename will contain the name of the file (if successful, NULL on error or if cached).
+ *    This file will be cleaned up after the callback returns, so save a copy to support caching.
+ *  - errorbuf will contain the error message (if failed, NULL otherwise).
+ *  - cached 1 if the specified cachetime is >= the current file on the server,
+ *    if so, errorbuf will be NULL, filename will contain the path to the file.
+ *  - data will be the value of callback_data, allowing you to figure
+ *    out how to use the data contained in the downloaded file ;-).
+ *    Make sure that if you access the contents of this pointer, you
+ *    know that this pointer will persist. A download could take more
+ *    than 10 seconds to happen and the config file can be rehashed
+ *    multiple times during that time.
+ */
+void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, int maxredirects)
+{
+	OutgoingWebRequest *request = safe_alloc(sizeof(OutgoingWebRequest));
+	safe_strdup(request->url, url);
+	request->http_method = HTTP_METHOD_GET;
+	request->cachetime = cachetime;
+	request->callback = callback;
+	request->callback_data = callback_data;
+	request->max_redirects = maxredirects;
+	request->store_in_file = 1;
+	url_start_async(request);
+}

src/modules/central-blocklist.c

@@ -980,6 +980,7 @@ void cbl_mdata_free(ModData *m)
 void send_request_for_pending_clients(void)
 {
 	Client *client, *next;
+	OutgoingWebRequest *w;
 	json_t *j, *requests;
 	NameValuePrioList *headers = NULL;
 	int num;
@@ -1030,9 +1031,16 @@ void send_request_for_pending_clients(void)
 	add_nvplist(&headers, 0, "Content-Type", "application/json; charset=utf-8");
 	add_nvplist(&headers, 0, "X-API-Key", cfg.api_key);
 	c = add_cbl_transfer(clientlist);
-	url_start_async(cfg.url, HTTP_METHOD_POST, json_serialized, headers, 0, 0, cbl_download_complete, c, cfg.url, 1);
-	safe_free(json_serialized);
-	safe_free_nvplist(headers);
+	/* Do the web request */
+	w = safe_alloc(sizeof(OutgoingWebRequest));
+	safe_strdup(w->url, cfg.url);
+	w->http_method = HTTP_METHOD_POST;
+	w->body = json_serialized;
+	w->headers = headers;
+	w->max_redirects = 1;
+	w->callback = cbl_download_complete;
+	w->callback_data = c;
+	url_start_async(w);
 }
 
 int cbl_any_pending_clients(void)
@@ -1082,6 +1090,7 @@ CMD_OVERRIDE_FUNC(cbl_override_spamreport_gather)
 void cbl_spamreport(Client *from, Client *client)
 {
 	json_t *j, *requests, *data, *cmds, *item;
+	OutgoingWebRequest *w;
 	NameValuePrioList *headers = NULL;
 	int num;
 	char *json_serialized;
@@ -1145,9 +1154,15 @@ void cbl_spamreport(Client *from, Client *client)
 	json_decref(j);
 	add_nvplist(&headers, 0, "Content-Type", "application/json; charset=utf-8");
 	add_nvplist(&headers, 0, "X-API-Key", cfg.api_key);
-	url_start_async(cfg.spamreport_url, HTTP_METHOD_POST, json_serialized, headers, 0, 0, download_complete_dontcare, NULL, cfg.spamreport_url, 1);
-	safe_free(json_serialized);
-	safe_free_nvplist(headers);
+	/* Do the web request */
+	w = safe_alloc(sizeof(OutgoingWebRequest));
+	safe_strdup(w->url, cfg.spamreport_url);
+	w->http_method = HTTP_METHOD_POST;
+	w->body = json_serialized;
+	w->headers = headers;
+	w->max_redirects = 1;
+	w->callback = download_complete_dontcare;
+	url_start_async(w);
 }
 
 CMD_OVERRIDE_FUNC(cbl_override_spamreport_cmd)

src/modules/spamreport.c

@@ -393,6 +393,7 @@ int spamfilter_block_rate_limited(Spamreport *spamreport)
 int _spamreport(Client *client, const char *ip, NameValuePrioList *details, const char *spamreport_block)
 {
 	Spamreport *s;
+	OutgoingWebRequest *request;
 	char urlbuf[512];
 	char bodybuf[512];
 	char *url = NULL;
@@ -472,8 +473,15 @@ int _spamreport(Client *client, const char *ip, NameValuePrioList *details, cons
 	           log_data_string("url", url),
 	           log_data_string("body", (body ? body : "")));
 #endif
-	url_start_async(url, s->http_method, body, headers, 0, 0, download_complete_dontcare, NULL, url, 3);
-	safe_free_nvplist(headers);
+	/* Do the web request */
+	request = safe_alloc(sizeof(OutgoingWebRequest));
+	safe_strdup(request->url, url);
+	request->http_method = s->http_method;
+	safe_strdup(request->body, body);
+	request->headers = headers;
+	request->callback = download_complete_dontcare;
+	request->max_redirects = 3;
+	url_start_async(request);
 	return 1;
 }
 

src/url_curl.c

@@ -33,17 +33,11 @@ typedef struct Download Download;
 struct Download
 {
 	Download *prev, *next;
-	vFP callback;
-	void *callback_data;
-	FILE *file_fd;		/**< File open for writing (otherwise NULL) */
-	char *filename;
-	char *url; /*< must be free()d by url_do_transfers_async() */
-	HttpMethod http_method;
-	char *body;
-	NameValuePrioList *request_headers;
+	OutgoingWebRequest *request;
 	struct curl_slist *request_headers_curl;
 	char errorbuf[CURL_ERROR_SIZE];
-	time_t cachetime;
+	FILE *file_fd;		/**< File open for writing (otherwise NULL) */
+	char *filename;
 	char *memory_data; /**< Memory for writing response (otherwise NULL) */
 	int memory_data_len; /**< Size of memory_data */
 	int memory_data_allocated; /**< Total allocated memory for 'memory_data' */
@@ -60,11 +54,9 @@ void url_free_handle(Download *handle)
 		fclose(handle->file_fd);
 	safe_free(handle->memory_data);
 	safe_free(handle->filename);
-	safe_free(handle->url);
-	safe_free(handle->body);
-	safe_free_nvplist(handle->request_headers);
 	if (handle->request_headers_curl)
 		curl_slist_free_all(handle->request_headers_curl);
+	safe_free_outgoingwebrequest(handle->request);
 	safe_free(handle);
 }
 
@@ -75,10 +67,10 @@ void url_cancel_handle_by_callback_data(void *ptr)
 	for (d = downloads; d; d = d_next)
 	{
 		d_next = d->next;
-		if (d->callback_data == ptr)
+		if (d->request->callback_data == ptr)
 		{
-			d->callback = NULL;
-			d->callback_data = NULL;
+			d->request->callback = NULL;
+			d->request->callback_data = NULL;
 		}
 	}
 }
@@ -182,7 +174,7 @@ static void url_check_multi_handles(void)
 				handle->file_fd = NULL;
 			}
 
-			if (handle->callback == NULL)
+			if (handle->request->callback == NULL)
 			{
 				/* Request is already canceled, we don't care about the result, just clean up */
 				if (handle->filename)
@@ -190,9 +182,9 @@ static void url_check_multi_handles(void)
 			} else
 			if (msg->data.result == CURLE_OK)
 			{
-				if (code == 304 || (last_mod != -1 && last_mod <= handle->cachetime))
+				if (code == 304 || (last_mod != -1 && last_mod <= handle->request->cachetime))
 				{
-					handle->callback(handle->url, NULL, handle->memory_data, handle->memory_data_len, NULL, 1, handle->callback_data);
+					handle->request->callback(handle->request->url, NULL, handle->memory_data, handle->memory_data_len, NULL, 1, handle->request->callback_data);
 					if (handle->filename)
 						remove(handle->filename);
 				}
@@ -201,14 +193,14 @@ static void url_check_multi_handles(void)
 					if ((last_mod != -1) && handle->filename)
 						unreal_setfilemodtime(handle->filename, last_mod);
 
-					handle->callback(handle->url, handle->filename, handle->memory_data, handle->memory_data_len, NULL, 0, handle->callback_data);
+					handle->request->callback(handle->request->url, handle->filename, handle->memory_data, handle->memory_data_len, NULL, 0, handle->request->callback_data);
 					if (handle->filename)
 						remove(handle->filename);
 				}
 			}
 			else
 			{
-				handle->callback(handle->url, NULL, NULL, 0, handle->errorbuf, 0, handle->callback_data);
+				handle->request->callback(handle->request->url, NULL, NULL, 0, handle->errorbuf, 0, handle->request->callback_data);
 				if (handle->filename)
 					remove(handle->filename);
 			}
@@ -294,33 +286,7 @@ void url_init(void)
 	url_socket_timeout_hdl = EventAdd(NULL, "url_socket_timeout", url_socket_timeout, NULL, 500, 0);
 }
 
-/*
- * Handles asynchronous downloading of a file. This function allows
- * a download to be made transparently without the caller having any
- * knowledge of how libcurl works. The specified callback function is
- * called when the download completes, or the download fails. The 
- * callback function is defined as:
- *
- * void callback(const char *url, const char *filename, const char *memory_data, int memory_data_len, char *errorbuf, int cached, void *data);
- *  - url will contain the original URL used to download the file.
- *  - filename will contain the name of the file (if successful, NULL on error or if cached).
- *    This file will be cleaned up after the callback returns, so save a copy to support caching.
- *  - errorbuf will contain the error message (if failed, NULL otherwise).
- *  - cached 1 if the specified cachetime is >= the current file on the server,
- *    if so, errorbuf will be NULL, filename will contain the path to the file.
- *  - data will be the value of callback_data, allowing you to figure
- *    out how to use the data contained in the downloaded file ;-).
- *    Make sure that if you access the contents of this pointer, you
- *    know that this pointer will persist. A download could take more
- *    than 10 seconds to happen and the config file can be rehashed
- *    multiple times during that time.
- */
-void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects)
-{
-	url_start_async(url, HTTP_METHOD_GET, NULL, NULL, 1, cachetime, callback, callback_data, original_url, maxredirects);
-}
-
-void url_start_async(const char *url, HttpMethod http_method, const char *body, NameValuePrioList *request_headers, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects)
+void url_start_async(OutgoingWebRequest *request)
 {
 	static char errorbuf[CURL_ERROR_SIZE];
 	char user_agent[256];
@@ -330,6 +296,10 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body,
 	char *tmp;
 	Download *handle;
 
+	/* Check for the bare minimum */
+	if (!request->url || !request->http_method)
+		abort();
+
 	curl = curl_easy_init();
 	if (!curl)
 	{
@@ -340,17 +310,18 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body,
 	}
 
 	handle = safe_alloc(sizeof(Download));
+	handle->request = request;
 
-	if (store_in_file)
+	if (handle->request->store_in_file)
 	{
-		file = url_getfilename(url);
+		file = url_getfilename(handle->request->url);
 		filename = unreal_getfilename(file);
 		tmp = unreal_mktemp(TMPDIR, filename ? filename : "download.conf");
 		handle->file_fd = fopen(tmp, "wb");
 		if (!handle->file_fd)
 		{
 			snprintf(errorbuf, sizeof(errorbuf), "Cannot create '%s': %s", tmp, strerror(ERRNO));
-			callback(url, NULL, NULL, 0, errorbuf, 0, callback_data);
+			handle->request->callback(handle->request->url, NULL, NULL, 0, errorbuf, 0, handle->request->callback_data);
 			safe_free(file);
 			safe_free(handle);
 			return;
@@ -361,15 +332,10 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body,
 
 	AddListItem(handle, downloads);
 
-	handle->callback = callback;
-	handle->callback_data = callback_data;
-	handle->cachetime = cachetime;
-	safe_strdup(handle->url, url);
-
-	curl_easy_setopt(curl, CURLOPT_URL, url);
+	curl_easy_setopt(curl, CURLOPT_URL, handle->request->url);
 	snprintf(user_agent, sizeof(user_agent), "UnrealIRCd %s", VERSIONONLY);
 	curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
-	if (store_in_file)
+	if (handle->request->store_in_file)
 	{
 		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, do_download_file);
 		curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)handle->file_fd);
@@ -379,26 +345,22 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body,
 		handle->memory_data_allocated = URL_MEMORY_BACKED_CHUNK_SIZE;
 		handle->memory_data = safe_alloc(URL_MEMORY_BACKED_CHUNK_SIZE+1);
 	}
-	if (http_method == HTTP_METHOD_POST)
+	if (handle->request->http_method == HTTP_METHOD_POST)
 	{
 		curl_easy_setopt(curl, CURLOPT_POST, 1);
 #if LIBCURL_VERSION_NUM >= 0x071301
 		curl_easy_setopt(curl, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
 #endif
-		if (body && strlen(body))
-		{
-			safe_strdup(handle->body, body); // actually redundant?
-			curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, body);
-		}
+		if (handle->request->body && strlen(handle->request->body))
+			curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, handle->request->body);
 	}
 
-	if (request_headers)
+	if (handle->request->headers)
 	{
 		NameValuePrioList *n;
 		char buf[512];
 
-		handle->request_headers = duplicate_nvplist(request_headers);
-		for (n = request_headers; n; n = n->next)
+		for (n = handle->request->headers; n; n = n->next)
 		{
 			if (n->value)
 				snprintf(buf, sizeof(buf), "%s: %s", n->name, n->value);
@@ -425,17 +387,17 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body,
 	 * As a side-effect we also fix useless CLOSE_WAIT connections.
 	 */
 	curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1);
-	if (cachetime)
+	if (handle->request->cachetime)
 	{
 		curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
-		curl_easy_setopt(curl, CURLOPT_TIMEVALUE, cachetime);
+		curl_easy_setopt(curl, CURLOPT_TIMEVALUE, handle->request->cachetime);
 	}
 	curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
 	curl_easy_setopt(curl, CURLOPT_TIMEOUT, DOWNLOAD_TRANSFER_TIMEOUT);
 	curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, DOWNLOAD_CONNECT_TIMEOUT);
 #if LIBCURL_VERSION_NUM >= 0x070f01
 	curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
-	curl_easy_setopt(curl, CURLOPT_MAXREDIRS, maxredirects);
+	curl_easy_setopt(curl, CURLOPT_MAXREDIRS, handle->request->max_redirects);
 #endif
 
 	curl_multi_add_handle(multihandle, curl);

src/url_unreal.c

@@ -31,20 +31,13 @@ typedef struct Download Download;
 struct Download
 {
 	Download *prev, *next;
-	vFP callback;
-	void *callback_data;
-	char *url; /**< must be free()d by url_do_transfers_async() */
-	HttpMethod http_method;
-	char *body;
-	NameValuePrioList *request_headers;
-	int store_in_file;
+	OutgoingWebRequest *request;
 	FILE *file_fd;		/**< File open for writing (otherwise NULL) */
 	char *filename;
 	char *memory_data; /**< Memory for writing response (otherwise NULL) */
 	int memory_data_len; /**< Size of memory_data */
 	int memory_data_allocated; /**< Total allocated memory for 'memory_data' */
 	char errorbuf[512];
-	time_t cachetime;
 	char *hostname;		/**< Parsed hostname (from 'url') */
 	int port;		/**< Parsed port (from 'url') */
 	char *username;
@@ -64,10 +57,7 @@ struct Download
 	int dns_refcnt;
 	TransferEncoding transfer_encoding;
 	long chunk_remaining;
-	/* for redirects: */
-	int redirects_remaining;
 	char *redirect_new_location;
-	char *redirect_original_url;
 };
 
 /* Variables */
@@ -105,8 +95,6 @@ void url_free_handle(Download *handle)
 		fclose(handle->file_fd);
 	safe_free(handle->filename);
 	safe_free(handle->memory_data);
-	safe_free(handle->body);
-	safe_free_nvplist(handle->request_headers);
 	safe_free(handle->hostname);
 	safe_free(handle->username);
 	safe_free(handle->password);
@@ -116,8 +104,7 @@ void url_free_handle(Download *handle)
 		SSL_free(handle->ssl);
 	safe_free(handle->lefttoparse);
 	safe_free(handle->redirect_new_location);
-	safe_free(handle->redirect_original_url);
-	safe_free(handle->url);
+	safe_free_outgoingwebrequest(handle->request);
 	safe_free(handle);
 }
 
@@ -128,15 +115,15 @@ void url_cancel_handle_by_callback_data(void *ptr)
 	for (d = downloads; d; d = d_next)
 	{
 		d_next = d->next;
-		if (d->callback_data == ptr)
+		if (d->request->callback_data == ptr)
 		{
-			d->callback = NULL;
-			d->callback_data = NULL;
+			d->request->callback = NULL;
+			d->request->callback_data = NULL;
 		}
 	}
 }
 
-/** Cancel and free the HTTPS request.
+/** Cancel and free the HTTPS request->
  * @returns Always returns -1
  */
 int https_cancel(Download *handle, FORMAT_STRING(const char *pattern), ...)
@@ -145,49 +132,41 @@ int https_cancel(Download *handle, FORMAT_STRING(const char *pattern), ...)
 	va_start(vl, pattern);
 	vsnprintf(handle->errorbuf, sizeof(handle->errorbuf), pattern, vl);
 	va_end(vl);
-	if (handle->callback)
-		handle->callback(handle->url, NULL, NULL, 0, handle->errorbuf, 0, handle->callback_data);
+	if (handle->request->callback)
+		handle->request->callback(handle->request->url, NULL, NULL, 0, handle->errorbuf, 0, handle->request->callback_data);
 	url_free_handle(handle);
 	return -1;
 }
 
-void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects)
-{
-	url_start_async(url, HTTP_METHOD_GET, NULL, NULL, 1, cachetime, callback, callback_data, original_url, maxredirects);
-}
-
-void url_start_async(const char *url, HttpMethod http_method, const char *body, NameValuePrioList *request_headers, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects)
+void url_start_async(OutgoingWebRequest *request)
 {
 	char *file;
 	const char *filename;
 	char *tmp;
 	Download *handle = NULL;
 	int ipv6 = 0;
+	const char *actual_url = request->actual_url ? request->actual_url : request->url;
 	char *host;
 	int port;
 	char *username;
 	char *password;
 	char *document;
 
+	/* Check for the bare minimum */
+	if (!request->url || !request->http_method)
+		abort();
+
 	handle = safe_alloc(sizeof(Download));
 	handle->download_started = TStime();
-	handle->callback = callback;
-	handle->callback_data = callback_data;
-	handle->cachetime = cachetime;
-	safe_strdup(handle->url, url);
-	safe_strdup(handle->redirect_original_url, original_url);
-	handle->redirects_remaining = maxredirects;
-	handle->http_method = http_method;
-	safe_strdup(handle->body, body);
-	handle->store_in_file = store_in_file;
+	handle->request = request;
 	AddListItem(handle, downloads);
 
-	if (strncmp(url, "https://", 8))
+	if (strncmp(actual_url, "https://", 8))
 	{
 		https_cancel(handle, "Only https:// is supported (either rebuild UnrealIRCd with curl support or use https)");
 		return;
 	}
-	if (!url_parse(url, &host, &port, &username, &password, &document))
+	if (!url_parse(actual_url, &host, &port, &username, &password, &document))
 	{
 		https_cancel(handle, "Failed to parse HTTP url");
 		return;
@@ -199,9 +178,9 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body,
 	safe_strdup(handle->password, password);
 	safe_strdup(handle->document, document);
 
-	if (store_in_file)
+	if (request->store_in_file)
 	{
-		file = url_getfilename(url);
+		file = url_getfilename(handle->request->url);
 		filename = unreal_getfilename(file);
 		tmp = unreal_mktemp(TMPDIR, filename ? filename : "download.conf");
 
@@ -220,13 +199,6 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body,
 		handle->memory_data = safe_alloc(URL_MEMORY_BACKED_CHUNK_SIZE);
 	}
 
-	if (request_headers)
-		handle->request_headers = duplicate_nvplist(request_headers);
-
-	// todo: allocate handle, select en weetikt allemaal
-	// add to some global struct linkedlist, for timeouts
-	// register in i/o
-
 	if (is_valid_ip(handle->hostname))
 	{
 		/* Nothing to resolve, eg https://127.0.0.1/ */
@@ -524,7 +496,7 @@ int https_connect_send_header(Download *handle)
 	snprintf(hostandport, sizeof(hostandport), "%s:%d", handle->hostname, handle->port);
 
 	/* Prepare the header */
-	if (handle->http_method == HTTP_METHOD_GET)
+	if (handle->request->http_method == HTTP_METHOD_GET)
 	{
 		snprintf(buf, sizeof(buf), "GET %s HTTP/1.1\r\n"
 				    "User-Agent: UnrealIRCd %s\r\n"
@@ -534,9 +506,9 @@ int https_connect_send_header(Download *handle)
 				    VERSIONONLY,
 				    hostandport);
 	} else
-	if (handle->http_method == HTTP_METHOD_POST)
+	if (handle->request->http_method == HTTP_METHOD_POST)
 	{
-		if (!handle->body || !strlen(handle->body))
+		if (!handle->request->body || !strlen(handle->request->body))
 		{
 			snprintf(buf, sizeof(buf), "POST %s HTTP/1.1\r\n"
 					    "User-Agent: UnrealIRCd %s\r\n"
@@ -547,7 +519,7 @@ int https_connect_send_header(Download *handle)
 					    hostandport);
 		} else {
 			char add_default_content_type = 0;
-			if (!find_nvplist(handle->request_headers, "Content-Type"))
+			if (!find_nvplist(handle->request->headers, "Content-Type"))
 				add_default_content_type = 1;
 
 			snprintf(buf, sizeof(buf), "POST %s HTTP/1.1\r\n"
@@ -560,7 +532,7 @@ int https_connect_send_header(Download *handle)
 					    VERSIONONLY,
 					    hostandport,
 					    add_default_content_type ? "Content-Type: application/x-www-form-urlencoded\r\n" : "",
-					    (long)strlen(handle->body));
+					    (long)strlen(handle->request->body));
 		}
 	} else
 		abort();
@@ -577,9 +549,9 @@ int https_connect_send_header(Download *handle)
 			strlcat(buf, header, sizeof(buf));
 		}
 	}
-	if (handle->cachetime > 0)
+	if (handle->request->cachetime > 0)
 	{
-		const char *datestr = rfc2616_time(handle->cachetime);
+		const char *datestr = rfc2616_time(handle->request->cachetime);
 		if (datestr)
 		{
 			// snprintf_append...
@@ -587,12 +559,12 @@ int https_connect_send_header(Download *handle)
 				 "If-Modified-Since: %s\r\n", datestr);
 		}
 	}
-	if (handle->request_headers)
+	if (handle->request->headers)
 	{
 		NameValuePrioList *n;
 		char nbuf[256];
 
-		for (n = handle->request_headers; n; n = n->next)
+		for (n = handle->request->headers; n; n = n->next)
 		{
 			if (n->value)
 				snprintf(nbuf, sizeof(nbuf), "%s: %s\r\n", n->name, n->value);
@@ -604,8 +576,8 @@ int https_connect_send_header(Download *handle)
 		}
 	}
 	strlcat(buf, "\r\n", sizeof(buf));
-	if (handle->body)
-		strlcat(buf, handle->body, sizeof(buf));
+	if (handle->request->body)
+		strlcat(buf, handle->request->body, sizeof(buf));
 
 	ssl_err = SSL_write(handle->ssl, buf, strlen(buf));
 	if (ssl_err < 0)
@@ -720,7 +692,7 @@ int https_handle_response_header(Download *handle, char *readbuf, int n)
 			else if ((handle->http_status_code >= 301) && (handle->http_status_code <= 308))
 			{
 				/* Redirect */
-				if (handle->redirects_remaining == 0)
+				if (handle->request->max_redirects == 0)
 				{
 					https_cancel(handle, "Too many HTTP redirects (%d)", DOWNLOAD_MAX_REDIRECTS);
 					return 0;
@@ -838,7 +810,7 @@ int https_handle_response_body(Download *handle, char *readbuf, int pktsize)
 	if (handle->transfer_encoding == TRANSFER_ENCODING_NONE)
 	{
 		/* Ohh.. so easy! */
-		if (handle->store_in_file == 0)
+		if (handle->request->store_in_file == 0)
 			https_handle_response_body_memory(handle, readbuf, pktsize);
 		else if (handle->file_fd)
 			fwrite(readbuf, 1, pktsize, handle->file_fd);
@@ -868,7 +840,7 @@ int https_handle_response_body(Download *handle, char *readbuf, int pktsize)
 		{
 			/* Eat it */
 			int eat = MIN(handle->chunk_remaining, n);
-			if (handle->store_in_file == 0)
+			if (handle->request->store_in_file == 0)
 				https_handle_response_body_memory(handle, buf, eat);
 			else if (handle->file_fd)
 				fwrite(buf, 1, eat, handle->file_fd);
@@ -948,23 +920,21 @@ int https_handle_response_body(Download *handle, char *readbuf, int pktsize)
 
 void https_done(Download *handle)
 {
-	char *url = handle->redirect_original_url ? handle->redirect_original_url : handle->url;
-
 	if (handle->file_fd)
 	{
 		fclose(handle->file_fd);
 		handle->file_fd = NULL;
 	}
 
-	if (!handle->callback)
+	if (!handle->request->callback)
 		; /* No special action, request was cancelled */
 	else if (!handle->got_response)
-		handle->callback(url, NULL, NULL, 0, "HTTPS response not received", 0, handle->callback_data);
+		handle->request->callback(handle->request->url, NULL, NULL, 0, "HTTPS response not received", 0, handle->request->callback_data);
 	else
 	{
 		if ((handle->last_modified > 0) && handle->filename)
 			unreal_setfilemodtime(handle->filename, handle->last_modified);
-		handle->callback(url, handle->filename, handle->memory_data, handle->memory_data_len, NULL, 0, handle->callback_data);
+		handle->request->callback(handle->request->url, handle->filename, handle->memory_data, handle->memory_data_len, NULL, 0, handle->request->callback_data);
 	}
 	url_free_handle(handle);
 	return;
@@ -972,37 +942,38 @@ void https_done(Download *handle)
 
 void https_done_cached(Download *handle)
 {
-	char *url = handle->redirect_original_url ? handle->redirect_original_url : handle->url;
-
 	if (handle->file_fd)
 	{
 		fclose(handle->file_fd);
 		handle->file_fd = NULL;
 	}
-	if (handle->callback)
-		handle->callback(url, NULL, NULL, 0, NULL, 1, handle->callback_data);
+	if (handle->request->callback)
+		handle->request->callback(handle->request->url, NULL, NULL, 0, NULL, 1, handle->request->callback_data);
 	url_free_handle(handle);
 }
 
 void https_redirect(Download *handle)
 {
-	if (handle->redirects_remaining == 0)
+	if (handle->request->max_redirects == 0)
 	{
 		https_cancel(handle, "Too many HTTP redirects (%d)", DOWNLOAD_MAX_REDIRECTS);
 		return;
 	}
-	handle->redirects_remaining--;
 
-	if (handle->callback)
+	/* If still an outstanding request (not cancelled), follow the redirect.. */
+	if (handle->request->callback)
 	{
-		/* If still an outstanding request (not cancelled), follow the redirect.. */
-		url_start_async(handle->redirect_new_location, handle->http_method, handle->body,
-		                handle->request_headers, handle->store_in_file,
-		                handle->cachetime, handle->callback, handle->callback_data,
-				handle->url, handle->redirects_remaining);
+		OutgoingWebRequest *r = duplicate_outgoingwebrequest(handle->request);
+		safe_strdup(r->actual_url, handle->redirect_new_location); // override actual url
+		r->max_redirects--; // safe, checked to be >0 a few lines up
+		url_free_handle(handle); // free old handle
+		url_start_async(r); // create new one
+	} else {
+		/* The callback is NULL, so we can just free without
+		 * following redirects and any error reporting
+		 */
+		url_free_handle(handle); // free old handle
 	}
-	/* Don't call the hook, just free this, the new redirect from above will call the hook later */
-	url_free_handle(handle);
 }
 
 /** Helper function to parse the HTTP header consisting of multiple 'Key: value' pairs */