From 3548b7e2afe5175feea04543e66e91d3ea996bdc Mon Sep 17 00:00:00 2001 From: Bram Matthys Date: Fri, 24 Nov 2023 11:27:39 +0100 Subject: [PATCH] New URL API (not really a unrealircd module api tho) - work in progress. No longer url_start_async(a,b,c,d,e,f,g,...) but usings structs so simply url_start_async(tehstruct); makes it easy to add fields later without forcing all modules to change the prototype. Work in progress.... --- include/h.h | 7 +- include/struct.h | 20 +++++ src/conf.c | 4 +- src/misc.c | 63 +++++++++++++++ src/modules/central-blocklist.c | 27 +++++-- src/modules/spamreport.c | 12 ++- src/url_curl.c | 100 ++++++++---------------- src/url_unreal.c | 131 +++++++++++++------------------- 8 files changed, 203 insertions(+), 161 deletions(-) diff --git a/include/h.h b/include/h.h index 22f2ce2b1..dd78a7e9d 100644 --- a/include/h.h +++ b/include/h.h @@ -1397,8 +1397,8 @@ extern int has_cached_version(const char *url); extern int url_is_valid(const char *); extern const char *displayurl(const char *url); extern char *url_getfilename(const char *url); -extern void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects); -extern void url_start_async(const char *url, HttpMethod http_method, const char *body, NameValuePrioList *request_headers, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects); +extern void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, int maxredirects); +extern void url_start_async(OutgoingWebRequest *request); extern void url_init(void); extern void url_cancel_handle_by_callback_data(void *ptr); extern EVENT(url_socket_timeout); @@ -1471,3 +1471,6 @@ extern int highest_channel_member_count(Client *client); extern MODVAR long long central_spamfilter_last_download; extern int valid_operclass_character(char c); extern int valid_operclass_name(const char *str); +#define safe_free_outgoingwebrequest(x) do { if (x) { free_outgoingwebrequest(x); x = NULL; } } while(0) +extern void free_outgoingwebrequest(OutgoingWebRequest *r); +extern OutgoingWebRequest *duplicate_outgoingwebrequest(OutgoingWebRequest *orig); diff --git a/include/struct.h b/include/struct.h index ac1703027..f52accae9 100644 --- a/include/struct.h +++ b/include/struct.h @@ -1881,7 +1881,27 @@ struct HTTPForwardedHeader char ip[IPLEN+1]; }; +typedef struct OutgoingWebRequest OutgoingWebRequest; +/** An outgoing web request (eg remote includes download) */ +struct OutgoingWebRequest +{ + vFP callback; + void *callback_data; + char *url; /**< must be freed by url_do_transfers_async() */ + char *actual_url; /**< if you actually want to use a different url, mostly for redirects (end-users: don't set this!) */ + HttpMethod http_method; + char *body; + NameValuePrioList *headers; + int store_in_file; + time_t cachetime; + int max_redirects; + // If you are adding allocated fields here: + // 1) update duplicate_outgoingwebrequest() in src/misc.c + // 2) and update url_free_handle_request_portion() there as well +}; + typedef struct WebRequest WebRequest; +/** An incoming web request */ struct WebRequest { HttpMethod method; /**< GET/PUT/POST */ char *uri; /**< Requested resource, eg "/api" */ diff --git a/src/conf.c b/src/conf.c index 616421275..590990158 100644 --- a/src/conf.c +++ b/src/conf.c @@ -11407,7 +11407,7 @@ int add_config_resource(const char *resource, int type, ConfigEntry *ce) prev = cep; } } - download_file_async(rs->url, modtime, resource_download_complete, (void *)rs, NULL, DOWNLOAD_MAX_REDIRECTS); + download_file_async(rs->url, modtime, resource_download_complete, (void *)rs, DOWNLOAD_MAX_REDIRECTS); } return 1; } @@ -11947,7 +11947,7 @@ void central_spamfilter_start_download(void) safe_free_nvplist(nvp); /* Start HTTPS request */ - download_file_async(url, CENTRAL_SPAMFILTER_CACHE_TIME, central_spamfilter_download_complete, NULL, NULL, DOWNLOAD_MAX_REDIRECTS); + download_file_async(url, CENTRAL_SPAMFILTER_CACHE_TIME, central_spamfilter_download_complete, NULL, DOWNLOAD_MAX_REDIRECTS); } EVENT(central_spamfilter_download_evt) diff --git a/src/misc.c b/src/misc.c index af68761db..d09e95307 100644 --- a/src/misc.c +++ b/src/misc.c @@ -3174,3 +3174,66 @@ int valid_operclass_name(const char *str) return 1; } + +/** Free an OutgoingWebRequest struct - note: use safe_free_outgoingwebrequest() instead (which calls us). */ +void free_outgoingwebrequest(OutgoingWebRequest *r) +{ + safe_free(r->url); + safe_free(r->actual_url); + safe_free(r->body); + safe_free_nvplist(r->headers); + safe_free(r); +} + +/** Safely duplicate an OutgoingWebRequest struct (eg for https redirects) */ +OutgoingWebRequest *duplicate_outgoingwebrequest(OutgoingWebRequest *orig) +{ + OutgoingWebRequest *e = safe_alloc(sizeof(OutgoingWebRequest)); + e->callback = orig->callback; + e->callback_data = orig->callback_data; + safe_strdup(e->url, orig->url); + safe_strdup(e->actual_url, orig->actual_url); + e->http_method = orig->http_method; + safe_strdup(e->body, orig->body); + e->headers = duplicate_nvplist(orig->headers); + e->store_in_file = orig->store_in_file; + e->cachetime = orig->cachetime; + e->max_redirects = orig->max_redirects; + return e; +} + +/* + * Handles asynchronous downloading of a file (simple non-flexible version). + * NOTE: url_start_async() is the more advanced one. + * + * This function allows a download to be made transparently without + * the caller having any knowledge of how libcurl works. The specified + * callback function is called when the download completes, or the + * download fails. The callback function is defined as: + * + * void callback(const char *url, const char *filename, const char *memory_data, int memory_data_len, char *errorbuf, int cached, void *data); + * - url will contain the original URL used to download the file. + * - filename will contain the name of the file (if successful, NULL on error or if cached). + * This file will be cleaned up after the callback returns, so save a copy to support caching. + * - errorbuf will contain the error message (if failed, NULL otherwise). + * - cached 1 if the specified cachetime is >= the current file on the server, + * if so, errorbuf will be NULL, filename will contain the path to the file. + * - data will be the value of callback_data, allowing you to figure + * out how to use the data contained in the downloaded file ;-). + * Make sure that if you access the contents of this pointer, you + * know that this pointer will persist. A download could take more + * than 10 seconds to happen and the config file can be rehashed + * multiple times during that time. + */ +void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, int maxredirects) +{ + OutgoingWebRequest *request = safe_alloc(sizeof(OutgoingWebRequest)); + safe_strdup(request->url, url); + request->http_method = HTTP_METHOD_GET; + request->cachetime = cachetime; + request->callback = callback; + request->callback_data = callback_data; + request->max_redirects = maxredirects; + request->store_in_file = 1; + url_start_async(request); +} diff --git a/src/modules/central-blocklist.c b/src/modules/central-blocklist.c index f2db7126e..09abf19bb 100644 --- a/src/modules/central-blocklist.c +++ b/src/modules/central-blocklist.c @@ -980,6 +980,7 @@ void cbl_mdata_free(ModData *m) void send_request_for_pending_clients(void) { Client *client, *next; + OutgoingWebRequest *w; json_t *j, *requests; NameValuePrioList *headers = NULL; int num; @@ -1030,9 +1031,16 @@ void send_request_for_pending_clients(void) add_nvplist(&headers, 0, "Content-Type", "application/json; charset=utf-8"); add_nvplist(&headers, 0, "X-API-Key", cfg.api_key); c = add_cbl_transfer(clientlist); - url_start_async(cfg.url, HTTP_METHOD_POST, json_serialized, headers, 0, 0, cbl_download_complete, c, cfg.url, 1); - safe_free(json_serialized); - safe_free_nvplist(headers); + /* Do the web request */ + w = safe_alloc(sizeof(OutgoingWebRequest)); + safe_strdup(w->url, cfg.url); + w->http_method = HTTP_METHOD_POST; + w->body = json_serialized; + w->headers = headers; + w->max_redirects = 1; + w->callback = cbl_download_complete; + w->callback_data = c; + url_start_async(w); } int cbl_any_pending_clients(void) @@ -1082,6 +1090,7 @@ CMD_OVERRIDE_FUNC(cbl_override_spamreport_gather) void cbl_spamreport(Client *from, Client *client) { json_t *j, *requests, *data, *cmds, *item; + OutgoingWebRequest *w; NameValuePrioList *headers = NULL; int num; char *json_serialized; @@ -1145,9 +1154,15 @@ void cbl_spamreport(Client *from, Client *client) json_decref(j); add_nvplist(&headers, 0, "Content-Type", "application/json; charset=utf-8"); add_nvplist(&headers, 0, "X-API-Key", cfg.api_key); - url_start_async(cfg.spamreport_url, HTTP_METHOD_POST, json_serialized, headers, 0, 0, download_complete_dontcare, NULL, cfg.spamreport_url, 1); - safe_free(json_serialized); - safe_free_nvplist(headers); + /* Do the web request */ + w = safe_alloc(sizeof(OutgoingWebRequest)); + safe_strdup(w->url, cfg.spamreport_url); + w->http_method = HTTP_METHOD_POST; + w->body = json_serialized; + w->headers = headers; + w->max_redirects = 1; + w->callback = download_complete_dontcare; + url_start_async(w); } CMD_OVERRIDE_FUNC(cbl_override_spamreport_cmd) diff --git a/src/modules/spamreport.c b/src/modules/spamreport.c index b8f1744ca..d58d2faaa 100644 --- a/src/modules/spamreport.c +++ b/src/modules/spamreport.c @@ -393,6 +393,7 @@ int spamfilter_block_rate_limited(Spamreport *spamreport) int _spamreport(Client *client, const char *ip, NameValuePrioList *details, const char *spamreport_block) { Spamreport *s; + OutgoingWebRequest *request; char urlbuf[512]; char bodybuf[512]; char *url = NULL; @@ -472,8 +473,15 @@ int _spamreport(Client *client, const char *ip, NameValuePrioList *details, cons log_data_string("url", url), log_data_string("body", (body ? body : ""))); #endif - url_start_async(url, s->http_method, body, headers, 0, 0, download_complete_dontcare, NULL, url, 3); - safe_free_nvplist(headers); + /* Do the web request */ + request = safe_alloc(sizeof(OutgoingWebRequest)); + safe_strdup(request->url, url); + request->http_method = s->http_method; + safe_strdup(request->body, body); + request->headers = headers; + request->callback = download_complete_dontcare; + request->max_redirects = 3; + url_start_async(request); return 1; } diff --git a/src/url_curl.c b/src/url_curl.c index ed401573c..a352560c0 100644 --- a/src/url_curl.c +++ b/src/url_curl.c @@ -33,17 +33,11 @@ typedef struct Download Download; struct Download { Download *prev, *next; - vFP callback; - void *callback_data; - FILE *file_fd; /**< File open for writing (otherwise NULL) */ - char *filename; - char *url; /*< must be free()d by url_do_transfers_async() */ - HttpMethod http_method; - char *body; - NameValuePrioList *request_headers; + OutgoingWebRequest *request; struct curl_slist *request_headers_curl; char errorbuf[CURL_ERROR_SIZE]; - time_t cachetime; + FILE *file_fd; /**< File open for writing (otherwise NULL) */ + char *filename; char *memory_data; /**< Memory for writing response (otherwise NULL) */ int memory_data_len; /**< Size of memory_data */ int memory_data_allocated; /**< Total allocated memory for 'memory_data' */ @@ -60,11 +54,9 @@ void url_free_handle(Download *handle) fclose(handle->file_fd); safe_free(handle->memory_data); safe_free(handle->filename); - safe_free(handle->url); - safe_free(handle->body); - safe_free_nvplist(handle->request_headers); if (handle->request_headers_curl) curl_slist_free_all(handle->request_headers_curl); + safe_free_outgoingwebrequest(handle->request); safe_free(handle); } @@ -75,10 +67,10 @@ void url_cancel_handle_by_callback_data(void *ptr) for (d = downloads; d; d = d_next) { d_next = d->next; - if (d->callback_data == ptr) + if (d->request->callback_data == ptr) { - d->callback = NULL; - d->callback_data = NULL; + d->request->callback = NULL; + d->request->callback_data = NULL; } } } @@ -182,7 +174,7 @@ static void url_check_multi_handles(void) handle->file_fd = NULL; } - if (handle->callback == NULL) + if (handle->request->callback == NULL) { /* Request is already canceled, we don't care about the result, just clean up */ if (handle->filename) @@ -190,9 +182,9 @@ static void url_check_multi_handles(void) } else if (msg->data.result == CURLE_OK) { - if (code == 304 || (last_mod != -1 && last_mod <= handle->cachetime)) + if (code == 304 || (last_mod != -1 && last_mod <= handle->request->cachetime)) { - handle->callback(handle->url, NULL, handle->memory_data, handle->memory_data_len, NULL, 1, handle->callback_data); + handle->request->callback(handle->request->url, NULL, handle->memory_data, handle->memory_data_len, NULL, 1, handle->request->callback_data); if (handle->filename) remove(handle->filename); } @@ -201,14 +193,14 @@ static void url_check_multi_handles(void) if ((last_mod != -1) && handle->filename) unreal_setfilemodtime(handle->filename, last_mod); - handle->callback(handle->url, handle->filename, handle->memory_data, handle->memory_data_len, NULL, 0, handle->callback_data); + handle->request->callback(handle->request->url, handle->filename, handle->memory_data, handle->memory_data_len, NULL, 0, handle->request->callback_data); if (handle->filename) remove(handle->filename); } } else { - handle->callback(handle->url, NULL, NULL, 0, handle->errorbuf, 0, handle->callback_data); + handle->request->callback(handle->request->url, NULL, NULL, 0, handle->errorbuf, 0, handle->request->callback_data); if (handle->filename) remove(handle->filename); } @@ -294,33 +286,7 @@ void url_init(void) url_socket_timeout_hdl = EventAdd(NULL, "url_socket_timeout", url_socket_timeout, NULL, 500, 0); } -/* - * Handles asynchronous downloading of a file. This function allows - * a download to be made transparently without the caller having any - * knowledge of how libcurl works. The specified callback function is - * called when the download completes, or the download fails. The - * callback function is defined as: - * - * void callback(const char *url, const char *filename, const char *memory_data, int memory_data_len, char *errorbuf, int cached, void *data); - * - url will contain the original URL used to download the file. - * - filename will contain the name of the file (if successful, NULL on error or if cached). - * This file will be cleaned up after the callback returns, so save a copy to support caching. - * - errorbuf will contain the error message (if failed, NULL otherwise). - * - cached 1 if the specified cachetime is >= the current file on the server, - * if so, errorbuf will be NULL, filename will contain the path to the file. - * - data will be the value of callback_data, allowing you to figure - * out how to use the data contained in the downloaded file ;-). - * Make sure that if you access the contents of this pointer, you - * know that this pointer will persist. A download could take more - * than 10 seconds to happen and the config file can be rehashed - * multiple times during that time. - */ -void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects) -{ - url_start_async(url, HTTP_METHOD_GET, NULL, NULL, 1, cachetime, callback, callback_data, original_url, maxredirects); -} - -void url_start_async(const char *url, HttpMethod http_method, const char *body, NameValuePrioList *request_headers, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects) +void url_start_async(OutgoingWebRequest *request) { static char errorbuf[CURL_ERROR_SIZE]; char user_agent[256]; @@ -330,6 +296,10 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body, char *tmp; Download *handle; + /* Check for the bare minimum */ + if (!request->url || !request->http_method) + abort(); + curl = curl_easy_init(); if (!curl) { @@ -340,17 +310,18 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body, } handle = safe_alloc(sizeof(Download)); + handle->request = request; - if (store_in_file) + if (handle->request->store_in_file) { - file = url_getfilename(url); + file = url_getfilename(handle->request->url); filename = unreal_getfilename(file); tmp = unreal_mktemp(TMPDIR, filename ? filename : "download.conf"); handle->file_fd = fopen(tmp, "wb"); if (!handle->file_fd) { snprintf(errorbuf, sizeof(errorbuf), "Cannot create '%s': %s", tmp, strerror(ERRNO)); - callback(url, NULL, NULL, 0, errorbuf, 0, callback_data); + handle->request->callback(handle->request->url, NULL, NULL, 0, errorbuf, 0, handle->request->callback_data); safe_free(file); safe_free(handle); return; @@ -361,15 +332,10 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body, AddListItem(handle, downloads); - handle->callback = callback; - handle->callback_data = callback_data; - handle->cachetime = cachetime; - safe_strdup(handle->url, url); - - curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_URL, handle->request->url); snprintf(user_agent, sizeof(user_agent), "UnrealIRCd %s", VERSIONONLY); curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent); - if (store_in_file) + if (handle->request->store_in_file) { curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, do_download_file); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)handle->file_fd); @@ -379,26 +345,22 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body, handle->memory_data_allocated = URL_MEMORY_BACKED_CHUNK_SIZE; handle->memory_data = safe_alloc(URL_MEMORY_BACKED_CHUNK_SIZE+1); } - if (http_method == HTTP_METHOD_POST) + if (handle->request->http_method == HTTP_METHOD_POST) { curl_easy_setopt(curl, CURLOPT_POST, 1); #if LIBCURL_VERSION_NUM >= 0x071301 curl_easy_setopt(curl, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL); #endif - if (body && strlen(body)) - { - safe_strdup(handle->body, body); // actually redundant? - curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, body); - } + if (handle->request->body && strlen(handle->request->body)) + curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, handle->request->body); } - if (request_headers) + if (handle->request->headers) { NameValuePrioList *n; char buf[512]; - handle->request_headers = duplicate_nvplist(request_headers); - for (n = request_headers; n; n = n->next) + for (n = handle->request->headers; n; n = n->next) { if (n->value) snprintf(buf, sizeof(buf), "%s: %s", n->name, n->value); @@ -425,17 +387,17 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body, * As a side-effect we also fix useless CLOSE_WAIT connections. */ curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1); - if (cachetime) + if (handle->request->cachetime) { curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); - curl_easy_setopt(curl, CURLOPT_TIMEVALUE, cachetime); + curl_easy_setopt(curl, CURLOPT_TIMEVALUE, handle->request->cachetime); } curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curl, CURLOPT_TIMEOUT, DOWNLOAD_TRANSFER_TIMEOUT); curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, DOWNLOAD_CONNECT_TIMEOUT); #if LIBCURL_VERSION_NUM >= 0x070f01 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); - curl_easy_setopt(curl, CURLOPT_MAXREDIRS, maxredirects); + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, handle->request->max_redirects); #endif curl_multi_add_handle(multihandle, curl); diff --git a/src/url_unreal.c b/src/url_unreal.c index 3d8333c92..4c8dc0f5d 100644 --- a/src/url_unreal.c +++ b/src/url_unreal.c @@ -31,20 +31,13 @@ typedef struct Download Download; struct Download { Download *prev, *next; - vFP callback; - void *callback_data; - char *url; /**< must be free()d by url_do_transfers_async() */ - HttpMethod http_method; - char *body; - NameValuePrioList *request_headers; - int store_in_file; + OutgoingWebRequest *request; FILE *file_fd; /**< File open for writing (otherwise NULL) */ char *filename; char *memory_data; /**< Memory for writing response (otherwise NULL) */ int memory_data_len; /**< Size of memory_data */ int memory_data_allocated; /**< Total allocated memory for 'memory_data' */ char errorbuf[512]; - time_t cachetime; char *hostname; /**< Parsed hostname (from 'url') */ int port; /**< Parsed port (from 'url') */ char *username; @@ -64,10 +57,7 @@ struct Download int dns_refcnt; TransferEncoding transfer_encoding; long chunk_remaining; - /* for redirects: */ - int redirects_remaining; char *redirect_new_location; - char *redirect_original_url; }; /* Variables */ @@ -105,8 +95,6 @@ void url_free_handle(Download *handle) fclose(handle->file_fd); safe_free(handle->filename); safe_free(handle->memory_data); - safe_free(handle->body); - safe_free_nvplist(handle->request_headers); safe_free(handle->hostname); safe_free(handle->username); safe_free(handle->password); @@ -116,8 +104,7 @@ void url_free_handle(Download *handle) SSL_free(handle->ssl); safe_free(handle->lefttoparse); safe_free(handle->redirect_new_location); - safe_free(handle->redirect_original_url); - safe_free(handle->url); + safe_free_outgoingwebrequest(handle->request); safe_free(handle); } @@ -128,15 +115,15 @@ void url_cancel_handle_by_callback_data(void *ptr) for (d = downloads; d; d = d_next) { d_next = d->next; - if (d->callback_data == ptr) + if (d->request->callback_data == ptr) { - d->callback = NULL; - d->callback_data = NULL; + d->request->callback = NULL; + d->request->callback_data = NULL; } } } -/** Cancel and free the HTTPS request. +/** Cancel and free the HTTPS request-> * @returns Always returns -1 */ int https_cancel(Download *handle, FORMAT_STRING(const char *pattern), ...) @@ -145,49 +132,41 @@ int https_cancel(Download *handle, FORMAT_STRING(const char *pattern), ...) va_start(vl, pattern); vsnprintf(handle->errorbuf, sizeof(handle->errorbuf), pattern, vl); va_end(vl); - if (handle->callback) - handle->callback(handle->url, NULL, NULL, 0, handle->errorbuf, 0, handle->callback_data); + if (handle->request->callback) + handle->request->callback(handle->request->url, NULL, NULL, 0, handle->errorbuf, 0, handle->request->callback_data); url_free_handle(handle); return -1; } -void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects) -{ - url_start_async(url, HTTP_METHOD_GET, NULL, NULL, 1, cachetime, callback, callback_data, original_url, maxredirects); -} - -void url_start_async(const char *url, HttpMethod http_method, const char *body, NameValuePrioList *request_headers, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects) +void url_start_async(OutgoingWebRequest *request) { char *file; const char *filename; char *tmp; Download *handle = NULL; int ipv6 = 0; + const char *actual_url = request->actual_url ? request->actual_url : request->url; char *host; int port; char *username; char *password; char *document; + /* Check for the bare minimum */ + if (!request->url || !request->http_method) + abort(); + handle = safe_alloc(sizeof(Download)); handle->download_started = TStime(); - handle->callback = callback; - handle->callback_data = callback_data; - handle->cachetime = cachetime; - safe_strdup(handle->url, url); - safe_strdup(handle->redirect_original_url, original_url); - handle->redirects_remaining = maxredirects; - handle->http_method = http_method; - safe_strdup(handle->body, body); - handle->store_in_file = store_in_file; + handle->request = request; AddListItem(handle, downloads); - if (strncmp(url, "https://", 8)) + if (strncmp(actual_url, "https://", 8)) { https_cancel(handle, "Only https:// is supported (either rebuild UnrealIRCd with curl support or use https)"); return; } - if (!url_parse(url, &host, &port, &username, &password, &document)) + if (!url_parse(actual_url, &host, &port, &username, &password, &document)) { https_cancel(handle, "Failed to parse HTTP url"); return; @@ -199,9 +178,9 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body, safe_strdup(handle->password, password); safe_strdup(handle->document, document); - if (store_in_file) + if (request->store_in_file) { - file = url_getfilename(url); + file = url_getfilename(handle->request->url); filename = unreal_getfilename(file); tmp = unreal_mktemp(TMPDIR, filename ? filename : "download.conf"); @@ -220,13 +199,6 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body, handle->memory_data = safe_alloc(URL_MEMORY_BACKED_CHUNK_SIZE); } - if (request_headers) - handle->request_headers = duplicate_nvplist(request_headers); - - // todo: allocate handle, select en weetikt allemaal - // add to some global struct linkedlist, for timeouts - // register in i/o - if (is_valid_ip(handle->hostname)) { /* Nothing to resolve, eg https://127.0.0.1/ */ @@ -524,7 +496,7 @@ int https_connect_send_header(Download *handle) snprintf(hostandport, sizeof(hostandport), "%s:%d", handle->hostname, handle->port); /* Prepare the header */ - if (handle->http_method == HTTP_METHOD_GET) + if (handle->request->http_method == HTTP_METHOD_GET) { snprintf(buf, sizeof(buf), "GET %s HTTP/1.1\r\n" "User-Agent: UnrealIRCd %s\r\n" @@ -534,9 +506,9 @@ int https_connect_send_header(Download *handle) VERSIONONLY, hostandport); } else - if (handle->http_method == HTTP_METHOD_POST) + if (handle->request->http_method == HTTP_METHOD_POST) { - if (!handle->body || !strlen(handle->body)) + if (!handle->request->body || !strlen(handle->request->body)) { snprintf(buf, sizeof(buf), "POST %s HTTP/1.1\r\n" "User-Agent: UnrealIRCd %s\r\n" @@ -547,7 +519,7 @@ int https_connect_send_header(Download *handle) hostandport); } else { char add_default_content_type = 0; - if (!find_nvplist(handle->request_headers, "Content-Type")) + if (!find_nvplist(handle->request->headers, "Content-Type")) add_default_content_type = 1; snprintf(buf, sizeof(buf), "POST %s HTTP/1.1\r\n" @@ -560,7 +532,7 @@ int https_connect_send_header(Download *handle) VERSIONONLY, hostandport, add_default_content_type ? "Content-Type: application/x-www-form-urlencoded\r\n" : "", - (long)strlen(handle->body)); + (long)strlen(handle->request->body)); } } else abort(); @@ -577,9 +549,9 @@ int https_connect_send_header(Download *handle) strlcat(buf, header, sizeof(buf)); } } - if (handle->cachetime > 0) + if (handle->request->cachetime > 0) { - const char *datestr = rfc2616_time(handle->cachetime); + const char *datestr = rfc2616_time(handle->request->cachetime); if (datestr) { // snprintf_append... @@ -587,12 +559,12 @@ int https_connect_send_header(Download *handle) "If-Modified-Since: %s\r\n", datestr); } } - if (handle->request_headers) + if (handle->request->headers) { NameValuePrioList *n; char nbuf[256]; - for (n = handle->request_headers; n; n = n->next) + for (n = handle->request->headers; n; n = n->next) { if (n->value) snprintf(nbuf, sizeof(nbuf), "%s: %s\r\n", n->name, n->value); @@ -604,8 +576,8 @@ int https_connect_send_header(Download *handle) } } strlcat(buf, "\r\n", sizeof(buf)); - if (handle->body) - strlcat(buf, handle->body, sizeof(buf)); + if (handle->request->body) + strlcat(buf, handle->request->body, sizeof(buf)); ssl_err = SSL_write(handle->ssl, buf, strlen(buf)); if (ssl_err < 0) @@ -720,7 +692,7 @@ int https_handle_response_header(Download *handle, char *readbuf, int n) else if ((handle->http_status_code >= 301) && (handle->http_status_code <= 308)) { /* Redirect */ - if (handle->redirects_remaining == 0) + if (handle->request->max_redirects == 0) { https_cancel(handle, "Too many HTTP redirects (%d)", DOWNLOAD_MAX_REDIRECTS); return 0; @@ -838,7 +810,7 @@ int https_handle_response_body(Download *handle, char *readbuf, int pktsize) if (handle->transfer_encoding == TRANSFER_ENCODING_NONE) { /* Ohh.. so easy! */ - if (handle->store_in_file == 0) + if (handle->request->store_in_file == 0) https_handle_response_body_memory(handle, readbuf, pktsize); else if (handle->file_fd) fwrite(readbuf, 1, pktsize, handle->file_fd); @@ -868,7 +840,7 @@ int https_handle_response_body(Download *handle, char *readbuf, int pktsize) { /* Eat it */ int eat = MIN(handle->chunk_remaining, n); - if (handle->store_in_file == 0) + if (handle->request->store_in_file == 0) https_handle_response_body_memory(handle, buf, eat); else if (handle->file_fd) fwrite(buf, 1, eat, handle->file_fd); @@ -948,23 +920,21 @@ int https_handle_response_body(Download *handle, char *readbuf, int pktsize) void https_done(Download *handle) { - char *url = handle->redirect_original_url ? handle->redirect_original_url : handle->url; - if (handle->file_fd) { fclose(handle->file_fd); handle->file_fd = NULL; } - if (!handle->callback) + if (!handle->request->callback) ; /* No special action, request was cancelled */ else if (!handle->got_response) - handle->callback(url, NULL, NULL, 0, "HTTPS response not received", 0, handle->callback_data); + handle->request->callback(handle->request->url, NULL, NULL, 0, "HTTPS response not received", 0, handle->request->callback_data); else { if ((handle->last_modified > 0) && handle->filename) unreal_setfilemodtime(handle->filename, handle->last_modified); - handle->callback(url, handle->filename, handle->memory_data, handle->memory_data_len, NULL, 0, handle->callback_data); + handle->request->callback(handle->request->url, handle->filename, handle->memory_data, handle->memory_data_len, NULL, 0, handle->request->callback_data); } url_free_handle(handle); return; @@ -972,37 +942,38 @@ void https_done(Download *handle) void https_done_cached(Download *handle) { - char *url = handle->redirect_original_url ? handle->redirect_original_url : handle->url; - if (handle->file_fd) { fclose(handle->file_fd); handle->file_fd = NULL; } - if (handle->callback) - handle->callback(url, NULL, NULL, 0, NULL, 1, handle->callback_data); + if (handle->request->callback) + handle->request->callback(handle->request->url, NULL, NULL, 0, NULL, 1, handle->request->callback_data); url_free_handle(handle); } void https_redirect(Download *handle) { - if (handle->redirects_remaining == 0) + if (handle->request->max_redirects == 0) { https_cancel(handle, "Too many HTTP redirects (%d)", DOWNLOAD_MAX_REDIRECTS); return; } - handle->redirects_remaining--; - if (handle->callback) + /* If still an outstanding request (not cancelled), follow the redirect.. */ + if (handle->request->callback) { - /* If still an outstanding request (not cancelled), follow the redirect.. */ - url_start_async(handle->redirect_new_location, handle->http_method, handle->body, - handle->request_headers, handle->store_in_file, - handle->cachetime, handle->callback, handle->callback_data, - handle->url, handle->redirects_remaining); + OutgoingWebRequest *r = duplicate_outgoingwebrequest(handle->request); + safe_strdup(r->actual_url, handle->redirect_new_location); // override actual url + r->max_redirects--; // safe, checked to be >0 a few lines up + url_free_handle(handle); // free old handle + url_start_async(r); // create new one + } else { + /* The callback is NULL, so we can just free without + * following redirects and any error reporting + */ + url_free_handle(handle); // free old handle } - /* Don't call the hook, just free this, the new redirect from above will call the hook later */ - url_free_handle(handle); } /** Helper function to parse the HTTP header consisting of multiple 'Key: value' pairs */