From 2c7d2c8f86ef0f540406c4faa0843da80ec6d1ce Mon Sep 17 00:00:00 2001 From: Bram Matthys Date: Sat, 21 Aug 2021 08:47:38 +0200 Subject: [PATCH] Split url.c into url_curl.c (curl implementation) and url_unreal (the new fallback https-only implementation). ./configure will set URL= to either url_curl.o or url_unreal.o depending on whether curl is enabled or not. The 3 functions that both implementations had in common are now in src/misc.c: url_is_valid(), displayurl() and url_getfilename(). --- Makefile.in | 2 + autoconf/m4/unreal.m4 | 7 + configure | 8 + src/Makefile.in | 11 +- src/misc.c | 84 ++++++++ src/url_curl.c | 312 +++++++++++++++++++++++++++ src/{url.c => url_unreal.c} | 409 +++--------------------------------- 7 files changed, 445 insertions(+), 388 deletions(-) create mode 100644 src/url_curl.c rename src/{url.c => url_unreal.c} (66%) diff --git a/Makefile.in b/Makefile.in index bfac85019..abff53ead 100644 --- a/Makefile.in +++ b/Makefile.in @@ -90,6 +90,7 @@ XCFLAGS=@PTHREAD_CFLAGS@ @PCRE2_CFLAGS@ @ARGON2_CFLAGS@ @CARES_CFLAGS@ @SODIUM_C IRCDMODE = 711 # Objects that are optional due to optional libraries: +URL=@URL@ GEOIP_CLASSIC_OBJECTS=@GEOIP_CLASSIC_OBJECTS@ GEOIP_CLASSIC_LIBS=@GEOIP_CLASSIC_LIBS@ GEOIP_CLASSIC_CFLAGS=@GEOIP_CLASSIC_CFLAGS@ @@ -119,6 +120,7 @@ MAKEARGS = 'CFLAGS=${CFLAGS}' 'CC=${CC}' 'IRCDLIBS=${IRCDLIBS}' \ 'SHELL=${SHELL}' \ 'CRYPTOLIB=${CRYPTOLIB}' \ 'CRYPTOINCLUDES=${CRYPTOINCLUDES}' \ + 'URL=${URL}' \ 'GEOIP_CLASSIC_OBJECTS=${GEOIP_CLASSIC_OBJECTS}' \ 'GEOIP_CLASSIC_LIBS=${GEOIP_CLASSIC_LIBS}' \ 'GEOIP_CLASSIC_CFLAGS=${GEOIP_CLASSIC_CFLAGS}' diff --git a/autoconf/m4/unreal.m4 b/autoconf/m4/unreal.m4 index 2a3d4f0b4..4c66a32e5 100644 --- a/autoconf/m4/unreal.m4 +++ b/autoconf/m4/unreal.m4 @@ -129,7 +129,14 @@ AC_DEFUN([CHECK_LIBCURL], ]) LIBS="$LIBS_SAVEDA" CFLAGS="$CFLAGS_SAVEDA" + + dnl Finally, choose the cURL implementation of url.c + URL="url_curl.o" + ],[ + dnl Choose UnrealIRCds internal implementation of url.c + URL="url_unreal.o" ]) dnl AS_IF(enable_curl) + AC_SUBST(URL) ]) dnl the following 2 macros are based on CHECK_SSL by Mark Ethan Trostler diff --git a/configure b/configure index 5c998ac66..ed4c5560f 100755 --- a/configure +++ b/configure @@ -629,6 +629,7 @@ IRCDLIBS GEOIP_CLASSIC_OBJECTS GEOIP_CLASSIC_LIBS GEOIP_CLASSIC_CFLAGS +URL PTHREAD_CFLAGS PTHREAD_LIBS PTHREAD_CC @@ -8546,9 +8547,16 @@ rm -f core conftest.err conftest.$ac_objext \ LIBS="$LIBS_SAVEDA" CFLAGS="$CFLAGS_SAVEDA" + URL="url_curl.o" + +else + + URL="url_unreal.o" + fi + # Check whether --enable-geoip_classic was given. if test "${enable_geoip_classic+set}" = set; then : enableval=$enable_geoip_classic; enable_geoip_classic=$enableval diff --git a/src/Makefile.in b/src/Makefile.in index ee8a67c53..8d99b6109 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -33,7 +33,7 @@ OBJS=dns.o auth.o channel.o crule.o dbuf.o \ api-event.o \ crypt_blowfish.o unrealdb.o updconf.o crashreport.o modulemanager.o \ utf8.o log.o \ - openssl_hostname_validation.o url.o + openssl_hostname_validation.o $(URL) SRC=$(OBJS:%.o=%.c) @@ -245,8 +245,13 @@ log.o: log.c $(INCLUDES) openssl_hostname_validation.o: openssl_hostname_validation.c $(INCLUDES) $(CC) $(CFLAGS) $(BINCFLAGS) -c openssl_hostname_validation.c -url.o: url.c $(INCLUDES) - $(CC) $(CFLAGS) $(BINCFLAGS) -c url.c +# The cURL implementation... +url_curl.o: url_curl.c $(INCLUDES) + $(CC) $(CFLAGS) $(BINCFLAGS) -c url_curl.c + +# ... or our implementation. +url_unreal.o: url_unreal.c $(INCLUDES) + $(CC) $(CFLAGS) $(BINCFLAGS) -c url_unreal.c # DO NOT DELETE THIS LINE -- make depend depends on it. diff --git a/src/misc.c b/src/misc.c index e270c22ce..0e122b5e1 100644 --- a/src/misc.c +++ b/src/misc.c @@ -2260,3 +2260,87 @@ void write_pidfile(void) write_pidfile_failed(); #endif } + +/* + * Determines if the given string is a valid URL. Since libcurl + * supports telnet, ldap, and dict such strings are treated as + * invalid URLs here since we don't want them supported in + * unreal. + */ +int url_is_valid(const char *string) +{ + if (strstr(string, " ") || strstr(string, "\t")) + return 0; + + if (strstr(string, "telnet://") == string || + strstr(string, "ldap://") == string || + strstr(string, "dict://") == string) + { + return 0; + } + return (strstr(string, "://") != NULL); +} + +/** A displayable URL for in error messages and such. + * This leaves out any authentication information (user:pass) + * the URL may contain. + */ +const char *displayurl(const char *url) +{ + static char buf[512]; + char *proto, *rest; + + /* protocol://user:pass@host/etc.. */ + rest = strchr(url, '@'); + + if (!rest) + return url; /* contains no auth information */ + + rest++; /* now points to the rest (remainder) of the URL */ + + proto = strstr(url, "://"); + if (!proto || (proto > rest) || (proto == url)) + return url; /* incorrectly formatted, just show entire URL. */ + + /* funny, we don't ship strlncpy.. */ + *buf = '\0'; + strlncat(buf, url, sizeof(buf), proto - url); + strlcat(buf, "://***:***@", sizeof(buf)); + strlcat(buf, rest, sizeof(buf)); + + return buf; +} + +/* + * Returns the filename portion of the URL. The returned string + * is malloc()'ed and must be freed by the caller. If the specified + * URL does not contain a filename, a '-' is allocated and returned. + */ +char *url_getfilename(const char *url) +{ + const char *c, *start; + + if ((c = strstr(url, "://"))) + c += 3; + else + c = url; + + while (*c && *c != '/') + c++; + + if (*c == '/') + { + c++; + if (!*c || *c == '?') + return raw_strdup("-"); + start = c; + while (*c && *c != '?') + c++; + if (!*c) + return raw_strdup(start); + else + return raw_strldup(start, c-start+1); + + } + return raw_strdup("-"); +} diff --git a/src/url_curl.c b/src/url_curl.c new file mode 100644 index 000000000..85a4eb419 --- /dev/null +++ b/src/url_curl.c @@ -0,0 +1,312 @@ +/* + * Unreal Internet Relay Chat Daemon, src/url.c + * (C) 2003 Dominick Meglio and the UnrealIRCd Team + * (C) 2004-2021 Bram Matthys + * (C) 2012 William Pitcock + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 1, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include "unrealircd.h" +#include "dns.h" + +extern char *TLSKeyPasswd; + +/* Stores information about the async transfer. + * Used to maintain information about the transfer + * to trigger the callback upon completion. + */ +typedef struct Download Download; + +struct Download +{ + vFP callback; + void *callback_data; + FILE *file_fd; /**< File open for writing (otherwise NULL) */ + char filename[PATH_MAX]; + char *url; /*< must be free()d by url_do_transfers_async() */ + char errorbuf[CURL_ERROR_SIZE]; + time_t cachetime; +}; + +CURLM *multihandle = NULL; + +void url_free_handle(Download *handle) +{ + if (handle->file_fd) + fclose(handle->file_fd); + safe_free(handle->url); + safe_free(handle); +} + +/* + * Sets up all of the SSL options necessary to support HTTPS/FTPS + * transfers. + */ +static void set_curl_tls_options(CURL *curl) +{ + char buf[512]; + +#if 0 + /* This would only be necessary if you use client certificates over HTTPS and such. + * But this information is not known yet since the configuration file has not been + * parsed yet at this point. + */ + curl_easy_setopt(curl, CURLOPT_SSLCERT, iConf.tls_options->certificate_file); + if (TLSKeyPasswd) + curl_easy_setopt(curl, CURLOPT_SSLKEYPASSWD, TLSKeyPasswd); + curl_easy_setopt(curl, CURLOPT_SSLKEY, iConf.tls_options->key_file); +#endif + + snprintf(buf, sizeof(buf), "%s/tls/curl-ca-bundle.crt", CONFDIR); + curl_easy_setopt(curl, CURLOPT_CAINFO, buf); +} + +/* + * Used by CURLOPT_WRITEFUNCTION to actually write the data to + * a stream. + */ +static size_t do_download(void *ptr, size_t size, size_t nmemb, void *stream) +{ + return fwrite(ptr, size, nmemb, (FILE *)stream); +} + +/* + * Interface for new-style evented I/O. + * + * url_socket_pollcb is the callback from our eventing system into + * cURL. + * + * The other callbacks are for cURL notifying our event system what + * it wants to do. + */ +static void url_check_multi_handles(void) +{ + CURLMsg *msg; + int msgs_left; + + while ((msg = curl_multi_info_read(multihandle, &msgs_left)) != NULL) + { + if (msg->msg == CURLMSG_DONE) + { + Download *handle; + long code; + long last_mod; + CURL *easyhand = msg->easy_handle; + + curl_easy_getinfo(easyhand, CURLINFO_RESPONSE_CODE, &code); + curl_easy_getinfo(easyhand, CURLINFO_PRIVATE, (char **) &handle); + curl_easy_getinfo(easyhand, CURLINFO_FILETIME, &last_mod); + fclose(handle->file_fd); + handle->file_fd = NULL; + + if (msg->data.result == CURLE_OK) + { + if (code == 304 || (last_mod != -1 && last_mod <= handle->cachetime)) + { + handle->callback(handle->url, NULL, NULL, 1, handle->callback_data); + remove(handle->filename); + } + else + { + if (last_mod != -1) + unreal_setfilemodtime(handle->filename, last_mod); + + handle->callback(handle->url, handle->filename, NULL, 0, handle->callback_data); + remove(handle->filename); + } + } + else + { + handle->callback(handle->url, NULL, handle->errorbuf, 0, handle->callback_data); + remove(handle->filename); + } + + url_free_handle(handle); + curl_multi_remove_handle(multihandle, easyhand); + + /* NOTE: after curl_multi_remove_handle() you cannot use + * 'msg' anymore because it has freed by curl (as of v7.11.0), + * therefore 'easyhand' is used... fun! -- Syzop + */ + curl_easy_cleanup(easyhand); + } + } +} + +static void url_socket_pollcb(int fd, int revents, void *data) +{ + int flags = 0; + int dummy; + + if (revents & FD_SELECT_READ) + flags |= CURL_CSELECT_IN; + if (revents & FD_SELECT_WRITE) + flags |= CURL_CSELECT_OUT; + + curl_multi_socket_action(multihandle, fd, flags, &dummy); + url_check_multi_handles(); +} + +static int url_socket_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp) +{ + if (what == CURL_POLL_REMOVE) + { + fd_close(s); + } + else + { + FDEntry *fde = &fd_table[s]; + int flags = 0; + + if (!fde->is_open) + { + /* NOTE: We use FDCLOSE_NONE here because cURL will take + * care of the closing of the socket. So *WE* must never + * close the socket ourselves. + */ + fd_open(s, "CURL transfer", FDCLOSE_NONE); + } + + if (what == CURL_POLL_IN || what == CURL_POLL_INOUT) + flags |= FD_SELECT_READ; + + if (what == CURL_POLL_OUT || what == CURL_POLL_INOUT) + flags |= FD_SELECT_WRITE; + + fd_setselect(s, flags, url_socket_pollcb, NULL); + } + + return 0; +} + +/* Handle timeouts. */ +EVENT(curl_socket_timeout) +{ + int dummy; + + curl_multi_socket_action(multihandle, CURL_SOCKET_TIMEOUT, 0, &dummy); + url_check_multi_handles(); +} + +static Event *curl_socket_timeout_hdl = NULL; + +/* + * Initializes the URL system + */ +void url_init(void) +{ + curl_global_init(CURL_GLOBAL_ALL); + multihandle = curl_multi_init(); + + curl_multi_setopt(multihandle, CURLMOPT_SOCKETFUNCTION, url_socket_cb); + curl_socket_timeout_hdl = EventAdd(NULL, "curl_socket_timeout", curl_socket_timeout, NULL, 500, 0); +} + +/* + * Handles asynchronous downloading of a file. This function allows + * a download to be made transparently without the caller having any + * knowledge of how libcurl works. The specified callback function is + * called when the download completes, or the download fails. The + * callback function is defined as: + * + * void callback(const char *url, const char *filename, char *errorbuf, int cached, void *data); + * - url will contain the original URL used to download the file. + * - filename will contain the name of the file (if successful, NULL on error or if cached). + * This file will be cleaned up after the callback returns, so save a copy to support caching. + * - errorbuf will contain the error message (if failed, NULL otherwise). + * - cached 1 if the specified cachetime is >= the current file on the server, + * if so, errorbuf will be NULL, filename will contain the path to the file. + * - data will be the value of callback_data, allowing you to figure + * out how to use the data contained in the downloaded file ;-). + * Make sure that if you access the contents of this pointer, you + * know that this pointer will persist. A download could take more + * than 10 seconds to happen and the config file can be rehashed + * multiple times during that time. + */ +void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data) +{ + static char errorbuf[CURL_ERROR_SIZE]; + CURL *curl; + char *file; + char *filename; + char *tmp; + Download *handle; + + curl = curl_easy_init(); + if (!curl) + { + unreal_log(ULOG_ERROR, "main", "CURL_INTERNAL_FAILURE", NULL, + "Could not initialize curl handle. Maybe out of memory/resources?"); + snprintf(errorbuf, sizeof(errorbuf), "Could not initialize curl handle"); + return; + } + + file = url_getfilename(url); + filename = unreal_getfilename(file); + tmp = unreal_mktemp(TMPDIR, filename ? filename : "download.conf"); + + handle = safe_alloc(sizeof(Download)); + handle->file_fd = fopen(tmp, "wb"); + if (!handle->file_fd) + { + snprintf(errorbuf, sizeof(errorbuf), "Cannot create '%s': %s", tmp, strerror(ERRNO)); + callback(url, NULL, errorbuf, 0, callback_data); + safe_free(file); + safe_free(handle); + return; + } + + handle->callback = callback; + handle->callback_data = callback_data; + handle->cachetime = cachetime; + safe_strdup(handle->url, url); + strlcpy(handle->filename, tmp, sizeof(handle->filename)); + safe_free(file); + + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, do_download); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)handle->file_fd); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); + set_curl_tls_options(curl); + memset(handle->errorbuf, 0, CURL_ERROR_SIZE); + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, handle->errorbuf); + curl_easy_setopt(curl, CURLOPT_PRIVATE, (char *)handle); + curl_easy_setopt(curl, CURLOPT_FILETIME, 1); + /* We need to set CURLOPT_FORBID_REUSE because otherwise libcurl does not + * notify us (or not in time) about FD close/opens, thus we end up closing and + * screwing up another innocent FD, like a listener (BAD!). In my view a bug, but + * mailing list archives seem to indicate curl devs have a different opinion + * on these matters... + * Actually I don't know for sure if this option alone fixes 100% of the cases + * but at least I can't crash my server anymore. + * As a side-effect we also fix useless CLOSE_WAIT connections. + */ + curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1); + if (cachetime) + { + curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); + curl_easy_setopt(curl, CURLOPT_TIMEVALUE, cachetime); + } + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 45); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 15); +#if LIBCURL_VERSION_NUM >= 0x070f01 + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 1); +#endif + + curl_multi_add_handle(multihandle, curl); +} diff --git a/src/url.c b/src/url_unreal.c similarity index 66% rename from src/url.c rename to src/url_unreal.c index be73ca1b6..9dc6cbbd5 100644 --- a/src/url.c +++ b/src/url_unreal.c @@ -1,7 +1,6 @@ /* * Unreal Internet Relay Chat Daemon, src/url.c - * (C) 2003 Dominick Meglio and the UnrealIRCd Team - * (C) 2012 William Pitcock + * (C) 2021 Bram Matthys and the UnrealIRCd team * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,11 +20,10 @@ #include "unrealircd.h" #include "dns.h" -extern char *TLSKeyPasswd; +/* Defines */ +#define URL_ERROR_SIZE 512 -#ifndef CURL_ERROR_SIZE - #define CURL_ERROR_SIZE 512 -#endif +/* Structs */ /* Stores information about the async transfer. * Used to maintain information about the transfer @@ -35,17 +33,14 @@ typedef struct Download Download; struct Download { -#ifndef USE_LIBCURL Download *prev, *next; -#endif vFP callback; void *callback_data; FILE *file_fd; /**< File open for writing (otherwise NULL) */ char filename[PATH_MAX]; char *url; /*< must be free()d by url_do_transfers_async() */ - char errorbuf[CURL_ERROR_SIZE]; + char errorbuf[URL_ERROR_SIZE]; time_t cachetime; -#ifndef USE_LIBCURL char *hostname; /**< Parsed hostname (from 'url') */ int port; /**< Parsed port (from 'url') */ char *document; /**< Parsed document (from 'url') */ @@ -56,378 +51,10 @@ struct Download int got_response; char *lefttoparse; time_t last_modified; -#endif }; -#ifdef USE_LIBCURL -CURLM *multihandle = NULL; -#else +/* Variables */ Download *downloads = NULL; -#endif - -/* - * Determines if the given string is a valid URL. Since libcurl - * supports telnet, ldap, and dict such strings are treated as - * invalid URLs here since we don't want them supported in - * unreal. - */ -int url_is_valid(const char *string) -{ - if (strstr(string, " ") || strstr(string, "\t")) - return 0; - - if (strstr(string, "telnet://") == string || - strstr(string, "ldap://") == string || - strstr(string, "dict://") == string) - { - return 0; - } - return (strstr(string, "://") != NULL); -} - -/** A displayable URL for in error messages and such. - * This leaves out any authentication information (user:pass) - * the URL may contain. - */ -const char *displayurl(const char *url) -{ - static char buf[512]; - char *proto, *rest; - - /* protocol://user:pass@host/etc.. */ - rest = strchr(url, '@'); - - if (!rest) - return url; /* contains no auth information */ - - rest++; /* now points to the rest (remainder) of the URL */ - - proto = strstr(url, "://"); - if (!proto || (proto > rest) || (proto == url)) - return url; /* incorrectly formatted, just show entire URL. */ - - /* funny, we don't ship strlncpy.. */ - *buf = '\0'; - strlncat(buf, url, sizeof(buf), proto - url); - strlcat(buf, "://***:***@", sizeof(buf)); - strlcat(buf, rest, sizeof(buf)); - - return buf; -} - -/* - * Returns the filename portion of the URL. The returned string - * is malloc()'ed and must be freed by the caller. If the specified - * URL does not contain a filename, a '-' is allocated and returned. - */ -char *url_getfilename(const char *url) -{ - const char *c, *start; - - if ((c = strstr(url, "://"))) - c += 3; - else - c = url; - - while (*c && *c != '/') - c++; - - if (*c == '/') - { - c++; - if (!*c || *c == '?') - return raw_strdup("-"); - start = c; - while (*c && *c != '?') - c++; - if (!*c) - return raw_strdup(start); - else - return raw_strldup(start, c-start+1); - - } - return raw_strdup("-"); -} - -void url_free_handle(Download *handle) -{ - DelListItem(handle, downloads); - if (handle->fd > 0) - { - fd_close(handle->fd); - fd_unnotify(handle->fd); - } - if (handle->file_fd) - fclose(handle->file_fd); - safe_free(handle->url); - safe_free(handle->lefttoparse); - safe_free(handle); -} - -#ifdef USE_LIBCURL -/* - * Sets up all of the SSL options necessary to support HTTPS/FTPS - * transfers. - */ -static void set_curl_tls_options(CURL *curl) -{ - char buf[512]; - -#if 0 - /* This would only be necessary if you use client certificates over HTTPS and such. - * But this information is not known yet since the configuration file has not been - * parsed yet at this point. - */ - curl_easy_setopt(curl, CURLOPT_SSLCERT, iConf.tls_options->certificate_file); - if (TLSKeyPasswd) - curl_easy_setopt(curl, CURLOPT_SSLKEYPASSWD, TLSKeyPasswd); - curl_easy_setopt(curl, CURLOPT_SSLKEY, iConf.tls_options->key_file); -#endif - - snprintf(buf, sizeof(buf), "%s/tls/curl-ca-bundle.crt", CONFDIR); - curl_easy_setopt(curl, CURLOPT_CAINFO, buf); -} - -/* - * Used by CURLOPT_WRITEFUNCTION to actually write the data to - * a stream. - */ -static size_t do_download(void *ptr, size_t size, size_t nmemb, void *stream) -{ - return fwrite(ptr, size, nmemb, (FILE *)stream); -} - -/* - * Interface for new-style evented I/O. - * - * url_socket_pollcb is the callback from our eventing system into - * cURL. - * - * The other callbacks are for cURL notifying our event system what - * it wants to do. - */ -static void url_check_multi_handles(void) -{ - CURLMsg *msg; - int msgs_left; - - while ((msg = curl_multi_info_read(multihandle, &msgs_left)) != NULL) - { - if (msg->msg == CURLMSG_DONE) - { - Download *handle; - long code; - long last_mod; - CURL *easyhand = msg->easy_handle; - - curl_easy_getinfo(easyhand, CURLINFO_RESPONSE_CODE, &code); - curl_easy_getinfo(easyhand, CURLINFO_PRIVATE, (char **) &handle); - curl_easy_getinfo(easyhand, CURLINFO_FILETIME, &last_mod); - fclose(handle->file_fd); - handle->file_fd = NULL; - - if (msg->data.result == CURLE_OK) - { - if (code == 304 || (last_mod != -1 && last_mod <= handle->cachetime)) - { - handle->callback(handle->url, NULL, NULL, 1, handle->callback_data); - remove(handle->filename); - } - else - { - if (last_mod != -1) - unreal_setfilemodtime(handle->filename, last_mod); - - handle->callback(handle->url, handle->filename, NULL, 0, handle->callback_data); - remove(handle->filename); - } - } - else - { - handle->callback(handle->url, NULL, handle->errorbuf, 0, handle->callback_data); - remove(handle->filename); - } - - handle->fd = -1; /* curl will close it */ - url_free_handle(handle); - curl_multi_remove_handle(multihandle, easyhand); - - /* NOTE: after curl_multi_remove_handle() you cannot use - * 'msg' anymore because it has freed by curl (as of v7.11.0), - * therefore 'easyhand' is used... fun! -- Syzop - */ - curl_easy_cleanup(easyhand); - } - } -} - -static void url_socket_pollcb(int fd, int revents, void *data) -{ - int flags = 0; - int dummy; - - if (revents & FD_SELECT_READ) - flags |= CURL_CSELECT_IN; - if (revents & FD_SELECT_WRITE) - flags |= CURL_CSELECT_OUT; - - curl_multi_socket_action(multihandle, fd, flags, &dummy); - url_check_multi_handles(); -} - -static int url_socket_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp) -{ - if (what == CURL_POLL_REMOVE) - { - fd_close(s); - } - else - { - FDEntry *fde = &fd_table[s]; - int flags = 0; - - if (!fde->is_open) - { - /* NOTE: We use FDCLOSE_NONE here because cURL will take - * care of the closing of the socket. So *WE* must never - * close the socket ourselves. - */ - fd_open(s, "CURL transfer", FDCLOSE_NONE); - } - - if (what == CURL_POLL_IN || what == CURL_POLL_INOUT) - flags |= FD_SELECT_READ; - - if (what == CURL_POLL_OUT || what == CURL_POLL_INOUT) - flags |= FD_SELECT_WRITE; - - fd_setselect(s, flags, url_socket_pollcb, NULL); - } - - return 0; -} - -/* Handle timeouts. */ -EVENT(curl_socket_timeout) -{ - int dummy; - - curl_multi_socket_action(multihandle, CURL_SOCKET_TIMEOUT, 0, &dummy); - url_check_multi_handles(); -} - -static Event *curl_socket_timeout_hdl = NULL; - -/* - * Initializes the URL system - */ -void url_init(void) -{ - curl_global_init(CURL_GLOBAL_ALL); - multihandle = curl_multi_init(); - - curl_multi_setopt(multihandle, CURLMOPT_SOCKETFUNCTION, url_socket_cb); - curl_socket_timeout_hdl = EventAdd(NULL, "curl_socket_timeout", curl_socket_timeout, NULL, 500, 0); -} - -/* - * Handles asynchronous downloading of a file. This function allows - * a download to be made transparently without the caller having any - * knowledge of how libcurl works. The specified callback function is - * called when the download completes, or the download fails. The - * callback function is defined as: - * - * void callback(const char *url, const char *filename, char *errorbuf, int cached, void *data); - * - url will contain the original URL used to download the file. - * - filename will contain the name of the file (if successful, NULL on error or if cached). - * This file will be cleaned up after the callback returns, so save a copy to support caching. - * - errorbuf will contain the error message (if failed, NULL otherwise). - * - cached 1 if the specified cachetime is >= the current file on the server, - * if so, errorbuf will be NULL, filename will contain the path to the file. - * - data will be the value of callback_data, allowing you to figure - * out how to use the data contained in the downloaded file ;-). - * Make sure that if you access the contents of this pointer, you - * know that this pointer will persist. A download could take more - * than 10 seconds to happen and the config file can be rehashed - * multiple times during that time. - */ -void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data) -{ - static char errorbuf[CURL_ERROR_SIZE]; - CURL *curl; - char *file; - char *filename; - char *tmp; - Download *handle; - - curl = curl_easy_init(); - if (!curl) - { - unreal_log(ULOG_ERROR, "main", "CURL_INTERNAL_FAILURE", NULL, - "Could not initialize curl handle. Maybe out of memory/resources?"); - snprintf(errorbuf, sizeof(errorbuf), "Could not initialize curl handle"); - return; - } - - file = url_getfilename(url); - filename = unreal_getfilename(file); - tmp = unreal_mktemp(TMPDIR, filename ? filename : "download.conf"); - - handle = safe_alloc(sizeof(Download)); - handle->file_fd = fopen(tmp, "wb"); - if (!handle->file_fd) - { - snprintf(errorbuf, sizeof(errorbuf), "Cannot create '%s': %s", tmp, strerror(ERRNO)); - callback(url, NULL, errorbuf, 0, callback_data); - safe_free(file); - safe_free(handle); - return; - } - - handle->callback = callback; - handle->callback_data = callback_data; - handle->cachetime = cachetime; - safe_strdup(handle->url, url); - strlcpy(handle->filename, tmp, sizeof(handle->filename)); - safe_free(file); - - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, do_download); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)handle->file_fd); - curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); - set_curl_tls_options(curl); - memset(handle->errorbuf, 0, CURL_ERROR_SIZE); - curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, handle->errorbuf); - curl_easy_setopt(curl, CURLOPT_PRIVATE, (char *)handle); - curl_easy_setopt(curl, CURLOPT_FILETIME, 1); - /* We need to set CURLOPT_FORBID_REUSE because otherwise libcurl does not - * notify us (or not in time) about FD close/opens, thus we end up closing and - * screwing up another innocent FD, like a listener (BAD!). In my view a bug, but - * mailing list archives seem to indicate curl devs have a different opinion - * on these matters... - * Actually I don't know for sure if this option alone fixes 100% of the cases - * but at least I can't crash my server anymore. - * As a side-effect we also fix useless CLOSE_WAIT connections. - */ - curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1); - if (cachetime) - { - curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); - curl_easy_setopt(curl, CURLOPT_TIMEVALUE, cachetime); - } - curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 45); - curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 15); -#if LIBCURL_VERSION_NUM >= 0x070f01 - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); - curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 1); -#endif - - curl_multi_add_handle(multihandle, curl); -} - -#else -/************************* START OF OWN IMPLEMENTION (NON-CURL CASE) **********************/ /* Forward declarations */ void url_resolve_cb(void *arg, int status, int timeouts, struct hostent *he); @@ -446,9 +73,24 @@ void https_done_cached(Download *handle); int https_parse_header(char *buffer, int len, char **key, char **value, char **lastloc, int *end_of_request); char *url_find_end_of_request(char *header, int totalsize, int *remaining_bytes); +void url_free_handle(Download *handle) +{ + DelListItem(handle, downloads); + if (handle->fd > 0) + { + fd_close(handle->fd); + fd_unnotify(handle->fd); + } + if (handle->file_fd) + fclose(handle->file_fd); + safe_free(handle->url); + safe_free(handle->lefttoparse); + safe_free(handle); +} + void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data) { - static char errorbuf[CURL_ERROR_SIZE]; + static char errorbuf[URL_ERROR_SIZE]; char *file; char *filename; char *tmp; @@ -589,7 +231,7 @@ fail: void unreal_https_connect_handshake(int fd, int revents, void *data) { Download *handle = data; - char errorbuf[CURL_ERROR_SIZE]; + char errorbuf[URL_ERROR_SIZE]; SSL_CTX *ctx; ctx = https_new_ctx(); @@ -721,7 +363,7 @@ int https_fatal_tls_error(int ssl_error, int my_errno, Download *handle) char *ssl_errstr; unsigned long additional_errno = ERR_get_error(); char additional_info[256]; - char errorbuf[CURL_ERROR_SIZE]; + char errorbuf[URL_ERROR_SIZE]; const char *one, *two; #if OPENSSL_VERSION_NUMBER >= 0x30000000L @@ -1186,6 +828,3 @@ char *url_find_end_of_request(char *header, int totalsize, int *remaining_bytes) } return NULL; } - - -#endif