diff --git a/include/config.h b/include/config.h index 6ae2f3833..36b888917 100644 --- a/include/config.h +++ b/include/config.h @@ -211,11 +211,16 @@ #define DOWNLOAD_MAX_REDIRECTS 2 /* Default maximum size (in bytes) for memory-backed HTTP responses - * (i.e. when OutgoingWebRequest.store_in_file == 0). Responses exceeding - * this are rejected and the transfer is aborted. Callers can override - * by setting OutgoingWebRequest.max_size before url_start_async(). + * (store_in_file being 0). Responses exceeding this are rejected. + * API callers override this by setting .max_size before url_start_async(). */ -#define DOWNLOAD_MAX_SIZE 1048576 +#define DOWNLOAD_MAX_SIZE_MEMORY_BACKED 1048576 + +/* Default maximum size (in bytes) for file-backed HTTP responses + * (store_in_file being 1). Responses exceeding this are rejected. + * API callers override this by setting .max_size before url_start_async(). + */ +#define DOWNLOAD_MAX_SIZE_FILE_BACKED 52428800 /* * Max time from the nickname change that still causes KILL diff --git a/include/struct.h b/include/struct.h index 7a922e6d4..5ec08e1b1 100644 --- a/include/struct.h +++ b/include/struct.h @@ -1982,8 +1982,11 @@ struct OutgoingWebRequest int connect_timeout; /**< How many seconds to wait for the (TLS) connect to succeed */ int transfer_timeout; /**< How many seconds the total transfer may take (connect+reading everything) */ int minimum_tls_version; - long long max_size; /**< Max response size for memory-backed downloads, in bytes. - * 0 = use DOWNLOAD_MAX_SIZE. Ignored for file-backed. */ + long long max_size; /**< Max response size, in bytes. 0 selects a default + * based on the download mode: + * DOWNLOAD_MAX_SIZE_MEMORY_BACKED (small, since it + * sits in RAM) or DOWNLOAD_MAX_SIZE_FILE_BACKED + * (larger). */ // If you are adding fields here: // 1) update duplicate_outgoingwebrequest() in src/misc.c // 2) and update free_outgoingwebrequest() there as well (if something needs to be freed) diff --git a/src/url_curl.c b/src/url_curl.c index 8ecd05c01..44d225def 100644 --- a/src/url_curl.c +++ b/src/url_curl.c @@ -40,7 +40,8 @@ struct Download char *memory_data; /**< Memory for writing response (otherwise NULL) */ long long memory_data_len; /**< Size of memory_data */ long long memory_data_allocated; /**< Total allocated memory for 'memory_data' */ - int cap_exceeded; /**< Set to 1 by do_download_memory when max_size was hit */ + long long bytes_written_to_file; /**< Bytes written to file_fd so far (for max_size cap) */ + int cap_exceeded; /**< Set to 1 by do_download_memory/file when max_size was hit */ }; CURLM *multihandle = NULL; @@ -163,7 +164,21 @@ static void set_curl_tls_options(CURL *curl, int minimum_tls_version) */ static size_t do_download_file(void *ptr, size_t size, size_t nmemb, void *stream) { - return fwrite(ptr, size, nmemb, (FILE *)stream); + Download *handle = (Download *)stream; + size_t write_sz = size * nmemb; + long long size_required = handle->bytes_written_to_file + (long long)write_sz; + + if (size_required > handle->request->max_size) + { + handle->cap_exceeded = 1; + return 0; /* aborts the curl transfer; real error surfaced in url_check_multi_handles */ + } + + if (fwrite(ptr, size, nmemb, handle->file_fd) != nmemb) + return 0; /* short write -> abort */ + + handle->bytes_written_to_file += (long long)write_sz; + return nmemb; } /* @@ -370,7 +385,7 @@ void url_start_async(OutgoingWebRequest *request) /* Set request defaults */ if (request->max_size <= 0) - request->max_size = DOWNLOAD_MAX_SIZE; + request->max_size = request->store_in_file ? DOWNLOAD_MAX_SIZE_FILE_BACKED : DOWNLOAD_MAX_SIZE_MEMORY_BACKED; curl = curl_easy_init(); if (!curl) @@ -410,7 +425,7 @@ void url_start_async(OutgoingWebRequest *request) if (handle->request->store_in_file) { curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, do_download_file); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)handle->file_fd); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)handle); } else { curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, do_download_memory); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)handle); @@ -467,6 +482,8 @@ void url_start_async(OutgoingWebRequest *request) curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curl, CURLOPT_TIMEOUT, request->transfer_timeout ? request->transfer_timeout : DOWNLOAD_TRANSFER_TIMEOUT); curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, request->connect_timeout ? request->connect_timeout : DOWNLOAD_CONNECT_TIMEOUT); + /* Reject early if Content-Length exceeds the cap. */ + curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, (curl_off_t)request->max_size); #if LIBCURL_VERSION_NUM >= 0x070f01 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curl, CURLOPT_MAXREDIRS, handle->request->max_redirects); diff --git a/src/url_unreal.c b/src/url_unreal.c index 98ad6975c..bf3a1ac29 100644 --- a/src/url_unreal.c +++ b/src/url_unreal.c @@ -63,6 +63,7 @@ struct Download char *memory_data; /**< Memory for writing response (otherwise NULL) */ long long memory_data_len; /**< Size of memory_data */ long long memory_data_allocated; /**< Total allocated memory for 'memory_data' */ + long long bytes_written_to_file; /**< Bytes written to file_fd so far (for max_size cap) */ char errorbuf[512]; char *hostname; /**< Parsed hostname (from 'url') */ int port; /**< Parsed port (from 'url') */ @@ -190,7 +191,7 @@ void url_start_async(OutgoingWebRequest *request) if (request->transfer_timeout == 0) request->transfer_timeout = DOWNLOAD_TRANSFER_TIMEOUT; if (request->max_size <= 0) - request->max_size = DOWNLOAD_MAX_SIZE; + request->max_size = request->store_in_file ? DOWNLOAD_MAX_SIZE_FILE_BACKED : DOWNLOAD_MAX_SIZE_MEMORY_BACKED; handle = safe_alloc(sizeof(Download)); handle->download_started = TStime(); @@ -879,7 +880,15 @@ int https_handle_response_body(Download *handle, char *readbuf, int pktsize) https_handle_response_body_memory(handle, readbuf, pktsize); } else if (handle->file_fd) + { + if (handle->bytes_written_to_file + pktsize > handle->request->max_size) + { + https_cancel(handle, "Response too large (maximum: %lld bytes)", handle->request->max_size); + return 0; /* handle freed */ + } fwrite(readbuf, 1, pktsize, handle->file_fd); + handle->bytes_written_to_file += pktsize; + } return 1; } @@ -917,7 +926,16 @@ int https_handle_response_body(Download *handle, char *readbuf, int pktsize) https_handle_response_body_memory(handle, buf, eat); } else if (handle->file_fd) + { + if (handle->bytes_written_to_file + eat > handle->request->max_size) + { + https_cancel(handle, "Response too large (maximum: %lld bytes)", handle->request->max_size); + safe_free(free_this_buffer); + return 0; /* handle freed */ + } fwrite(buf, 1, eat, handle->file_fd); + handle->bytes_written_to_file += eat; + } n -= eat; buf += eat; handle->chunk_remaining -= eat;