OutgoingWebRequest max_size is now also obeyed for file-backed URL API.

And the defines are more clear now (if .max_size is not set by caller.

DOWNLOAD_MAX_SIZE_MEMORY_BACKED: 1M
DOWNLOAD_MAX_SIZE_FILE_BACKED: 50M

The file-backed is mostly a defense-in-depth measure, so we don't
store infinite amounts of data in a download. Even though, in practice,
these - at least at the moment in unrealircd itself - all come from
trusted paths like remote includes.

In url_unreal.c we do the counting ourselves. In url_curl.c we use the
option CURLOPT_MAXFILESIZE_LARGE but this does not ensure it in all
cases so we still do our own counting as well in that file as well.

include/config.h

@@ -211,11 +211,16 @@
 #define DOWNLOAD_MAX_REDIRECTS 2
 
 /* Default maximum size (in bytes) for memory-backed HTTP responses
- * (i.e. when OutgoingWebRequest.store_in_file == 0). Responses exceeding
- * this are rejected and the transfer is aborted. Callers can override
- * by setting OutgoingWebRequest.max_size before url_start_async().
+ * (store_in_file being 0). Responses exceeding this are rejected.
+ * API callers override this by setting .max_size before url_start_async().
  */
-#define DOWNLOAD_MAX_SIZE 1048576
+#define DOWNLOAD_MAX_SIZE_MEMORY_BACKED 1048576
+
+/* Default maximum size (in bytes) for file-backed HTTP responses
+ * (store_in_file being 1). Responses exceeding this are rejected.
+ * API callers override this by setting .max_size before url_start_async().
+ */
+#define DOWNLOAD_MAX_SIZE_FILE_BACKED 52428800
 
 /*
  * Max time from the nickname change that still causes KILL

include/struct.h

@@ -1982,8 +1982,11 @@ struct OutgoingWebRequest
 	int connect_timeout; /**< How many seconds to wait for the (TLS) connect to succeed */
 	int transfer_timeout; /**< How many seconds the total transfer may take (connect+reading everything) */
 	int minimum_tls_version;
-	long long max_size; /**< Max response size for memory-backed downloads, in bytes.
-	                     *   0 = use DOWNLOAD_MAX_SIZE. Ignored for file-backed. */
+	long long max_size; /**< Max response size, in bytes. 0 selects a default
+	                     *   based on the download mode:
+	                     *   DOWNLOAD_MAX_SIZE_MEMORY_BACKED (small, since it
+	                     *   sits in RAM) or DOWNLOAD_MAX_SIZE_FILE_BACKED
+	                     *   (larger). */
 	// If you are adding fields here:
 	// 1) update duplicate_outgoingwebrequest() in src/misc.c
 	// 2) and update free_outgoingwebrequest() there as well (if something needs to be freed)

src/url_curl.c

@@ -40,7 +40,8 @@ struct Download
 	char *memory_data; /**< Memory for writing response (otherwise NULL) */
 	long long memory_data_len; /**< Size of memory_data */
 	long long memory_data_allocated; /**< Total allocated memory for 'memory_data' */
-	int cap_exceeded; /**< Set to 1 by do_download_memory when max_size was hit */
+	long long bytes_written_to_file; /**< Bytes written to file_fd so far (for max_size cap) */
+	int cap_exceeded; /**< Set to 1 by do_download_memory/file when max_size was hit */
 };
 
 CURLM *multihandle = NULL;
@@ -163,7 +164,21 @@ static void set_curl_tls_options(CURL *curl, int minimum_tls_version)
  */
 static size_t do_download_file(void *ptr, size_t size, size_t nmemb, void *stream)
 {
-	return fwrite(ptr, size, nmemb, (FILE *)stream);
+	Download *handle = (Download *)stream;
+	size_t write_sz = size * nmemb;
+	long long size_required = handle->bytes_written_to_file + (long long)write_sz;
+
+	if (size_required > handle->request->max_size)
+	{
+		handle->cap_exceeded = 1;
+		return 0; /* aborts the curl transfer; real error surfaced in url_check_multi_handles */
+	}
+
+	if (fwrite(ptr, size, nmemb, handle->file_fd) != nmemb)
+		return 0; /* short write -> abort */
+
+	handle->bytes_written_to_file += (long long)write_sz;
+	return nmemb;
 }
 
 /*
@@ -370,7 +385,7 @@ void url_start_async(OutgoingWebRequest *request)
 
 	/* Set request defaults */
 	if (request->max_size <= 0)
-		request->max_size = DOWNLOAD_MAX_SIZE;
+		request->max_size = request->store_in_file ? DOWNLOAD_MAX_SIZE_FILE_BACKED : DOWNLOAD_MAX_SIZE_MEMORY_BACKED;
 
 	curl = curl_easy_init();
 	if (!curl)
@@ -410,7 +425,7 @@ void url_start_async(OutgoingWebRequest *request)
 	if (handle->request->store_in_file)
 	{
 		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, do_download_file);
-		curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)handle->file_fd);
+		curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)handle);
 	} else {
 		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, do_download_memory);
 		curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)handle);
@@ -467,6 +482,8 @@ void url_start_async(OutgoingWebRequest *request)
 	curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
 	curl_easy_setopt(curl, CURLOPT_TIMEOUT, request->transfer_timeout ? request->transfer_timeout : DOWNLOAD_TRANSFER_TIMEOUT);
 	curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, request->connect_timeout ? request->connect_timeout : DOWNLOAD_CONNECT_TIMEOUT);
+	/* Reject early if Content-Length exceeds the cap. */
+	curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, (curl_off_t)request->max_size);
 #if LIBCURL_VERSION_NUM >= 0x070f01
 	curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
 	curl_easy_setopt(curl, CURLOPT_MAXREDIRS, handle->request->max_redirects);

src/url_unreal.c

@@ -63,6 +63,7 @@ struct Download
 	char *memory_data; /**< Memory for writing response (otherwise NULL) */
 	long long memory_data_len; /**< Size of memory_data */
 	long long memory_data_allocated; /**< Total allocated memory for 'memory_data' */
+	long long bytes_written_to_file; /**< Bytes written to file_fd so far (for max_size cap) */
 	char errorbuf[512];
 	char *hostname;		/**< Parsed hostname (from 'url') */
 	int port;		/**< Parsed port (from 'url') */
@@ -190,7 +191,7 @@ void url_start_async(OutgoingWebRequest *request)
 	if (request->transfer_timeout == 0)
 		request->transfer_timeout = DOWNLOAD_TRANSFER_TIMEOUT;
 	if (request->max_size <= 0)
-		request->max_size = DOWNLOAD_MAX_SIZE;
+		request->max_size = request->store_in_file ? DOWNLOAD_MAX_SIZE_FILE_BACKED : DOWNLOAD_MAX_SIZE_MEMORY_BACKED;
 
 	handle = safe_alloc(sizeof(Download));
 	handle->download_started = TStime();
@@ -879,7 +880,15 @@ int https_handle_response_body(Download *handle, char *readbuf, int pktsize)
 			https_handle_response_body_memory(handle, readbuf, pktsize);
 		}
 		else if (handle->file_fd)
+		{
+			if (handle->bytes_written_to_file + pktsize > handle->request->max_size)
+			{
+				https_cancel(handle, "Response too large (maximum: %lld bytes)", handle->request->max_size);
+				return 0; /* handle freed */
+			}
 			fwrite(readbuf, 1, pktsize, handle->file_fd);
+			handle->bytes_written_to_file += pktsize;
+		}
 		return 1;
 	}
 
@@ -917,7 +926,16 @@ int https_handle_response_body(Download *handle, char *readbuf, int pktsize)
 				https_handle_response_body_memory(handle, buf, eat);
 			}
 			else if (handle->file_fd)
+			{
+				if (handle->bytes_written_to_file + eat > handle->request->max_size)
+				{
+					https_cancel(handle, "Response too large (maximum: %lld bytes)", handle->request->max_size);
+					safe_free(free_this_buffer);
+					return 0; /* handle freed */
+				}
 				fwrite(buf, 1, eat, handle->file_fd);
+				handle->bytes_written_to_file += eat;
+			}
 			n -= eat;
 			buf += eat;
 			handle->chunk_remaining -= eat;