Reporting spam to DroneBL now works, see https://www.unrealircd.org/docs/Spamreport_block

This uses the RPC2 API. Tested with staging.

Note that there are likely some bugs here or there, like memory leaks,
but the functionality is there.

Also still need to implement various stuff, including spamreport::rate-limit

doc/RELEASE-NOTES.md

@@ -4,13 +4,19 @@ This is the git version (development version) for future 6.1.2. This is work
 in progress and may not be a stable version.
 
 ### Enhancements:
-* Optional [Central Spamfilter](https://www.unrealircd.org/docs/Central_spamfilter):
-  This will fetch and refresh spamfilter rules every 30 minutes from unrealircd.org. 
-  This feature is not enabled default. Use `set { central-spamfitler { enabled yes; } }` to enable
 * [spamfilter { } block](https://www.unrealircd.org/docs/Spamfilter_block) improvements:
   * The `action` item now supports multiple actions
   * A new action is setting a TAG on a user, or increasing the value of a TAG
   * A new option `rule` with minimal 'if'-like preconditions and functions
+  * A new option `report` to call a spamreport block, see next.
+* A new [spamreport { } block](https://www.unrealircd.org/docs/Spamreport_block):
+  * This can do a HTTP(S) call to services like DroneBL to report spam hits,
+    so they can blacklist the IP address and other users on IRC can benefit.
+* Optional [Central Spamfilter](https://www.unrealircd.org/docs/Central_spamfilter):
+  This will fetch and refresh spamfilter rules every 30 minutes from unrealircd.org. 
+  This feature is not enabled default. Use `set { central-spamfitler { enabled yes; } }`
+  to enable. At the moment this does not contain rules yet, but it will be
+  at a later point.
 * [set::spamfilter::utf8](https://www.unrealircd.org/docs/Set_block#set::spamfilter::utf8)
   is now on by default:
   * This means you can safely use UTF8 characters in like `[]` in regex.
@@ -24,7 +30,8 @@ in progress and may not be a stable version.
   * You can turn it off via: `set { spamfilter { utf8 no; } }`
 * The module [antimixedutf8](https://www.unrealircd.org/docs/Set_block#set::antimixedutf8)
   now recognizes more code block transitions and should do a better job at
-  catching mixed UTF8 spam.
+  catching mixed UTF8 spam. It may also have more false positives.
+  This module is (still) not loaded by default.
 * You can restrict includes to only contain certain blocks, the style is:
   ```
   include "some-file-or-url" { restrict-config { name-of-block; name-of-block2; } }

include/h.h

@@ -1388,7 +1388,7 @@ extern int url_is_valid(const char *);
 extern const char *displayurl(const char *url);
 extern char *url_getfilename(const char *url);
 extern void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects);
-extern void url_start_async(const char *url, HttpMethod http_method, const char *body, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects);
+extern void url_start_async(const char *url, HttpMethod http_method, const char *body, NameValuePrioList *request_headers, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects);
 extern void url_init(void);
 extern void url_cancel_handle_by_callback_data(void *ptr);
 extern EVENT(url_socket_timeout);

include/struct.h

@@ -2602,7 +2602,9 @@ typedef enum JsonRpcError {
 /** Valid character for variable names in logging engine buildlogstring and also in buildvarstring* */
 #define validvarcharacter(x)    (isalnum((x)) || ((x) == '_'))
 
-#define BUILDVARSTRING_URLENCODE	0x1
+#define BUILDVARSTRING_URLENCODE		0x1
+#define BUILDVARSTRING_XML			0x2
+#define BUILDVARSTRING_UNKNOWN_VAR_IS_EMPTY	0x4
 
 #endif /* __struct_include__ */
 

src/misc.c

@@ -3050,4 +3050,13 @@ int valid_spamfilter_id(const char *s)
 
 void download_complete_dontcare(const char *url, const char *file, const char *memory, int memory_len, const char *errorbuf, int cached, void *ptr)
 {
+#ifdef DEBUGMODE
+	if (memory)
+	{
+		unreal_log(ULOG_DEBUG, "url", "DEBUG_URL_RESPONSE", NULL,
+		           "Response for '$url': $response",
+		           log_data_string("url", url),
+		           log_data_string("response", memory));
+	}
+#endif
 }

src/modules/spamreport.c

@@ -110,6 +110,8 @@ int tkl_config_test_spamreport(ConfigFile *cf, ConfigEntry *ce, int type, int *e
 					has_dronebl_rpckey = 1;
 				else if (!strcmp(cepp->name, "type"))
 					has_dronebl_type = 1;
+				else if (!strcmp(cepp->name, "staging"))
+					;
 			}
 		}
 		else if (!cep->value)
@@ -256,7 +258,14 @@ int tkl_config_run_spamreport(ConfigFile *cf, ConfigEntry *ce, int type)
 		else if (!strcmp(cep->name, "parameters"))
 		{
 			for (cepp = cep->items; cepp; cepp = cepp->next)
+			{
+				if (!strcmp(cepp->name, "staging"))
+				{
+					if (cepp->value && config_checkval(cepp->value, CFG_YESNO)==0)
+						continue; /* skip on 'staging no;' */
+				}
 				add_nvplist(&s->parameters, 0, cepp->name, cepp->value);
+			}
 		}
 		else if (!strcmp(cep->name, "except"))
 		{
@@ -310,6 +319,7 @@ int _spamreport(Client *client, const char *ip, NameValuePrioList *details, cons
 	char bodybuf[512];
 	char *url = NULL;
 	char *body = NULL;
+	NameValuePrioList *headers = NULL;
 
 	if (!spamreport_block)
 	{
@@ -332,7 +342,7 @@ int _spamreport(Client *client, const char *ip, NameValuePrioList *details, cons
 		NameValuePrioList *list = NULL;
 		list = duplicate_nvplist(details);
 		add_nvplist(&list, -1, "ip", ip);
-		buildvarstring_nvp(s->url, urlbuf, sizeof(urlbuf), list, BUILDVARSTRING_URLENCODE);
+		buildvarstring_nvp(s->url, urlbuf, sizeof(urlbuf), list, BUILDVARSTRING_URLENCODE|BUILDVARSTRING_UNKNOWN_VAR_IS_EMPTY);
 		url = urlbuf;
 		safe_free_nvplist(list);
 		if (s->http_method == HTTP_METHOD_POST)
@@ -346,11 +356,18 @@ int _spamreport(Client *client, const char *ip, NameValuePrioList *details, cons
 	{
 		NameValuePrioList *list = NULL;
 		NameValuePrioList *list2 = NULL;
-		url = "https://www.unrealircd.org/spamreport.php";
+		char fmtstring[512];
+		url = "https://dronebl.org/rpc2";
 		list = duplicate_nvplist(details);
 		duplicate_nvplist_append(s->parameters, &list);
 		add_nvplist(&list, -1, "ip", ip);
-		buildvarstring_nvp("rpckey=$rpckey&action=add&ip=$ip&type=$type", bodybuf, sizeof(bodybuf), list, BUILDVARSTRING_URLENCODE);
+		snprintf(fmtstring, sizeof(fmtstring),
+		         "<?xml version='1.0'?>\n"
+		         "<request key='$rpckey'%s>\n"
+		         " <add ip='$ip' type='$type' comment='$comment'>\n"
+		         "</request>\n",
+		         find_nvplist(s->parameters, "staging") ? " staging='1'" : "");
+		buildvarstring_nvp(fmtstring, bodybuf, sizeof(bodybuf), list, BUILDVARSTRING_XML|BUILDVARSTRING_UNKNOWN_VAR_IS_EMPTY);
 		body = bodybuf;
 		safe_free_nvplist(list); // frees all the duplicated lists
 	} else
@@ -362,7 +379,8 @@ int _spamreport(Client *client, const char *ip, NameValuePrioList *details, cons
 	           log_data_string("url", url),
 	           log_data_string("body", (body ? body : "")));
 #endif
-	url_start_async(url, s->http_method, body, 0, 0, download_complete_dontcare, NULL, url, 3);
+	url_start_async(url, s->http_method, body, headers, 0, 0, download_complete_dontcare, NULL, url, 3);
+	safe_free_nvplist(headers);
 	return 1;
 }
 

src/support.c

@@ -1354,6 +1354,59 @@ void buildvarstring(const char *inbuf, char *outbuf, size_t len, const char *nam
 	safe_free_nvplist(list);
 }
 
+char *xmlescape(const char *i, char *buf, int bufsize)
+{
+	char *o = buf;
+
+	for (; *i; i++)
+	{
+		if (*i == '"')
+		{
+			if (bufsize <= 6)
+				break;
+			strcpy(o, "&quot;");
+			o += 6;
+		} else
+		if (*i == '\'')
+		{
+			if (bufsize <= 6)
+				break;
+			strcpy(o, "&apos;");
+			o += 6;
+		} else
+		if (*i == '<')
+		{
+			if (bufsize <= 4)
+				break;
+			strcpy(o, "&lt;");
+			o += 4;
+		} else
+		if (*i == '>')
+		{
+			if (bufsize <= 4)
+				break;
+			strcpy(o, "&gt;");
+			o += 4;
+		} else
+		if (*i == '&')
+		{
+			if (bufsize <= 5)
+				break;
+			strcpy(o, "&amp;");
+			o += 5;
+		} else
+		{
+			if (bufsize <= 1)
+				break;
+			*o++ = *i;
+		}
+	}
+
+	if (bufsize >= 0)
+		*o = '\0';
+
+	return buf;
+}
 /** Build a string and replace $variables where needed.
  * See src/modules/blacklist.c for an example.
  * @param inbuf		The input string
@@ -1406,6 +1459,8 @@ void buildvarstring_nvp(const char *inbuf, char *outbuf, size_t len, NameValuePr
 					const char *output = n->value;
 					if (flags & BUILDVARSTRING_URLENCODE)
 						output = urlencode(output, outputbuf, sizeof(outputbuf));
+					if (flags & BUILDVARSTRING_XML)
+						output = xmlescape(output, outputbuf, sizeof(outputbuf));
 					strlcpy(o, output, left);
 					left -= strlen(output); /* may become <0 */
 					if (left <= 0)
@@ -1414,9 +1469,16 @@ void buildvarstring_nvp(const char *inbuf, char *outbuf, size_t len, NameValuePr
 				}
 			} else
 			{
-				/* variable name does not exist -- treat literal */
-				i--;
-				goto literal;
+				/* variable name does not exist */
+				if (flags & BUILDVARSTRING_UNKNOWN_VAR_IS_EMPTY)
+				{
+					/* stays empty, so '' in the result */
+				} else
+				{
+					/* treat as literal value, so eg '$xyz' ends up in the result */
+					i--;
+					goto literal;
+				}
 			}
 
 			/* value written. we're done. */

src/url_curl.c

@@ -40,6 +40,8 @@ struct Download
 	char *url; /*< must be free()d by url_do_transfers_async() */
 	HttpMethod http_method;
 	char *body;
+	NameValuePrioList *request_headers;
+	struct curl_slist *request_headers_curl;
 	char errorbuf[CURL_ERROR_SIZE];
 	time_t cachetime;
 	char *memory_data; /**< Memory for writing response (otherwise NULL) */
@@ -60,6 +62,9 @@ void url_free_handle(Download *handle)
 	safe_free(handle->filename);
 	safe_free(handle->url);
 	safe_free(handle->body);
+	safe_free_nvplist(handle->request_headers);
+	if (handle->request_headers_curl)
+		curl_slist_free_all(handle->request_headers_curl);
 	safe_free(handle);
 }
 
@@ -117,7 +122,7 @@ static size_t do_download_file(void *ptr, size_t size, size_t nmemb, void *strea
 static size_t do_download_memory(void *ptr, size_t size, size_t nmemb, void *stream)
 {
 	// DUPLICATE CODE: same as src/url_unreal.c, well.. sortof
-	Download *handle = (Download *)ptr;
+	Download *handle = (Download *)stream;
 	int write_sz = size * nmemb;
 	int size_required = handle->memory_data_len + write_sz;
 
@@ -134,11 +139,13 @@ static size_t do_download_memory(void *ptr, size_t size, size_t nmemb, void *str
 			return 0;
 		}
 		handle->memory_data = newptr;
+		handle->memory_data_allocated = newsize;
 		/* fill rest with zeroes, yeah.. no trust! ;D */
 		memset(handle->memory_data + handle->memory_data_len, 0, handle->memory_data_allocated - handle->memory_data_len);
 	}
 
 	memcpy(handle->memory_data + handle->memory_data_len, ptr, write_sz);
+	handle->memory_data_len += write_sz;
 	handle->memory_data[handle->memory_data_len] = '\0';
 	return write_sz;
 }
@@ -310,10 +317,10 @@ void url_init(void)
  */
 void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects)
 {
-	url_start_async(url, HTTP_METHOD_GET, NULL, 1, cachetime, callback, callback_data, original_url, maxredirects);
+	url_start_async(url, HTTP_METHOD_GET, NULL, NULL, 1, cachetime, callback, callback_data, original_url, maxredirects);
 }
 
-void url_start_async(const char *url, HttpMethod http_method, const char *body, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects)
+void url_start_async(const char *url, HttpMethod http_method, const char *body, NameValuePrioList *request_headers, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects)
 {
 	static char errorbuf[CURL_ERROR_SIZE];
 	char user_agent[256];
@@ -384,6 +391,24 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body,
 			curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, body);
 		}
 	}
+
+	if (request_headers)
+	{
+		NameValuePrioList *n;
+		char buf[512];
+
+		handle->request_headers = duplicate_nvplist(request_headers);
+		for (n = request_headers; n; n = n->next)
+		{
+			if (n->value)
+				snprintf(buf, sizeof(buf), "%s: %s", n->name, n->value);
+			else
+				snprintf(buf, sizeof(buf), "%s:", n->name);
+			handle->request_headers_curl = curl_slist_append(handle->request_headers_curl, buf);
+		}
+		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, handle->request_headers_curl);
+	}
+
 	curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
 	set_curl_tls_options(curl);
 	memset(handle->errorbuf, 0, CURL_ERROR_SIZE);

src/url_unreal.c

@@ -36,6 +36,7 @@ struct Download
 	char *url; /**< must be free()d by url_do_transfers_async() */
 	HttpMethod http_method;
 	char *body;
+	NameValuePrioList *request_headers;
 	int store_in_file;
 	FILE *file_fd;		/**< File open for writing (otherwise NULL) */
 	char *filename;
@@ -105,6 +106,7 @@ void url_free_handle(Download *handle)
 	safe_free(handle->filename);
 	safe_free(handle->memory_data);
 	safe_free(handle->body);
+	safe_free_nvplist(handle->request_headers);
 	safe_free(handle->hostname);
 	safe_free(handle->username);
 	safe_free(handle->password);
@@ -147,10 +149,10 @@ void https_cancel(Download *handle, FORMAT_STRING(const char *pattern), ...)
 
 void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects)
 {
-	url_start_async(url, HTTP_METHOD_GET, NULL, 1, cachetime, callback, callback_data, original_url, maxredirects);
+	url_start_async(url, HTTP_METHOD_GET, NULL, NULL, 1, cachetime, callback, callback_data, original_url, maxredirects);
 }
 
-void url_start_async(const char *url, HttpMethod http_method, const char *body, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects)
+void url_start_async(const char *url, HttpMethod http_method, const char *body, NameValuePrioList *request_headers, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects)
 {
 	char *file;
 	const char *filename;
@@ -209,8 +211,13 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body,
 
 		safe_strdup(handle->filename, tmp);
 		safe_free(file);
+	} else {
+		handle->memory_data_allocated = URL_MEMORY_BACKED_CHUNK_SIZE;
+		handle->memory_data = safe_alloc(URL_MEMORY_BACKED_CHUNK_SIZE);
 	}
 
+	if (request_headers)
+		handle->request_headers = duplicate_nvplist(request_headers);
 
 	// todo: allocate handle, select en weetikt allemaal
 	// add to some global struct linkedlist, for timeouts
@@ -786,11 +793,13 @@ int https_handle_response_body_memory(Download *handle, const char *ptr, int wri
 			return 0;
 		}
 		handle->memory_data = newptr;
+		handle->memory_data_allocated = newsize;
 		/* fill rest with zeroes, yeah.. no trust! ;D */
 		memset(handle->memory_data + handle->memory_data_len, 0, handle->memory_data_allocated - handle->memory_data_len);
 	}
 
 	memcpy(handle->memory_data + handle->memory_data_len, ptr, write_sz);
+	handle->memory_data_len += write_sz;
 	handle->memory_data[handle->memory_data_len] = '\0';
 	return write_sz;
 }
@@ -965,7 +974,8 @@ void https_redirect(Download *handle)
 	if (handle->callback)
 	{
 		/* If still an outstanding request (not cancelled), follow the redirect.. */
-		url_start_async(handle->redirect_new_location, handle->http_method, handle->body, handle->store_in_file,
+		url_start_async(handle->redirect_new_location, handle->http_method, handle->body,
+		                handle->request_headers, handle->store_in_file,
 		                handle->cachetime, handle->callback, handle->callback_data,
 				handle->url, handle->redirects_remaining);
 	}