diff --git a/doc/RELEASE-NOTES.md b/doc/RELEASE-NOTES.md index 51c7db6ab..9d05d346c 100644 --- a/doc/RELEASE-NOTES.md +++ b/doc/RELEASE-NOTES.md @@ -4,13 +4,19 @@ This is the git version (development version) for future 6.1.2. This is work in progress and may not be a stable version. ### Enhancements: -* Optional [Central Spamfilter](https://www.unrealircd.org/docs/Central_spamfilter): - This will fetch and refresh spamfilter rules every 30 minutes from unrealircd.org. - This feature is not enabled default. Use `set { central-spamfitler { enabled yes; } }` to enable * [spamfilter { } block](https://www.unrealircd.org/docs/Spamfilter_block) improvements: * The `action` item now supports multiple actions * A new action is setting a TAG on a user, or increasing the value of a TAG * A new option `rule` with minimal 'if'-like preconditions and functions + * A new option `report` to call a spamreport block, see next. +* A new [spamreport { } block](https://www.unrealircd.org/docs/Spamreport_block): + * This can do a HTTP(S) call to services like DroneBL to report spam hits, + so they can blacklist the IP address and other users on IRC can benefit. +* Optional [Central Spamfilter](https://www.unrealircd.org/docs/Central_spamfilter): + This will fetch and refresh spamfilter rules every 30 minutes from unrealircd.org. + This feature is not enabled default. Use `set { central-spamfitler { enabled yes; } }` + to enable. At the moment this does not contain rules yet, but it will be + at a later point. * [set::spamfilter::utf8](https://www.unrealircd.org/docs/Set_block#set::spamfilter::utf8) is now on by default: * This means you can safely use UTF8 characters in like `[]` in regex. @@ -24,7 +30,8 @@ in progress and may not be a stable version. * You can turn it off via: `set { spamfilter { utf8 no; } }` * The module [antimixedutf8](https://www.unrealircd.org/docs/Set_block#set::antimixedutf8) now recognizes more code block transitions and should do a better job at - catching mixed UTF8 spam. + catching mixed UTF8 spam. It may also have more false positives. + This module is (still) not loaded by default. * You can restrict includes to only contain certain blocks, the style is: ``` include "some-file-or-url" { restrict-config { name-of-block; name-of-block2; } } diff --git a/include/h.h b/include/h.h index 0ebd7a183..7cee49fae 100644 --- a/include/h.h +++ b/include/h.h @@ -1388,7 +1388,7 @@ extern int url_is_valid(const char *); extern const char *displayurl(const char *url); extern char *url_getfilename(const char *url); extern void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects); -extern void url_start_async(const char *url, HttpMethod http_method, const char *body, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects); +extern void url_start_async(const char *url, HttpMethod http_method, const char *body, NameValuePrioList *request_headers, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects); extern void url_init(void); extern void url_cancel_handle_by_callback_data(void *ptr); extern EVENT(url_socket_timeout); diff --git a/include/struct.h b/include/struct.h index 79a5aac08..4243a0312 100644 --- a/include/struct.h +++ b/include/struct.h @@ -2602,7 +2602,9 @@ typedef enum JsonRpcError { /** Valid character for variable names in logging engine buildlogstring and also in buildvarstring* */ #define validvarcharacter(x) (isalnum((x)) || ((x) == '_')) -#define BUILDVARSTRING_URLENCODE 0x1 +#define BUILDVARSTRING_URLENCODE 0x1 +#define BUILDVARSTRING_XML 0x2 +#define BUILDVARSTRING_UNKNOWN_VAR_IS_EMPTY 0x4 #endif /* __struct_include__ */ diff --git a/src/misc.c b/src/misc.c index 86ee5c035..b992756af 100644 --- a/src/misc.c +++ b/src/misc.c @@ -3050,4 +3050,13 @@ int valid_spamfilter_id(const char *s) void download_complete_dontcare(const char *url, const char *file, const char *memory, int memory_len, const char *errorbuf, int cached, void *ptr) { +#ifdef DEBUGMODE + if (memory) + { + unreal_log(ULOG_DEBUG, "url", "DEBUG_URL_RESPONSE", NULL, + "Response for '$url': $response", + log_data_string("url", url), + log_data_string("response", memory)); + } +#endif } diff --git a/src/modules/spamreport.c b/src/modules/spamreport.c index 1f3ce1dd2..6e360eecc 100644 --- a/src/modules/spamreport.c +++ b/src/modules/spamreport.c @@ -110,6 +110,8 @@ int tkl_config_test_spamreport(ConfigFile *cf, ConfigEntry *ce, int type, int *e has_dronebl_rpckey = 1; else if (!strcmp(cepp->name, "type")) has_dronebl_type = 1; + else if (!strcmp(cepp->name, "staging")) + ; } } else if (!cep->value) @@ -256,7 +258,14 @@ int tkl_config_run_spamreport(ConfigFile *cf, ConfigEntry *ce, int type) else if (!strcmp(cep->name, "parameters")) { for (cepp = cep->items; cepp; cepp = cepp->next) + { + if (!strcmp(cepp->name, "staging")) + { + if (cepp->value && config_checkval(cepp->value, CFG_YESNO)==0) + continue; /* skip on 'staging no;' */ + } add_nvplist(&s->parameters, 0, cepp->name, cepp->value); + } } else if (!strcmp(cep->name, "except")) { @@ -310,6 +319,7 @@ int _spamreport(Client *client, const char *ip, NameValuePrioList *details, cons char bodybuf[512]; char *url = NULL; char *body = NULL; + NameValuePrioList *headers = NULL; if (!spamreport_block) { @@ -332,7 +342,7 @@ int _spamreport(Client *client, const char *ip, NameValuePrioList *details, cons NameValuePrioList *list = NULL; list = duplicate_nvplist(details); add_nvplist(&list, -1, "ip", ip); - buildvarstring_nvp(s->url, urlbuf, sizeof(urlbuf), list, BUILDVARSTRING_URLENCODE); + buildvarstring_nvp(s->url, urlbuf, sizeof(urlbuf), list, BUILDVARSTRING_URLENCODE|BUILDVARSTRING_UNKNOWN_VAR_IS_EMPTY); url = urlbuf; safe_free_nvplist(list); if (s->http_method == HTTP_METHOD_POST) @@ -346,11 +356,18 @@ int _spamreport(Client *client, const char *ip, NameValuePrioList *details, cons { NameValuePrioList *list = NULL; NameValuePrioList *list2 = NULL; - url = "https://www.unrealircd.org/spamreport.php"; + char fmtstring[512]; + url = "https://dronebl.org/rpc2"; list = duplicate_nvplist(details); duplicate_nvplist_append(s->parameters, &list); add_nvplist(&list, -1, "ip", ip); - buildvarstring_nvp("rpckey=$rpckey&action=add&ip=$ip&type=$type", bodybuf, sizeof(bodybuf), list, BUILDVARSTRING_URLENCODE); + snprintf(fmtstring, sizeof(fmtstring), + "\n" + "\n" + " \n" + "\n", + find_nvplist(s->parameters, "staging") ? " staging='1'" : ""); + buildvarstring_nvp(fmtstring, bodybuf, sizeof(bodybuf), list, BUILDVARSTRING_XML|BUILDVARSTRING_UNKNOWN_VAR_IS_EMPTY); body = bodybuf; safe_free_nvplist(list); // frees all the duplicated lists } else @@ -362,7 +379,8 @@ int _spamreport(Client *client, const char *ip, NameValuePrioList *details, cons log_data_string("url", url), log_data_string("body", (body ? body : ""))); #endif - url_start_async(url, s->http_method, body, 0, 0, download_complete_dontcare, NULL, url, 3); + url_start_async(url, s->http_method, body, headers, 0, 0, download_complete_dontcare, NULL, url, 3); + safe_free_nvplist(headers); return 1; } diff --git a/src/support.c b/src/support.c index 0364e387a..ec8c680a0 100644 --- a/src/support.c +++ b/src/support.c @@ -1354,6 +1354,59 @@ void buildvarstring(const char *inbuf, char *outbuf, size_t len, const char *nam safe_free_nvplist(list); } +char *xmlescape(const char *i, char *buf, int bufsize) +{ + char *o = buf; + + for (; *i; i++) + { + if (*i == '"') + { + if (bufsize <= 6) + break; + strcpy(o, """); + o += 6; + } else + if (*i == '\'') + { + if (bufsize <= 6) + break; + strcpy(o, "'"); + o += 6; + } else + if (*i == '<') + { + if (bufsize <= 4) + break; + strcpy(o, "<"); + o += 4; + } else + if (*i == '>') + { + if (bufsize <= 4) + break; + strcpy(o, ">"); + o += 4; + } else + if (*i == '&') + { + if (bufsize <= 5) + break; + strcpy(o, "&"); + o += 5; + } else + { + if (bufsize <= 1) + break; + *o++ = *i; + } + } + + if (bufsize >= 0) + *o = '\0'; + + return buf; +} /** Build a string and replace $variables where needed. * See src/modules/blacklist.c for an example. * @param inbuf The input string @@ -1406,6 +1459,8 @@ void buildvarstring_nvp(const char *inbuf, char *outbuf, size_t len, NameValuePr const char *output = n->value; if (flags & BUILDVARSTRING_URLENCODE) output = urlencode(output, outputbuf, sizeof(outputbuf)); + if (flags & BUILDVARSTRING_XML) + output = xmlescape(output, outputbuf, sizeof(outputbuf)); strlcpy(o, output, left); left -= strlen(output); /* may become <0 */ if (left <= 0) @@ -1414,9 +1469,16 @@ void buildvarstring_nvp(const char *inbuf, char *outbuf, size_t len, NameValuePr } } else { - /* variable name does not exist -- treat literal */ - i--; - goto literal; + /* variable name does not exist */ + if (flags & BUILDVARSTRING_UNKNOWN_VAR_IS_EMPTY) + { + /* stays empty, so '' in the result */ + } else + { + /* treat as literal value, so eg '$xyz' ends up in the result */ + i--; + goto literal; + } } /* value written. we're done. */ diff --git a/src/url_curl.c b/src/url_curl.c index b2af8ec5e..d73ba78fe 100644 --- a/src/url_curl.c +++ b/src/url_curl.c @@ -40,6 +40,8 @@ struct Download char *url; /*< must be free()d by url_do_transfers_async() */ HttpMethod http_method; char *body; + NameValuePrioList *request_headers; + struct curl_slist *request_headers_curl; char errorbuf[CURL_ERROR_SIZE]; time_t cachetime; char *memory_data; /**< Memory for writing response (otherwise NULL) */ @@ -60,6 +62,9 @@ void url_free_handle(Download *handle) safe_free(handle->filename); safe_free(handle->url); safe_free(handle->body); + safe_free_nvplist(handle->request_headers); + if (handle->request_headers_curl) + curl_slist_free_all(handle->request_headers_curl); safe_free(handle); } @@ -117,7 +122,7 @@ static size_t do_download_file(void *ptr, size_t size, size_t nmemb, void *strea static size_t do_download_memory(void *ptr, size_t size, size_t nmemb, void *stream) { // DUPLICATE CODE: same as src/url_unreal.c, well.. sortof - Download *handle = (Download *)ptr; + Download *handle = (Download *)stream; int write_sz = size * nmemb; int size_required = handle->memory_data_len + write_sz; @@ -134,11 +139,13 @@ static size_t do_download_memory(void *ptr, size_t size, size_t nmemb, void *str return 0; } handle->memory_data = newptr; + handle->memory_data_allocated = newsize; /* fill rest with zeroes, yeah.. no trust! ;D */ memset(handle->memory_data + handle->memory_data_len, 0, handle->memory_data_allocated - handle->memory_data_len); } memcpy(handle->memory_data + handle->memory_data_len, ptr, write_sz); + handle->memory_data_len += write_sz; handle->memory_data[handle->memory_data_len] = '\0'; return write_sz; } @@ -310,10 +317,10 @@ void url_init(void) */ void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects) { - url_start_async(url, HTTP_METHOD_GET, NULL, 1, cachetime, callback, callback_data, original_url, maxredirects); + url_start_async(url, HTTP_METHOD_GET, NULL, NULL, 1, cachetime, callback, callback_data, original_url, maxredirects); } -void url_start_async(const char *url, HttpMethod http_method, const char *body, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects) +void url_start_async(const char *url, HttpMethod http_method, const char *body, NameValuePrioList *request_headers, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects) { static char errorbuf[CURL_ERROR_SIZE]; char user_agent[256]; @@ -384,6 +391,24 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body, curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, body); } } + + if (request_headers) + { + NameValuePrioList *n; + char buf[512]; + + handle->request_headers = duplicate_nvplist(request_headers); + for (n = request_headers; n; n = n->next) + { + if (n->value) + snprintf(buf, sizeof(buf), "%s: %s", n->name, n->value); + else + snprintf(buf, sizeof(buf), "%s:", n->name); + handle->request_headers_curl = curl_slist_append(handle->request_headers_curl, buf); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, handle->request_headers_curl); + } + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); set_curl_tls_options(curl); memset(handle->errorbuf, 0, CURL_ERROR_SIZE); diff --git a/src/url_unreal.c b/src/url_unreal.c index 1cffa1018..6ec552122 100644 --- a/src/url_unreal.c +++ b/src/url_unreal.c @@ -36,6 +36,7 @@ struct Download char *url; /**< must be free()d by url_do_transfers_async() */ HttpMethod http_method; char *body; + NameValuePrioList *request_headers; int store_in_file; FILE *file_fd; /**< File open for writing (otherwise NULL) */ char *filename; @@ -105,6 +106,7 @@ void url_free_handle(Download *handle) safe_free(handle->filename); safe_free(handle->memory_data); safe_free(handle->body); + safe_free_nvplist(handle->request_headers); safe_free(handle->hostname); safe_free(handle->username); safe_free(handle->password); @@ -147,10 +149,10 @@ void https_cancel(Download *handle, FORMAT_STRING(const char *pattern), ...) void download_file_async(const char *url, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects) { - url_start_async(url, HTTP_METHOD_GET, NULL, 1, cachetime, callback, callback_data, original_url, maxredirects); + url_start_async(url, HTTP_METHOD_GET, NULL, NULL, 1, cachetime, callback, callback_data, original_url, maxredirects); } -void url_start_async(const char *url, HttpMethod http_method, const char *body, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects) +void url_start_async(const char *url, HttpMethod http_method, const char *body, NameValuePrioList *request_headers, int store_in_file, time_t cachetime, vFP callback, void *callback_data, char *original_url, int maxredirects) { char *file; const char *filename; @@ -209,8 +211,13 @@ void url_start_async(const char *url, HttpMethod http_method, const char *body, safe_strdup(handle->filename, tmp); safe_free(file); + } else { + handle->memory_data_allocated = URL_MEMORY_BACKED_CHUNK_SIZE; + handle->memory_data = safe_alloc(URL_MEMORY_BACKED_CHUNK_SIZE); } + if (request_headers) + handle->request_headers = duplicate_nvplist(request_headers); // todo: allocate handle, select en weetikt allemaal // add to some global struct linkedlist, for timeouts @@ -786,11 +793,13 @@ int https_handle_response_body_memory(Download *handle, const char *ptr, int wri return 0; } handle->memory_data = newptr; + handle->memory_data_allocated = newsize; /* fill rest with zeroes, yeah.. no trust! ;D */ memset(handle->memory_data + handle->memory_data_len, 0, handle->memory_data_allocated - handle->memory_data_len); } memcpy(handle->memory_data + handle->memory_data_len, ptr, write_sz); + handle->memory_data_len += write_sz; handle->memory_data[handle->memory_data_len] = '\0'; return write_sz; } @@ -965,7 +974,8 @@ void https_redirect(Download *handle) if (handle->callback) { /* If still an outstanding request (not cancelled), follow the redirect.. */ - url_start_async(handle->redirect_new_location, handle->http_method, handle->body, handle->store_in_file, + url_start_async(handle->redirect_new_location, handle->http_method, handle->body, + handle->request_headers, handle->store_in_file, handle->cachetime, handle->callback, handle->callback_data, handle->url, handle->redirects_remaining); }