Add set::utf8-only: if set to 'yes' this means all IRC traffic is UTF only.

See https://www.unrealircd.org/docs/Set_block#set::utf8-only and the
UTF8ONLY specification at https://ircv3.net/specs/extensions/utf8-only
for more information.

Reported by PeGaSuS, who reported it based on a #unreal-support message
from uMut, who reported it based on a message from itsonlybinary.
This closes https://bugs.unrealircd.org/view.php?id=6458

This feature still needs to go through our internal tests.

doc/RELEASE-NOTES.md

@@ -5,6 +5,10 @@ This is the git version (development version) for future UnrealIRCd 6.2.1.
 This is work in progress and may not always be a stable version.
 
 ### Enhancements:
+* Add [set::utf8-only](https://www.unrealircd.org/docs/Set_block#set::utf8-only):
+  setting this to `yes` means all IRC traffic is UTF8 only. See the setting
+  and the [`UTF8ONLY`](https://ircv3.net/specs/extensions/utf8-only)
+  specification for more details.
 * Add `server-port` to the [security-group block](https://www.unrealircd.org/docs/Security-group_block)
   and [mask items](https://www.unrealircd.org/docs/Mask_item). And
   the `server_port()` function in [Crule](https://www.unrealircd.org/docs/Crule).

doc/conf/modules.default.conf

@@ -300,6 +300,7 @@ loadmodule "spamreport"; /* Spam reporting to a blacklist */
 loadmodule "crule"; /* Rules in spamfilter::rule and deny link::rule */
 loadmodule "maxperip"; /* allow::maxperip restrictions */
 loadmodule "utf8functions"; /* Various UTF8 helper functions */
+loadmodule "utf8only"; /* UTF8ONLY via set::utf8-only */
 loadmodule "portinfo"; /* storing local_port and server_port of users */
 
 loadmodule "geoip_classic";

include/dynconf.h

@@ -188,6 +188,7 @@ struct Configuration {
 	int dns_dnsbl_timeout;
 	int dns_dnsbl_retry;
 	int send_isupport_updates;
+	int utf8_only;
 };
 
 extern MODVAR Configuration iConf;
@@ -289,6 +290,8 @@ extern MODVAR BestPractices bestpractices;
 
 #define UHNAMES_ENABLED	iConf.uhnames
 
+#define UTF8ONLY	iConf.utf8_only
+
 /** Used for testing the set { } block configuration.
  * It tests if a setting is present and is also used for duplicate checking.
  */

include/h.h

@@ -1212,6 +1212,7 @@ extern NameList *find_name_list_integer(NameList *list, int v);
 extern int minimum_msec_since_last_run(struct timeval *tv_old, long minimum);
 extern int unrl_utf8_validate(const char *str, const char **end);
 extern char *unrl_utf8_make_valid(const char *str, char *outputbuf, size_t outputbuflen, int strict_length_check);
+extern void utf8_valid_cutoff(char *msg, int *len);
 extern void utf8_test(void);
 extern MODVAR int non_utf8_nick_chars_in_use;
 extern void short_motd(Client *client);

src/modules/Makefile.in

@@ -85,7 +85,7 @@ MODULES= \
 	real-quit-reason.so \
 	spamreport.so crule.so \
 	central-api.so central-blocklist.so \
-	no-implicit-names.so maxperip.so utf8functions.so \
+	no-implicit-names.so maxperip.so utf8functions.so utf8only.so \
 	$(GEOIP_CLASSIC_OBJECTS) $(GEOIP_MAXMIND_OBJECTS)
 
 MODULEFLAGS=@MODULEFLAGS@

src/modules/clienttagdeny.c

@@ -1,5 +1,5 @@
 /*
- *   IRC - Internet Relay Chat, src/modules/echo-message.c
+ *   IRC - Internet Relay Chat, src/modules/clienttagdeny.c
  *   (C) 2020 k4be for The UnrealIRCd Team
  *
  *   See file AUTHORS in IRC package for additional names of

src/modules/utf8only.c

@@ -0,0 +1,178 @@
+/*
+ * UTF8ONLY ISUPPORT: Only allow UTF8 incoming and outgoing on IRC
+ * (C) Copyright 2025-.. Syzop and The UnrealIRCd Team
+ * License: GPLv2 or later
+ */
+
+#include "unrealircd.h"
+
+ModuleHeader MOD_HEADER
+  = {
+	"utf8only",
+	"1.0.0",
+	"only allow UTF8 traffic on IRC (UTF8ONLY)",
+	"UnrealIRCd Team",
+	"unrealircd-6",
+    };
+
+/* Forward declarations */
+int utf8only_config_test(ConfigFile *cf, ConfigEntry *ce, int type, int *errs);
+int utf8only_config_run(ConfigFile *cf, ConfigEntry *ce, int type);
+int utf8only_packet(Client *from, Client *to, Client *intended_to, char **msg, int *length);
+
+/* Variables */
+int utf8_only = 0;
+
+MOD_TEST()
+{
+	MARK_AS_OFFICIAL_MODULE(modinfo);
+	HookAdd(modinfo->handle, HOOKTYPE_CONFIGTEST, 0, utf8only_config_test);
+
+	return MOD_SUCCESS;
+}
+
+MOD_INIT()
+{
+	MARK_AS_OFFICIAL_MODULE(modinfo);
+	HookAdd(modinfo->handle, HOOKTYPE_CONFIGRUN, 0, utf8only_config_run);
+
+	return MOD_SUCCESS;
+}
+
+MOD_LOAD()
+{
+	/* You may wonder what this is... two variables for the same setting?
+	 * The thing is that iConf.utf8_only is external because it is also
+	 * used in src/send. But... if we only set iConf.utf8_only if
+	 * set::utf8-only is present then if it is first set to 1, and then
+	 * later after a config change is deleted, it won't be set to 0.
+	 * So we use this 'trick'.
+	 * The alternative would be to do an iConf.utf8_only = 0 in MOD_INIT
+	 * but then it is always zero for a short while, during rehashes,
+	 * which may or may not be an issue, depending on if traffic is
+	 * sent during a rehash (eg rehash output).
+	 */
+	iConf.utf8_only = utf8_only;
+
+	if (iConf.utf8_only)
+	{
+		ISupportAdd(modinfo->handle, "UTF8ONLY", NULL);
+		HookAdd(modinfo->handle, HOOKTYPE_PACKET, 0, utf8only_packet);
+	}
+
+	return MOD_SUCCESS;
+}
+
+MOD_UNLOAD()
+{
+	return MOD_SUCCESS;
+}
+
+int utf8only_config_test(ConfigFile *cf, ConfigEntry *ce, int type, int *errs)
+{
+	int errors = 0;
+
+	if (type != CONFIG_SET)
+		return 0;
+
+	if (ce && !strcmp(ce->name, "utf8-only"))
+	{
+		if (!ce->value)
+		{
+			config_error("%s:%d: set::utf8-only: no value", ce->file->filename, ce->line_number);
+			errors++;
+		}
+		*errs = errors;
+		return errors ? -1 : 1;
+	}
+	return 0;
+}
+
+int utf8only_config_run(ConfigFile *cf, ConfigEntry *ce, int type)
+{
+	if (type != CONFIG_SET)
+		return 0;
+
+	if (ce && !strcmp(ce->name, "utf8-only"))
+	{
+		iConf.utf8_only = utf8_only = config_checkval(ce->value, CFG_YESNO);
+		return 1;
+	}
+	return 0;
+}
+
+/* Get the command from a line from an IRC client.
+ * Returns "*" if not found (eg empty line, or malformed)
+ */
+const char *parse_get_command(const char *msg)
+{
+	const char *p = msg;
+	static char cmd[64];
+
+	/* Skip whitespace at beginning of the line */
+	for (; *p == ' '; p++);
+	/* Skip message tags (if any) */
+	if (*p == '@')
+	{
+		for (p = p + 1; *p != ' '; p++);
+		for (; *p == ' '; p++);
+	}
+
+	if (*p)
+	{
+		char *o = cmd;
+		int sz = sizeof(cmd) - 1;
+		for (; sz && *p && (*p != ' '); p++)
+		{
+			*o++ = *p;
+			sz--;
+		}
+		*o++ = '\0';
+	} else {
+		strlcpy(cmd, "*", sizeof(cmd));
+	}
+
+	return cmd;
+}
+
+int utf8only_packet(Client *from, Client *to, Client *intended_to, char **msg, int *length)
+{
+	static char buf[16384];
+
+	if (IsMe(from))
+		return 0;
+
+	if (IsServer(from) || IsUnknown(from))
+	{
+		/* For servers we convert the contents just in case,
+		 * since it may be 'poisoned' with non-UTF8 stuff
+		 * (eg if remote server does not enable UTF8ONLY).
+		 * For unknown connections we do this too, because
+		 * otherwise things like 'USER' with invalid UTF8
+		 * may not be allowed through, which is a bit
+		 * confusing to the user.
+		 */
+		char *ret = unrl_utf8_make_valid(*msg, buf, sizeof(buf), 0);
+		if (ret != *msg)
+		{
+			*msg = ret;
+			*length = strlen(*msg); /* Needs to be recalculated */
+		}
+		return 0;
+	} else if (IsUser(from)) {
+		/* Connected user: be strict */
+		if (!unrl_utf8_validate(*msg, NULL))
+		{
+			const char *cmd = parse_get_command(*msg);
+			if (HasCapability(from, "standard-replies"))
+				sendto_one(from, NULL, ":%s FAIL %s INVALID_UTF8 :Message rejected, your IRC client MUST use UTF-8 encoding on this network",
+				           me.name, cmd);
+			else
+				sendnumeric(from, ERR_CANNOTDOCOMMAND, cmd, "Message rejected, your IRC client MUST use UTF-8 encoding on this network");
+			*msg = NULL;
+			*length = 0;
+			return 0;
+		}
+	}
+	return 0;
+}

src/send.c

@@ -57,12 +57,10 @@ static void linecache_free(LineCache *cache);
 static void linecache_add(LineCache *cache, int line_opts, Client *to, const char *line, int linelen);
 static LineCacheLine *linecache_get(LineCache *cache, int line_opts, Client *to);
 
-#define ADD_CRLF(buf, len) { if (len > 510) len = 510; \
-                             buf[len++] = '\r'; buf[len++] = '\n'; buf[len] = '\0'; } while(0)
-
 /* These are two local (static) buffers used by the various send functions */
 static char sendbuf[MAXLINELENGTH];
 static char sendbuf2[MAXLINELENGTH];
+static char sendbuf3[MAXLINELENGTH];
 
 /** This is used to ensure no duplicate messages are sent
  * to the same server uplink/direction. In send functions
@@ -219,12 +217,41 @@ void vsendto_one(Client *to, MessageTag *mtags, const char *pattern, va_list vl)
 	}
 }
 
-/** Prepare a line for sendbufto_one() */
-static int sendbufto_one_prepare_line(Client *to, char *msg)
+/** Prepare a line for sendbufto_one().
+ * @param to	Client to send to
+ * @param input	Pointer to line to be sent.
+ * @notes The string '*input' can be changed or cut off (this is quite frequent).
+ * Or it may be replaced entirely, in which case 'input' will be set to a new string.
+ */
+static int sendbufto_one_prepare_line(Client *to, char **input)
 {
-	char *p = msg;
+	char *msg = *input;
+	char *p;
 	int len;
 
+	/* If we are in UTF8ONLY mode, we first convert the line to be sent
+	 * to valid UTF8. Of course, normally the line would not contain
+	 * invalid UTF8 to begin with (since we already reject it at the
+	 * input side from clients), but some external data may 'poison'
+	 * things like a MOTD or who knows what. Also, since we will be
+	 * cutting off strings, we need to take extra care in this function
+	 * as well when UTF8ONLY is enabled, since we may cut in the middle
+	 * of an UTF8 sequence.
+	 */
+	if (UTF8ONLY)
+	{
+		/* Note that last parameter strictlen must be 0, otherwise we might
+		 * end up cutting BIGLINES servers and also.. we already do all that
+		 * cutting of 512 etc below anyway, so no need to do double work.
+		 */
+		char *ret = unrl_utf8_make_valid(*input, sendbuf3, sizeof(sendbuf3), 0);
+		if (ret != *input)
+		{
+			/* Message had invalid UTF8 or was cut (eg 512 length restriction) */
+			*input = msg = ret;
+		}
+	}
+
 	if (*msg == '@')
 	{
 		/* The message includes one or more message tags:
@@ -249,7 +276,10 @@ static int sendbufto_one_prepare_line(Client *to, char *msg)
 			return 0;
 		}
 		p++; /* skip space character */
+	} else {
+		p = msg;
 	}
+
 	len = strlen(p);
 	if (!len || (p[len - 1] != '\n'))
 	{
@@ -258,6 +288,8 @@ static int sendbufto_one_prepare_line(Client *to, char *msg)
 			/* Normal case */
 			if (len > 510)
 				len = 510;
+			if (UTF8ONLY)
+				utf8_valid_cutoff(msg, &len);
 			p[len++] = '\r';
 			p[len++] = '\n';
 			p[len] = '\0';
@@ -271,6 +303,8 @@ static int sendbufto_one_prepare_line(Client *to, char *msg)
 			if ((p - msg) + len > MAXLINELENGTH-3)
 			{
 				len = MAXLINELENGTH-3;
+				if (UTF8ONLY)
+					utf8_valid_cutoff(msg, &len);
 				msg[len++] = '\r';
 				msg[len++] = '\n';
 				msg[len] = '\0';
@@ -282,6 +316,17 @@ static int sendbufto_one_prepare_line(Client *to, char *msg)
 		}
 	}
 
+#ifdef DEBUGMODE
+	if (UTF8ONLY)
+	{
+		/* if validation fails it means conversion above resulted
+		 * in an invalid UTF8 string.
+		 */
+		if (!unrl_utf8_validate(msg, NULL))
+			abort();
+	}
+#endif
+
 	/* Return length, that is:
 	 * p-msg = message tag len (can be 0)
 	 * len = normal IRC message including \r\n
@@ -333,7 +378,7 @@ void sendbufto_one(Client *to, char *msg, unsigned int quick)
 	 */
 	if (!quick)
 	{
-		len = sendbufto_one_prepare_line(to, msg);
+		len = sendbufto_one_prepare_line(to, &msg);
 		if (len == 0)
 			return;
 	} else {
@@ -956,7 +1001,16 @@ static int vmakebuf_local_withprefix(char *buf, size_t buflen, Client *from, con
 	}
 
 	len = strlen(buf);
-	ADD_CRLF(buf, len);
+	if (len > 510)
+	{
+		len = 510;
+		if (UTF8ONLY)
+			utf8_valid_cutoff(buf, &len);
+	}
+	buf[len++] = '\r';
+	buf[len++] = '\n';
+	buf[len] = '\0';
+
 	return len;
 }
 
@@ -1103,15 +1157,17 @@ static void vsendto_prefix_one_cached(LineCache *cache, int line_opts, Client *t
 	if (BadPtr(mtags_str))
 	{
 		/* Simple message without message tags */
-		len = sendbufto_one_prepare_line(to, sendbuf);
-		linecache_add(cache, line_opts, to, sendbuf, len);
-		sendbufto_one(to, sendbuf, len);
+		char *out = sendbuf;
+		len = sendbufto_one_prepare_line(to, &out);
+		linecache_add(cache, line_opts, to, out, len);
+		sendbufto_one(to, out, len);
 	} else {
 		/* Message tags need to be prepended */
+		char *out = sendbuf2;
 		snprintf(sendbuf2, sizeof(sendbuf2)-3, "@%s %s", mtags_str, sendbuf);
-		len = sendbufto_one_prepare_line(to, sendbuf2);
-		linecache_add(cache, line_opts, to, sendbuf2, len);
-		sendbufto_one(to, sendbuf2, 0);
+		len = sendbufto_one_prepare_line(to, &out);
+		linecache_add(cache, line_opts, to, out, len);
+		sendbufto_one(to, out, 0);
 	}
 }
 

src/utf8.c

@@ -266,6 +266,27 @@ char *unrl_utf8_make_valid(const char *str, char *outputbuf, size_t outputbuflen
 	return outputbuf;
 }
 
+/* Cut off string 'msg' at 'len' because it is >len size, while
+ * making sure that it still ends with proper UTF8 (eg not some
+ * cut in the middle of a multibyte UTF8 sequence).
+ * @param msg	The message to be edited
+ * @param len	Pointer to length of message. The caller hereby indicates that
+ *		'len+1' still contains valid data and it wants to cut off at 'len'.
+ * This function will modify 'msg' to be cut off after 'len' characters or
+ * slightly earlier if it was in the middle of an UTF8 sequence. In case of
+ * the latter, the 'len' will be adjusted to be slightly lower.
+ */
+void utf8_valid_cutoff(char *msg, int *len)
+{
+	/* This deliberately starts reading at len+1 and then backtracks */
+	char *cut_at = unrl_utf8_find_prev_char(msg, msg + *len + 1);
+	if (cut_at)
+	{
+		*cut_at = '\0';
+		*len = cut_at - msg; /* bit silly we have to recalculate that, but okay */
+	}
+}
+
 /**************** END OF UTF8 HELPER FUNCTIONS *****************/
 
 /** This is just for internal testing */