Add initial version of CHATHISTORY command

from https://ircv3.net/specs/extensions/chathistory

Current status of the module in UnrealIRCd:
* A significant part of this is done and working
* Currently in modules.optional.conf to get test exposure,
  not yet loaded by default.
* CHATHISTORY subcommands implemented: BEFORE, AFTER, LATEST, AROUND
* It does not implement the subcommand "BETWEEN" yet
* It does not announce or recognize the (draft) CAP's yet
* It does not announce the ISUPPORT token CHATHISTORY=xx yet
* Testcases need to be written to validate everything
* There will be bugs, now, and also while implementing the rest
  in the days to come.

Makefile.windows

@@ -304,6 +304,7 @@ DLL_FILES=SRC/MODULES/CLOAK.DLL \
  SRC/MODULES/REQUIRE-MODULE.DLL \
  SRC/MODULES/IDENT_LOOKUP.DLL \
  SRC/MODULES/HISTORY.DLL \
+ SRC/MODULES/CHATHISTORY.DLL \
  SRC/MODULES/TARGETFLOODPROT.DLL \
  SRC/MODULES/TYPING-INDICATOR.DLL \
  SRC/MODULES/CLIENTTAGDENY.DLL
@@ -1106,6 +1107,9 @@ src/modules/ident_lookup.dll: src/modules/ident_lookup.c $(INCLUDES)
 src/modules/history.dll: src/modules/history.c $(INCLUDES)
 	$(CC) $(MODCFLAGS) /Fosrc/modules/ /Fesrc/modules/ src/modules/history.c $(MODLFLAGS)
 
+src/modules/chathistory.dll: src/modules/chathistory.c $(INCLUDES)
+	$(CC) $(MODCFLAGS) /Fosrc/modules/ /Fesrc/modules/ src/modules/chathistory.c $(MODLFLAGS)
+
 src/modules/targetfloodprot.dll: src/modules/targetfloodprot.c $(INCLUDES)
 	$(CC) $(MODCFLAGS) /Fosrc/modules/ /Fesrc/modules/ src/modules/targetfloodprot.c $(MODLFLAGS)
 

doc/conf/modules.optional.conf

@@ -175,3 +175,6 @@ set {
 		ban-time 4h; // For other types
 	}
 }
+
+// Currently incomplete and experimental:
+loadmodule "chathistory";

include/h.h

@@ -972,6 +972,7 @@ extern int history_destroy(char *object);
 extern int can_receive_history(Client *client);
 extern void history_send_result(Client *client, HistoryResult *r);
 extern void free_history_result(HistoryResult *r);
+extern void free_history_filter(HistoryFilter *f);
 extern void special_delayed_unloading(void);
 extern int write_int64(FILE *fd, uint64_t t);
 extern int write_int32(FILE *fd, uint32_t t);

include/modules.h

@@ -486,11 +486,27 @@ typedef struct {
 
 /** @} */
 
+/** Filter for history: the command / type of the request */
+typedef enum HistoryFilterCommand {
+        HFC_SIMPLE=1,		/**< Simple history request for lines / unixtime */
+        HFC_BEFORE=2,		/**< CHATHISTORY BEFORE */
+        HFC_AFTER=3,		/**< CHATHISTORY AFTER */
+        HFC_LATEST=4,		/**< CHATHISTORY LATEST */
+        HFC_AROUND=5,		/**< CHATHISTORY AROUND */
+        HFC_BETWEEN=6		/**< CHATHISTORY BETWEEN */
+} HistoryFilterCommand;
+
 /** Filter for history get requests */
 typedef struct HistoryFilter HistoryFilter;
 struct HistoryFilter {
-    int last_lines;
-    int last_seconds;
+        HistoryFilterCommand cmd;	/**< Filter command, one of HistoryFilterCommand */
+        int last_lines;			/**< Used by HFC_SIMPLE */
+        int last_seconds;		/**< Used by HFC_SIMPLE */
+        char *timestamp_a;		/**< First parameter of HFC_* (either this or msgid_a) */
+        char *msgid_a;			/**< First parameter of HFC_* (either this or timestamp_a) */
+        char *timestamp_b;		/**< Second parameter of HFC_BETWEEN (either this or msgid_b) */
+        char *msgid_b;			/**< Second parameter of HFC_BETWEEN (either this or timestamp_b) */
+        int limit;			/**< Maximum number of lines to return */
 };
 
 /** History log lines, used by HistoryResult among others */

src/api-history-backend.c

@@ -273,3 +273,12 @@ void history_send_result(Client *client, HistoryResult *r)
 	if (*batch)
 		sendto_one(client, NULL, ":%s BATCH -%s", me.name, batch);
 }
+
+void free_history_filter(HistoryFilter *f)
+{
+	safe_free(f->timestamp_a);
+	safe_free(f->msgid_a);
+	safe_free(f->timestamp_b);
+	safe_free(f->msgid_b);
+	safe_free(f);
+}

src/modules/Makefile.in

@@ -72,7 +72,7 @@ R_MODULES= \
 	message-ids.so plaintext-policy.so server-time.so sts.so \
 	echo-message.so userip-tag.so userhost-tag.so \
 	typing-indicator.so \
-	ident_lookup.so history.so \
+	ident_lookup.so history.so chathistory.so \
 	targetfloodprot.so clienttagdeny.so
 
 MODULES=cloak.so $(R_MODULES)
@@ -632,6 +632,10 @@ history.so: history.c $(INCLUDES)
 	$(CC) $(CFLAGS) $(MODULEFLAGS) -DDYNAMIC_LINKING \
 		-o history.so history.c
 
+chathistory.so: chathistory.c $(INCLUDES)
+	$(CC) $(CFLAGS) $(MODULEFLAGS) -DDYNAMIC_LINKING \
+		-o chathistory.so chathistory.c
+
 targetfloodprot.so: targetfloodprot.c $(INCLUDES)
 	$(CC) $(CFLAGS) $(MODULEFLAGS) -DDYNAMIC_LINKING \
 		-o targetfloodprot.so targetfloodprot.c

src/modules/chanmodes/history.c

@@ -580,6 +580,7 @@ int history_join(Client *client, Channel *channel, MessageTag *mtags, char *parv
 		HistoryFilter filter;
 		HistoryResult *r;
 		memset(&filter, 0, sizeof(filter));
+		filter.cmd = HFC_SIMPLE;
 		filter.last_lines = cfg.playback_on_join.lines;
 		filter.last_seconds = cfg.playback_on_join.time;
 		r = history_request(channel->chname, &filter);

src/modules/chathistory.c

@@ -0,0 +1,162 @@
+/* src/modules/chathistory.c - IRCv3 CHATHISTORY command.
+ * (C) Copyright 2021 Bram Matthys (Syzop) and the UnrealIRCd team
+ * License: GPLv2
+ */
+#include "unrealircd.h"
+
+ModuleHeader MOD_HEADER
+= {
+	"chathistory",
+	"1.0",
+	"IRCv3 CHATHISTORY command",
+	"UnrealIRCd Team",
+	"unrealircd-5",
+};
+
+/* Forward declarations */
+CMD_FUNC(cmd_chathistory);
+
+// TODO: change to 50 and move to config:
+#define CHATHISTORY_LIMIT 50
+
+MOD_INIT()
+{
+	HistoryBackendInfo hbi;
+
+	MARK_AS_OFFICIAL_MODULE(modinfo);
+	CommandAdd(modinfo->handle, "CHATHISTORY", cmd_chathistory, MAXPARA, CMD_USER);
+
+	return MOD_SUCCESS;
+}
+
+MOD_LOAD()
+{
+	return MOD_SUCCESS;
+}
+
+MOD_UNLOAD()
+{
+	return MOD_SUCCESS;
+}
+
+int chathistory_token(char *str, char *token, char **store)
+{
+	char *p = strchr(str, '=');
+	if (!p)
+		return 0;
+	*p = '\0'; // frag
+	if (!strcmp(str, token))
+	{
+		*p = '='; // restore
+		*store = strdup(p + 1); // can be \0
+		return 1;
+	}
+	*p = '='; // restore
+	return 0;
+}
+
+CMD_FUNC(cmd_chathistory)
+{
+	HistoryFilter *filter = NULL;
+	HistoryResult *r = NULL;
+	Channel *channel;
+
+	memset(&filter, 0, sizeof(filter));
+
+	if ((parc < 5) || BadPtr(parv[4]))
+	{
+		sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_PARAMS :Insufficient parameters", me.name);
+		return;
+	}
+
+	if (!HasCapability(client, "server-time"))
+	{
+		sendnotice(client, "Your IRC client does not support the 'server-time' capability");
+		sendnotice(client, "https://ircv3.net/specs/extensions/server-time-3.2.html");
+		sendnotice(client, "History request refused.");
+		return;
+	}
+
+	channel = find_channel(parv[2], NULL);
+	if (!channel || !IsMember(client, channel) || !has_channel_mode(channel, 'H'))
+	{
+		sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_TARGET %s %s :Messages could not be retrieved",
+			me.name, parv[1], parv[2]);
+		return;
+	}
+
+	filter = safe_alloc(sizeof(HistoryFilter));
+	/* Below this point, instead of 'return', use 'goto end', which takes care of the freeing of 'filter' and 'history' */
+
+	if (!strcmp(parv[1], "BEFORE"))
+	{
+		filter->cmd = HFC_BEFORE;
+		if (!chathistory_token(parv[3], "timestamp", &filter->timestamp_a) &&
+		    !chathistory_token(parv[3], "msgid", &filter->msgid_a))
+		{
+			sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_PARAMS %s %s :Invalid parameter, must be timestamp=xxx or msgid=xxx",
+				me.name, parv[1], parv[3]);
+			goto end;
+		}
+		filter->limit = atoi(parv[4]);
+	} else
+	if (!strcmp(parv[1], "AFTER"))
+	{
+		filter->cmd = HFC_AFTER;
+		if (!chathistory_token(parv[3], "timestamp", &filter->timestamp_a) &&
+		    !chathistory_token(parv[3], "msgid", &filter->msgid_a))
+		{
+			sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_PARAMS %s %s :Invalid parameter, must be timestamp=xxx or msgid=xxx",
+				me.name, parv[1], parv[3]);
+			goto end;
+		}
+		filter->limit = atoi(parv[4]);
+	} else
+	if (!strcmp(parv[1], "LATEST"))
+	{
+		filter->cmd = HFC_LATEST;
+		if (!chathistory_token(parv[3], "timestamp", &filter->timestamp_a) &&
+		    !chathistory_token(parv[3], "msgid", &filter->msgid_a) &&
+		    strcmp(parv[3], "*"))
+		{
+			sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_PARAMS %s %s :Invalid parameter, must be timestamp=xxx or msgid=xxx or *",
+				me.name, parv[1], parv[3]);
+			goto end;
+		}
+		filter->limit = atoi(parv[4]);
+	} else
+	if (!strcmp(parv[1], "AROUND"))
+	{
+		filter->cmd = HFC_AROUND;
+		if (!chathistory_token(parv[3], "timestamp", &filter->timestamp_a) &&
+		    !chathistory_token(parv[3], "msgid", &filter->msgid_a))
+		{
+			sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_PARAMS %s %s :Invalid parameter, must be timestamp=xxx or msgid=xxx",
+				me.name, parv[1], parv[3]);
+			goto end;
+		}
+		filter->limit = atoi(parv[4]);
+	} else {
+		sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_PARAMS %s :Invalid subcommand", me.name, parv[1]);
+		goto end;
+	}
+
+	if (filter->limit <= 0)
+	{
+		sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_PARAMS %s %d :Specified limit is =<0",
+			me.name, parv[1], filter->limit);
+		goto end;
+	}
+
+	if (filter->limit > CHATHISTORY_LIMIT)
+		filter->limit = CHATHISTORY_LIMIT;
+
+	if ((r = history_request(channel->chname, filter)))
+		history_send_result(client, r);
+
+end:
+	if (filter)
+		free_history_filter(filter);
+	if (r)
+		free_history_result(r);
+}

src/modules/history.c

@@ -122,6 +122,7 @@ CMD_FUNC(cmd_history)
 	}
 
 	memset(&filter, 0, sizeof(filter));
+	filter.cmd = HFC_SIMPLE;
 	filter.last_lines = lines;
 
 	if ((r = history_request(channel->chname, &filter)))

src/modules/history_backend_mem.c

@@ -593,19 +593,159 @@ HistoryLogLine *duplicate_log_line(HistoryLogLine *l)
 	return n;
 }
 
-HistoryResult *hbm_history_request(char *object, HistoryFilter *filter)
+/** Quickly append a new line 'n' to result 'r' */
+static void hbm_result_append_line(HistoryResult *r, HistoryLogLine *n)
+{
+	if (!r->log)
+	{
+		/* First item */
+		r->log = r->log_tail = n;
+	} else
+	{
+		/* Quick append to tail */
+		r->log_tail->next = n;
+		n->prev = r->log_tail;
+		r->log_tail = n; /* we are the new tail */
+	}
+}
+
+/** Quickly prepend a new line 'n' to result 'r' */
+static void hbm_result_prepend_line(HistoryResult *r, HistoryLogLine *n)
+{
+	if (!r->log)
+		r->log_tail = n;
+	AddListItem(n, r->log);
+}
+
+/** Put lines in HistoryResult that are after a certain msgid or
+ *  timestamp (excluding said msgid/timestamp).
+ * @param r		The history result set that we will use
+ * @param h		The history log object
+ * @param filter	The filter that applies
+ * @returns Number of lines written, note that this could be zero,
+ *          which is a perfectly valid result.
+ */
+static int hbm_return_after(HistoryResult *r, HistoryLogObject *h, HistoryFilter *filter)
+{
+	HistoryLogLine *l, *n;
+	int written = 0;
+	int started = 0;
+
+	for (l = h->head; l; l = l->next)
+	{
+		/* Not started yet? Check if this is the starting point... */
+		if (!started)
+		{
+			MessageTag *m;
+			if (filter->timestamp_a && ((m = find_mtag(l->mtags, "time"))) && (strcmp(m->value, filter->timestamp_a) > 0))
+			{
+				started = 1;
+			} else
+			if (filter->msgid_a && ((m = find_mtag(l->mtags, "msgid"))) && !strcmp(m->value, filter->msgid_a))
+			{
+				started = 1;
+				continue;
+			}
+		}
+		if (started)
+		{
+			// TODO: check for timestamp_b / timestamp_b which define a 'stop' boundary
+			n = duplicate_log_line(l);
+			hbm_result_append_line(r, n);
+			if (++written >= filter->limit)
+				break;
+		}
+	}
+
+	return written;
+}
+
+/** Put lines in HistoryResult that before after a certain msgid or
+ *  timestamp (excluding said msgid/timestamp).
+ * @param r		The history result set that we will use
+ * @param h		The history log object
+ * @param filter	The filter that applies
+ * @returns Number of lines written, note that this could be zero,
+ *          which is a perfectly valid result.
+ */
+static int hbm_return_before(HistoryResult *r, HistoryLogObject *h, HistoryFilter *filter)
+{
+	HistoryLogLine *l, *n;
+	int written = 0;
+	int started = 0;
+
+	for (l = h->tail; l; l = l->prev)
+	{
+		/* Not started yet? Check if this is the starting point... */
+		if (!started)
+		{
+			MessageTag *m;
+			if (filter->timestamp_a && ((m = find_mtag(l->mtags, "time"))) && (strcmp(m->value, filter->timestamp_a) < 0))
+			{
+				started = 1;
+			} else
+			if (filter->msgid_a && ((m = find_mtag(l->mtags, "msgid"))) && !strcmp(m->value, filter->msgid_a))
+			{
+				started = 1;
+				continue;
+			}
+		}
+		if (started)
+		{
+			// TODO: check for timestamp_b / timestamp_b which define a 'stop' boundary
+			n = duplicate_log_line(l);
+			hbm_result_prepend_line(r, n);
+			if (++written >= filter->limit)
+				break;
+		}
+	}
+
+	return written;
+}
+
+/** Put lines in HistoryResult that are 'latest'
+ * @param r		The history result set that we will use
+ * @param h		The history log object
+ * @param filter	The filter that applies
+ * @returns Number of lines written, note that this could be zero,
+ *          which is a perfectly valid result.
+ */
+static int hbm_return_latest(HistoryResult *r, HistoryLogObject *h, HistoryFilter *filter)
+{
+	HistoryLogLine *l, *n;
+	int written = 0;
+	MessageTag *m;
+
+	for (l = h->tail; l; l = l->prev)
+	{
+		if (filter->timestamp_a && ((m = find_mtag(l->mtags, "time"))) && (strcmp(m->value, filter->timestamp_a) <= 0))
+			break; /* Stop now */
+		else
+		if (filter->msgid_a && ((m = find_mtag(l->mtags, "msgid"))) && !strcmp(m->value, filter->msgid_a))
+			break; /* Stop now */
+
+		n = duplicate_log_line(l);
+		hbm_result_prepend_line(r, n);
+		if (++written >= filter->limit)
+			break;
+	}
+
+	return written;
+}
+
+/** Put lines in HistoryResult based on a 'simple' request, that is: maximum lines or time
+ * @param r		The history result set that we will use
+ * @param h		The history log object
+ * @param filter	The filter that applies
+ * @returns Number of lines written, note that this could be zero,
+ *          which is a perfectly valid result.
+ */
+static int hbm_return_simple(HistoryResult *r, HistoryLogObject *h, HistoryFilter *filter)
 {
-	HistoryResult *r;
-	HistoryLogObject *h = hbm_find_object(object);
 	HistoryLogLine *l;
-	long redline; /* Imaginary timestamp. Before the red line, history is too old. */
 	int lines_sendable = 0, lines_to_skip = 0, cnt = 0;
-
-	if (!h)
-		return NULL; /* nothing found */
-
-	r = safe_alloc(sizeof(HistoryResult));
-	safe_strdup(r->object, object);
+	long redline;
+	int written = 0;
 
 	/* Decide on red line, under this the history is too old.
 	 * Filter can be more strict than history object (but not the other way around):
@@ -635,20 +775,73 @@ HistoryResult *hbm_history_request(char *object, HistoryFilter *filter)
 		{
 			/* Add to result */
 			HistoryLogLine *n = duplicate_log_line(l);
-			if (!r->log)
-			{
-				/* First item */
-				r->log = r->log_tail = n;
-			} else
-			{
-				/* Quick append to tail */
-				r->log_tail->next = n;
-				n->prev = r->log_tail;
-				r->log_tail = n; /* we are the new tail */
-			}
+			hbm_result_append_line(r, n);
+			written++;
 		}
 	}
 
+	return written;
+}
+
+HistoryResult *hbm_history_request(char *object, HistoryFilter *filter)
+{
+	HistoryResult *r;
+	HistoryLogObject *h = hbm_find_object(object);
+	HistoryLogLine *l;
+	int lines_sendable = 0, lines_to_skip = 0, cnt = 0;
+	long redline;
+
+	if (!h)
+		return NULL; /* nothing found */
+
+	/* Check if we need to remove some history entries due to 'time'.
+	 * No need to worry about 'count' as that is being taken care off
+	 * by hbm_history_add().
+	 */
+	if (h->oldest_t < TStime() - h->max_time)
+		hbm_history_cleanup(h);
+
+	r = safe_alloc(sizeof(HistoryResult));
+	safe_strdup(r->object, object);
+
+	switch(filter->cmd)
+	{
+		case HFC_BEFORE:
+			hbm_return_before(r, h, filter);
+			break;
+		case HFC_AFTER:
+			hbm_return_after(r, h, filter);
+			break;
+		case HFC_LATEST:
+			hbm_return_latest(r, h, filter);
+			break;
+		case HFC_AROUND:
+		{
+			int n = 0;
+			int orig_limit = filter->limit;
+
+			/* First request 50% above the search term */
+			if (filter->limit > 1)
+				filter->limit = filter->limit / 2;
+			n = hbm_return_before(r, h, filter);
+			/* Then the remainder (50% or more) below the search term.
+			 *
+			 * Ok, well, unless the original limit was 1 and we already
+			 * sent 1 line, then we may not send anything anymore..
+			 */
+			filter->limit = orig_limit - n;
+			if (filter->limit > 0)
+				hbm_return_after(r, h, filter);
+			break;
+		}
+		case HFC_SIMPLE:
+			hbm_return_simple(r, h, filter);
+			break;
+		default:
+			// unhandled
+			break;
+	}
+
 	return r;
 }