From cda145b62a2e788bc3cd171056192bd8ef22a754 Mon Sep 17 00:00:00 2001 From: Bram Matthys Date: Wed, 19 May 2021 14:30:42 +0200 Subject: [PATCH] Add initial version of CHATHISTORY command from https://ircv3.net/specs/extensions/chathistory Current status of the module in UnrealIRCd: * A significant part of this is done and working * Currently in modules.optional.conf to get test exposure, not yet loaded by default. * CHATHISTORY subcommands implemented: BEFORE, AFTER, LATEST, AROUND * It does not implement the subcommand "BETWEEN" yet * It does not announce or recognize the (draft) CAP's yet * It does not announce the ISUPPORT token CHATHISTORY=xx yet * Testcases need to be written to validate everything * There will be bugs, now, and also while implementing the rest in the days to come. --- Makefile.windows | 4 + doc/conf/modules.optional.conf | 3 + include/h.h | 1 + include/modules.h | 20 ++- src/api-history-backend.c | 9 ++ src/modules/Makefile.in | 6 +- src/modules/chanmodes/history.c | 1 + src/modules/chathistory.c | 162 ++++++++++++++++++++ src/modules/history.c | 1 + src/modules/history_backend_mem.c | 235 +++++++++++++++++++++++++++--- 10 files changed, 418 insertions(+), 24 deletions(-) create mode 100644 src/modules/chathistory.c diff --git a/Makefile.windows b/Makefile.windows index 5c4c45367..bdcacf1be 100644 --- a/Makefile.windows +++ b/Makefile.windows @@ -304,6 +304,7 @@ DLL_FILES=SRC/MODULES/CLOAK.DLL \ SRC/MODULES/REQUIRE-MODULE.DLL \ SRC/MODULES/IDENT_LOOKUP.DLL \ SRC/MODULES/HISTORY.DLL \ + SRC/MODULES/CHATHISTORY.DLL \ SRC/MODULES/TARGETFLOODPROT.DLL \ SRC/MODULES/TYPING-INDICATOR.DLL \ SRC/MODULES/CLIENTTAGDENY.DLL @@ -1106,6 +1107,9 @@ src/modules/ident_lookup.dll: src/modules/ident_lookup.c $(INCLUDES) src/modules/history.dll: src/modules/history.c $(INCLUDES) $(CC) $(MODCFLAGS) /Fosrc/modules/ /Fesrc/modules/ src/modules/history.c $(MODLFLAGS) +src/modules/chathistory.dll: src/modules/chathistory.c $(INCLUDES) + $(CC) $(MODCFLAGS) /Fosrc/modules/ /Fesrc/modules/ src/modules/chathistory.c $(MODLFLAGS) + src/modules/targetfloodprot.dll: src/modules/targetfloodprot.c $(INCLUDES) $(CC) $(MODCFLAGS) /Fosrc/modules/ /Fesrc/modules/ src/modules/targetfloodprot.c $(MODLFLAGS) diff --git a/doc/conf/modules.optional.conf b/doc/conf/modules.optional.conf index 2e4395e7c..201a44c4a 100644 --- a/doc/conf/modules.optional.conf +++ b/doc/conf/modules.optional.conf @@ -175,3 +175,6 @@ set { ban-time 4h; // For other types } } + +// Currently incomplete and experimental: +loadmodule "chathistory"; diff --git a/include/h.h b/include/h.h index 28456acaa..f75084662 100644 --- a/include/h.h +++ b/include/h.h @@ -972,6 +972,7 @@ extern int history_destroy(char *object); extern int can_receive_history(Client *client); extern void history_send_result(Client *client, HistoryResult *r); extern void free_history_result(HistoryResult *r); +extern void free_history_filter(HistoryFilter *f); extern void special_delayed_unloading(void); extern int write_int64(FILE *fd, uint64_t t); extern int write_int32(FILE *fd, uint32_t t); diff --git a/include/modules.h b/include/modules.h index 84a165027..6a7c1372a 100644 --- a/include/modules.h +++ b/include/modules.h @@ -486,11 +486,27 @@ typedef struct { /** @} */ +/** Filter for history: the command / type of the request */ +typedef enum HistoryFilterCommand { + HFC_SIMPLE=1, /**< Simple history request for lines / unixtime */ + HFC_BEFORE=2, /**< CHATHISTORY BEFORE */ + HFC_AFTER=3, /**< CHATHISTORY AFTER */ + HFC_LATEST=4, /**< CHATHISTORY LATEST */ + HFC_AROUND=5, /**< CHATHISTORY AROUND */ + HFC_BETWEEN=6 /**< CHATHISTORY BETWEEN */ +} HistoryFilterCommand; + /** Filter for history get requests */ typedef struct HistoryFilter HistoryFilter; struct HistoryFilter { - int last_lines; - int last_seconds; + HistoryFilterCommand cmd; /**< Filter command, one of HistoryFilterCommand */ + int last_lines; /**< Used by HFC_SIMPLE */ + int last_seconds; /**< Used by HFC_SIMPLE */ + char *timestamp_a; /**< First parameter of HFC_* (either this or msgid_a) */ + char *msgid_a; /**< First parameter of HFC_* (either this or timestamp_a) */ + char *timestamp_b; /**< Second parameter of HFC_BETWEEN (either this or msgid_b) */ + char *msgid_b; /**< Second parameter of HFC_BETWEEN (either this or timestamp_b) */ + int limit; /**< Maximum number of lines to return */ }; /** History log lines, used by HistoryResult among others */ diff --git a/src/api-history-backend.c b/src/api-history-backend.c index 4d3bcd15b..2c1668db5 100644 --- a/src/api-history-backend.c +++ b/src/api-history-backend.c @@ -273,3 +273,12 @@ void history_send_result(Client *client, HistoryResult *r) if (*batch) sendto_one(client, NULL, ":%s BATCH -%s", me.name, batch); } + +void free_history_filter(HistoryFilter *f) +{ + safe_free(f->timestamp_a); + safe_free(f->msgid_a); + safe_free(f->timestamp_b); + safe_free(f->msgid_b); + safe_free(f); +} diff --git a/src/modules/Makefile.in b/src/modules/Makefile.in index 256f6643d..425238124 100644 --- a/src/modules/Makefile.in +++ b/src/modules/Makefile.in @@ -72,7 +72,7 @@ R_MODULES= \ message-ids.so plaintext-policy.so server-time.so sts.so \ echo-message.so userip-tag.so userhost-tag.so \ typing-indicator.so \ - ident_lookup.so history.so \ + ident_lookup.so history.so chathistory.so \ targetfloodprot.so clienttagdeny.so MODULES=cloak.so $(R_MODULES) @@ -632,6 +632,10 @@ history.so: history.c $(INCLUDES) $(CC) $(CFLAGS) $(MODULEFLAGS) -DDYNAMIC_LINKING \ -o history.so history.c +chathistory.so: chathistory.c $(INCLUDES) + $(CC) $(CFLAGS) $(MODULEFLAGS) -DDYNAMIC_LINKING \ + -o chathistory.so chathistory.c + targetfloodprot.so: targetfloodprot.c $(INCLUDES) $(CC) $(CFLAGS) $(MODULEFLAGS) -DDYNAMIC_LINKING \ -o targetfloodprot.so targetfloodprot.c diff --git a/src/modules/chanmodes/history.c b/src/modules/chanmodes/history.c index 137f1610f..c593f9e9d 100644 --- a/src/modules/chanmodes/history.c +++ b/src/modules/chanmodes/history.c @@ -580,6 +580,7 @@ int history_join(Client *client, Channel *channel, MessageTag *mtags, char *parv HistoryFilter filter; HistoryResult *r; memset(&filter, 0, sizeof(filter)); + filter.cmd = HFC_SIMPLE; filter.last_lines = cfg.playback_on_join.lines; filter.last_seconds = cfg.playback_on_join.time; r = history_request(channel->chname, &filter); diff --git a/src/modules/chathistory.c b/src/modules/chathistory.c new file mode 100644 index 000000000..4e06665b0 --- /dev/null +++ b/src/modules/chathistory.c @@ -0,0 +1,162 @@ +/* src/modules/chathistory.c - IRCv3 CHATHISTORY command. + * (C) Copyright 2021 Bram Matthys (Syzop) and the UnrealIRCd team + * License: GPLv2 + */ +#include "unrealircd.h" + +ModuleHeader MOD_HEADER += { + "chathistory", + "1.0", + "IRCv3 CHATHISTORY command", + "UnrealIRCd Team", + "unrealircd-5", +}; + +/* Forward declarations */ +CMD_FUNC(cmd_chathistory); + +// TODO: change to 50 and move to config: +#define CHATHISTORY_LIMIT 50 + +MOD_INIT() +{ + HistoryBackendInfo hbi; + + MARK_AS_OFFICIAL_MODULE(modinfo); + CommandAdd(modinfo->handle, "CHATHISTORY", cmd_chathistory, MAXPARA, CMD_USER); + + return MOD_SUCCESS; +} + +MOD_LOAD() +{ + return MOD_SUCCESS; +} + +MOD_UNLOAD() +{ + return MOD_SUCCESS; +} + +int chathistory_token(char *str, char *token, char **store) +{ + char *p = strchr(str, '='); + if (!p) + return 0; + *p = '\0'; // frag + if (!strcmp(str, token)) + { + *p = '='; // restore + *store = strdup(p + 1); // can be \0 + return 1; + } + *p = '='; // restore + return 0; +} + +CMD_FUNC(cmd_chathistory) +{ + HistoryFilter *filter = NULL; + HistoryResult *r = NULL; + Channel *channel; + + memset(&filter, 0, sizeof(filter)); + + if ((parc < 5) || BadPtr(parv[4])) + { + sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_PARAMS :Insufficient parameters", me.name); + return; + } + + if (!HasCapability(client, "server-time")) + { + sendnotice(client, "Your IRC client does not support the 'server-time' capability"); + sendnotice(client, "https://ircv3.net/specs/extensions/server-time-3.2.html"); + sendnotice(client, "History request refused."); + return; + } + + channel = find_channel(parv[2], NULL); + if (!channel || !IsMember(client, channel) || !has_channel_mode(channel, 'H')) + { + sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_TARGET %s %s :Messages could not be retrieved", + me.name, parv[1], parv[2]); + return; + } + + filter = safe_alloc(sizeof(HistoryFilter)); + /* Below this point, instead of 'return', use 'goto end', which takes care of the freeing of 'filter' and 'history' */ + + if (!strcmp(parv[1], "BEFORE")) + { + filter->cmd = HFC_BEFORE; + if (!chathistory_token(parv[3], "timestamp", &filter->timestamp_a) && + !chathistory_token(parv[3], "msgid", &filter->msgid_a)) + { + sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_PARAMS %s %s :Invalid parameter, must be timestamp=xxx or msgid=xxx", + me.name, parv[1], parv[3]); + goto end; + } + filter->limit = atoi(parv[4]); + } else + if (!strcmp(parv[1], "AFTER")) + { + filter->cmd = HFC_AFTER; + if (!chathistory_token(parv[3], "timestamp", &filter->timestamp_a) && + !chathistory_token(parv[3], "msgid", &filter->msgid_a)) + { + sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_PARAMS %s %s :Invalid parameter, must be timestamp=xxx or msgid=xxx", + me.name, parv[1], parv[3]); + goto end; + } + filter->limit = atoi(parv[4]); + } else + if (!strcmp(parv[1], "LATEST")) + { + filter->cmd = HFC_LATEST; + if (!chathistory_token(parv[3], "timestamp", &filter->timestamp_a) && + !chathistory_token(parv[3], "msgid", &filter->msgid_a) && + strcmp(parv[3], "*")) + { + sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_PARAMS %s %s :Invalid parameter, must be timestamp=xxx or msgid=xxx or *", + me.name, parv[1], parv[3]); + goto end; + } + filter->limit = atoi(parv[4]); + } else + if (!strcmp(parv[1], "AROUND")) + { + filter->cmd = HFC_AROUND; + if (!chathistory_token(parv[3], "timestamp", &filter->timestamp_a) && + !chathistory_token(parv[3], "msgid", &filter->msgid_a)) + { + sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_PARAMS %s %s :Invalid parameter, must be timestamp=xxx or msgid=xxx", + me.name, parv[1], parv[3]); + goto end; + } + filter->limit = atoi(parv[4]); + } else { + sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_PARAMS %s :Invalid subcommand", me.name, parv[1]); + goto end; + } + + if (filter->limit <= 0) + { + sendto_one(client, NULL, ":%s FAIL CHATHISTORY INVALID_PARAMS %s %d :Specified limit is =<0", + me.name, parv[1], filter->limit); + goto end; + } + + if (filter->limit > CHATHISTORY_LIMIT) + filter->limit = CHATHISTORY_LIMIT; + + if ((r = history_request(channel->chname, filter))) + history_send_result(client, r); + +end: + if (filter) + free_history_filter(filter); + if (r) + free_history_result(r); +} diff --git a/src/modules/history.c b/src/modules/history.c index 07b34ec82..27490892c 100644 --- a/src/modules/history.c +++ b/src/modules/history.c @@ -122,6 +122,7 @@ CMD_FUNC(cmd_history) } memset(&filter, 0, sizeof(filter)); + filter.cmd = HFC_SIMPLE; filter.last_lines = lines; if ((r = history_request(channel->chname, &filter))) diff --git a/src/modules/history_backend_mem.c b/src/modules/history_backend_mem.c index df895ab11..a7bd6d852 100644 --- a/src/modules/history_backend_mem.c +++ b/src/modules/history_backend_mem.c @@ -593,19 +593,159 @@ HistoryLogLine *duplicate_log_line(HistoryLogLine *l) return n; } -HistoryResult *hbm_history_request(char *object, HistoryFilter *filter) +/** Quickly append a new line 'n' to result 'r' */ +static void hbm_result_append_line(HistoryResult *r, HistoryLogLine *n) +{ + if (!r->log) + { + /* First item */ + r->log = r->log_tail = n; + } else + { + /* Quick append to tail */ + r->log_tail->next = n; + n->prev = r->log_tail; + r->log_tail = n; /* we are the new tail */ + } +} + +/** Quickly prepend a new line 'n' to result 'r' */ +static void hbm_result_prepend_line(HistoryResult *r, HistoryLogLine *n) +{ + if (!r->log) + r->log_tail = n; + AddListItem(n, r->log); +} + +/** Put lines in HistoryResult that are after a certain msgid or + * timestamp (excluding said msgid/timestamp). + * @param r The history result set that we will use + * @param h The history log object + * @param filter The filter that applies + * @returns Number of lines written, note that this could be zero, + * which is a perfectly valid result. + */ +static int hbm_return_after(HistoryResult *r, HistoryLogObject *h, HistoryFilter *filter) +{ + HistoryLogLine *l, *n; + int written = 0; + int started = 0; + + for (l = h->head; l; l = l->next) + { + /* Not started yet? Check if this is the starting point... */ + if (!started) + { + MessageTag *m; + if (filter->timestamp_a && ((m = find_mtag(l->mtags, "time"))) && (strcmp(m->value, filter->timestamp_a) > 0)) + { + started = 1; + } else + if (filter->msgid_a && ((m = find_mtag(l->mtags, "msgid"))) && !strcmp(m->value, filter->msgid_a)) + { + started = 1; + continue; + } + } + if (started) + { + // TODO: check for timestamp_b / timestamp_b which define a 'stop' boundary + n = duplicate_log_line(l); + hbm_result_append_line(r, n); + if (++written >= filter->limit) + break; + } + } + + return written; +} + +/** Put lines in HistoryResult that before after a certain msgid or + * timestamp (excluding said msgid/timestamp). + * @param r The history result set that we will use + * @param h The history log object + * @param filter The filter that applies + * @returns Number of lines written, note that this could be zero, + * which is a perfectly valid result. + */ +static int hbm_return_before(HistoryResult *r, HistoryLogObject *h, HistoryFilter *filter) +{ + HistoryLogLine *l, *n; + int written = 0; + int started = 0; + + for (l = h->tail; l; l = l->prev) + { + /* Not started yet? Check if this is the starting point... */ + if (!started) + { + MessageTag *m; + if (filter->timestamp_a && ((m = find_mtag(l->mtags, "time"))) && (strcmp(m->value, filter->timestamp_a) < 0)) + { + started = 1; + } else + if (filter->msgid_a && ((m = find_mtag(l->mtags, "msgid"))) && !strcmp(m->value, filter->msgid_a)) + { + started = 1; + continue; + } + } + if (started) + { + // TODO: check for timestamp_b / timestamp_b which define a 'stop' boundary + n = duplicate_log_line(l); + hbm_result_prepend_line(r, n); + if (++written >= filter->limit) + break; + } + } + + return written; +} + +/** Put lines in HistoryResult that are 'latest' + * @param r The history result set that we will use + * @param h The history log object + * @param filter The filter that applies + * @returns Number of lines written, note that this could be zero, + * which is a perfectly valid result. + */ +static int hbm_return_latest(HistoryResult *r, HistoryLogObject *h, HistoryFilter *filter) +{ + HistoryLogLine *l, *n; + int written = 0; + MessageTag *m; + + for (l = h->tail; l; l = l->prev) + { + if (filter->timestamp_a && ((m = find_mtag(l->mtags, "time"))) && (strcmp(m->value, filter->timestamp_a) <= 0)) + break; /* Stop now */ + else + if (filter->msgid_a && ((m = find_mtag(l->mtags, "msgid"))) && !strcmp(m->value, filter->msgid_a)) + break; /* Stop now */ + + n = duplicate_log_line(l); + hbm_result_prepend_line(r, n); + if (++written >= filter->limit) + break; + } + + return written; +} + +/** Put lines in HistoryResult based on a 'simple' request, that is: maximum lines or time + * @param r The history result set that we will use + * @param h The history log object + * @param filter The filter that applies + * @returns Number of lines written, note that this could be zero, + * which is a perfectly valid result. + */ +static int hbm_return_simple(HistoryResult *r, HistoryLogObject *h, HistoryFilter *filter) { - HistoryResult *r; - HistoryLogObject *h = hbm_find_object(object); HistoryLogLine *l; - long redline; /* Imaginary timestamp. Before the red line, history is too old. */ int lines_sendable = 0, lines_to_skip = 0, cnt = 0; - - if (!h) - return NULL; /* nothing found */ - - r = safe_alloc(sizeof(HistoryResult)); - safe_strdup(r->object, object); + long redline; + int written = 0; /* Decide on red line, under this the history is too old. * Filter can be more strict than history object (but not the other way around): @@ -635,20 +775,73 @@ HistoryResult *hbm_history_request(char *object, HistoryFilter *filter) { /* Add to result */ HistoryLogLine *n = duplicate_log_line(l); - if (!r->log) - { - /* First item */ - r->log = r->log_tail = n; - } else - { - /* Quick append to tail */ - r->log_tail->next = n; - n->prev = r->log_tail; - r->log_tail = n; /* we are the new tail */ - } + hbm_result_append_line(r, n); + written++; } } + return written; +} + +HistoryResult *hbm_history_request(char *object, HistoryFilter *filter) +{ + HistoryResult *r; + HistoryLogObject *h = hbm_find_object(object); + HistoryLogLine *l; + int lines_sendable = 0, lines_to_skip = 0, cnt = 0; + long redline; + + if (!h) + return NULL; /* nothing found */ + + /* Check if we need to remove some history entries due to 'time'. + * No need to worry about 'count' as that is being taken care off + * by hbm_history_add(). + */ + if (h->oldest_t < TStime() - h->max_time) + hbm_history_cleanup(h); + + r = safe_alloc(sizeof(HistoryResult)); + safe_strdup(r->object, object); + + switch(filter->cmd) + { + case HFC_BEFORE: + hbm_return_before(r, h, filter); + break; + case HFC_AFTER: + hbm_return_after(r, h, filter); + break; + case HFC_LATEST: + hbm_return_latest(r, h, filter); + break; + case HFC_AROUND: + { + int n = 0; + int orig_limit = filter->limit; + + /* First request 50% above the search term */ + if (filter->limit > 1) + filter->limit = filter->limit / 2; + n = hbm_return_before(r, h, filter); + /* Then the remainder (50% or more) below the search term. + * + * Ok, well, unless the original limit was 1 and we already + * sent 1 line, then we may not send anything anymore.. + */ + filter->limit = orig_limit - n; + if (filter->limit > 0) + hbm_return_after(r, h, filter); + break; + } + case HFC_SIMPLE: + hbm_return_simple(r, h, filter); + break; + default: + // unhandled + break; + } + return r; }