From 74e17b7a26813eae1e82de8d29e3ebbb89b22cf3 Mon Sep 17 00:00:00 2001 From: Bram Matthys Date: Sun, 23 Mar 2025 13:03:58 +0100 Subject: [PATCH] =?UTF-8?q?Make=20SPAMINFO=20show=20the=20UTF8=20block=20n?= =?UTF-8?q?ames=20a=20text=20uses.=20Example=20output:=20***=20SPAMINFO=20?= =?UTF-8?q?***=20This=20will=20show=20the=20original=20text=20and=20the=20?= =?UTF-8?q?deconfused=20text=20which=20can=20be=20used=20in=20a=20spamfilt?= =?UTF-8?q?er=20block=20with=20input-conversion=20deconfused;=20Original?= =?UTF-8?q?=20spam=20text:=20=E1=BA=94=C5=BD=C5=BD=E1=BA=90=F0=9D=9E=95?= =?UTF-8?q?=C8=A4=E2=84=A4=CE=96=E2=84=A8=E2=84=A8=F0=9D=92=81=F0=9D=93=A9?= =?UTF-8?q?=E1=BA=92=C5=B9=C6=B5=E1=8F=83=C5=BB=E1=BA=92=C5=BD=E2=84=A8?= =?UTF-8?q?=C5=B9=F0=9D=92=B5=F0=9D=9B=A7=C5=BB=F0=9D=9D=9B=F0=9D=9B=A7?= =?UTF-8?q?=E2=84=A8=E2=84=A4=F0=9D=9C=A1=C6=B5=F0=9D=9E=95=F0=9D=98=A1?= =?UTF-8?q?=C5=B9=E1=BA=90=F0=9D=91=8D=E1=BA=94=E1=BA=90=E1=BA=90=CE=96?= =?UTF-8?q?=F0=9D=9C=A1=E1=BA=94=F0=9D=9C=A1=E1=BA=94=F0=9D=9E=95=EA=93=9C?= =?UTF-8?q?=F0=9D=9A=AD=E1=8F=83=E1=BA=90=E1=BA=94=F0=9D=99=95=20Deconfuse?= =?UTF-8?q?d=20spam=20text:=20ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ?= =?UTF-8?q?ZZZZZZ=20AntiMixedUTF8=20points:=2064=20Number=20of=20Unicode?= =?UTF-8?q?=20characters=20in=20total:=2050=20Number=20of=20different=20Un?= =?UTF-8?q?icode=20blocks=20used:=208=20Unicode=20Block=20breakdown=20(nam?= =?UTF-8?q?e:=20bytes=20[capped=20at=20255]):=20-=20Latin=20Extended-A:=20?= =?UTF-8?q?8=20-=20Latin=20Extended-B:=203=20-=20Greek=20and=20Coptic:=202?= =?UTF-8?q?=20-=20Cherokee:=202=20-=20Latin=20Extended=20Additional:=2012?= =?UTF-8?q?=20-=20Letterlike=20Symbols:=206=20-=20Lisu:=201=20-=20Mathemat?= =?UTF-8?q?ical=20Alphanumeric=20Symbols:=2016?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- include/h.h | 4 ++++ include/modules.h | 2 ++ src/api-efunctions.c | 4 ++++ src/misc.c | 10 ++++++++++ src/modules/tkl.c | 34 ++++++++++++++++++++++++++++------ src/modules/utf8functions.c | 22 ++++++++++++++++++++++ 6 files changed, 70 insertions(+), 6 deletions(-) diff --git a/include/h.h b/include/h.h index 8f54b8550..b01b2e893 100644 --- a/include/h.h +++ b/include/h.h @@ -937,6 +937,8 @@ extern MODVAR void (*exit_client_ex)(Client *client, Client *origin, MessageTag extern MODVAR void (*banned_client)(Client *client, const char *bantype, const char *reason, int global, int noexit); extern MODVAR char *(*unreal_expand_string)(const char *str, char *buf, size_t buflen, NameValuePrioList *nvp, int buildvarstring_options, Client *client); extern MODVAR char *(*utf8_convert_confusables)(const char *i, char *obuf, int olen); +extern MODVAR const char *(*utf8_get_block_name)(int i); +extern MODVAR int (*utf8_get_block_number)(const char *name); /* /Efuncs */ /* TLS functions */ @@ -998,6 +1000,8 @@ extern void sasl_succeeded_default_handler(Client *client); extern void sasl_failed_default_handler(Client *client); extern int decode_authenticate_plain_default_handler(const char *param, char **authorization_id, char **authentication_id, char **passwd); extern char *utf8_convert_confusables_default_handler(const char *i, char *obuf, int olen); +extern const char *utf8_get_block_name_default_handler(int i); +extern int utf8_get_block_number_default_handler(const char *name); /* End of default handlers for efunctions */ extern MODVAR MOTDFile opermotd, svsmotd, motd, botmotd, smotd, rules; diff --git a/include/modules.h b/include/modules.h index fd4d06704..6e804c376 100644 --- a/include/modules.h +++ b/include/modules.h @@ -2751,6 +2751,8 @@ enum EfunctionType { EFUNC_UNREAL_EXPAND_STRING, EFUNC_UTF8_CONVERT_CONFUSABLES, EFUNC_UTF8_ANALYZE_TEXT, + EFUNC_UTF8_GET_BLOCK_NAME, + EFUNC_UTF8_GET_BLOCK_NUMBER, }; /* Module flags */ diff --git a/src/api-efunctions.c b/src/api-efunctions.c index 7615d0477..83d344de7 100644 --- a/src/api-efunctions.c +++ b/src/api-efunctions.c @@ -186,6 +186,8 @@ void (*exit_client_ex)(Client *client, Client *origin, MessageTag *recv_mtags, c void (*banned_client)(Client *client, const char *bantype, const char *reason, int global, int noexit); char *(*unreal_expand_string)(const char *str, char *buf, size_t buflen, NameValuePrioList *nvp, int buildvarstring_options, Client *client); char *(*utf8_convert_confusables)(const char *i, char *obuf, int olen); +const char *(*utf8_get_block_name)(int i); +int (*utf8_get_block_number)(const char *name); Efunction *EfunctionAddMain(Module *module, EfunctionType eftype, int (*func)(), void (*vfunc)(), void *(*pvfunc)(), char *(*stringfunc)(), const char *(*conststringfunc)()) { @@ -514,4 +516,6 @@ void efunctions_init(void) efunc_init_function(EFUNC_BANNED_CLIENT, banned_client, NULL, 0); efunc_init_function(EFUNC_UNREAL_EXPAND_STRING, unreal_expand_string, NULL, 0); efunc_init_function(EFUNC_UTF8_CONVERT_CONFUSABLES, utf8_convert_confusables, utf8_convert_confusables_default_handler, 0); + efunc_init_function(EFUNC_UTF8_GET_BLOCK_NAME, utf8_get_block_name, utf8_get_block_name, 0); + efunc_init_function(EFUNC_UTF8_GET_BLOCK_NUMBER, utf8_get_block_number, utf8_get_block_number, 0); } diff --git a/src/misc.c b/src/misc.c index 323ffd243..8c1300e48 100644 --- a/src/misc.c +++ b/src/misc.c @@ -1542,6 +1542,16 @@ char *utf8_convert_confusables_default_handler(const char *i, char *obuf, int ol return obuf; } +const char *utf8_get_block_name_default_handler(int i) +{ + return NULL; +} + +int utf8_get_block_number_default_handler(const char *name) +{ + return -1; +} + /** my_timegm: mktime()-like function which will use GMT/UTC. * Strangely enough there is no standard function for this. * On some *NIX OS's timegm() may be available, sometimes only diff --git a/src/modules/tkl.c b/src/modules/tkl.c index 87f27d506..53e77e06a 100644 --- a/src/modules/tkl.c +++ b/src/modules/tkl.c @@ -255,7 +255,7 @@ MOD_INIT() CommandAdd(modinfo->handle, "SPAMFILTER", cmd_spamfilter, 7, CMD_OPER); CommandAdd(modinfo->handle, "ELINE", cmd_eline, 4, CMD_OPER); CommandAdd(modinfo->handle, "TKL", _cmd_tkl, MAXPARA, CMD_OPER|CMD_SERVER); - CommandAdd(modinfo->handle, "SPAMINFO", cmd_spaminfo, 1, CMD_OPER); + CommandAdd(modinfo->handle, "SPAMINFO", cmd_spaminfo, 1, CMD_OPER|CMD_TEXTANALYSIS); add_default_exempts(); return MOD_SUCCESS; } @@ -6199,7 +6199,7 @@ int spamfilter_pre_command(Client *from, MessageTag *mtags, const char *buf) CMD_FUNC(cmd_spaminfo) { const char *line; - char deconfused[512], *s; + int i, cnt; if (!IsOper(client)) { @@ -6213,14 +6213,36 @@ CMD_FUNC(cmd_spaminfo) return; } + if (!clictx->textanalysis) + { + sendnotice(client, "ERROR: Text analysis is not available. Maybe the utf8functions module is not loaded?"); + return; + } + sendnotice(client, "*** SPAMINFO ***"); sendnotice(client, "This will show the original text and the deconfused text which can be used in a spamfilter block with input-conversion deconfused;"); line = parv[1]; sendnotice(client, "Original spam text: %s", line); - - s = utf8_convert_confusables(line, deconfused, sizeof(deconfused)); - if (s) - sendnotice(client, "Deconfused spam text: %s", s); + sendnotice(client, "Deconfused spam text: %s", clictx->textanalysis->deconfused); + sendnotice(client, "AntiMixedUTF8 points: %d", clictx->textanalysis->antimixedutf8_points); + sendnotice(client, "Number of Unicode characters in total: %d", clictx->textanalysis->num_unicode_characters); + sendnotice(client, "Number of different Unicode blocks used: %d", clictx->textanalysis->unicode_blocks); + sendnotice(client, "Unicode Block breakdown (name: bytes [capped at 255]):"); + for (i = 0, cnt = 0; i < UNICODE_BLOCK_COUNT; i++) + { + if (clictx->textanalysis->unicode_blockmap[i]) + { + cnt += clictx->textanalysis->unicode_blockmap[i]; + sendnotice(client, "- %s: %d", + utf8_get_block_name(i), + (int)clictx->textanalysis->unicode_blockmap[i]); + } + } + if (clictx->textanalysis->num_unicode_characters != cnt) + { + sendnotice(client, "- Non-alpha ASCII characters (digits/spaces/etc.): %d", + clictx->textanalysis->num_unicode_characters - cnt); + } } diff --git a/src/modules/utf8functions.c b/src/modules/utf8functions.c index 8c299dc7b..486a2e696 100644 --- a/src/modules/utf8functions.c +++ b/src/modules/utf8functions.c @@ -5550,11 +5550,15 @@ ConfusablesConversionTable confusables_table[] = /* Forward declarations */ char *_utf8_convert_confusables(const char *i, char *obuf, int olen); int utf8_text_analysis(Client *client, const char *text, TextAnalysis *e); +const char *_utf8_get_block_name(int i); +int _utf8_get_block_number(const char *name); MOD_TEST() { MARK_AS_OFFICIAL_MODULE(modinfo); EfunctionAddString(modinfo->handle, EFUNC_UTF8_CONVERT_CONFUSABLES, _utf8_convert_confusables); + EfunctionAddConstString(modinfo->handle, EFUNC_UTF8_GET_BLOCK_NAME, _utf8_get_block_name); + EfunctionAdd(modinfo->handle, EFUNC_UTF8_GET_BLOCK_NUMBER, _utf8_get_block_number); return MOD_SUCCESS; } @@ -5937,3 +5941,21 @@ char *_utf8_convert_confusables(const char *i, char *obuf, int olen) *o = '\0'; return obuf; } + +/** Get UTF8 name for a block number (eg 0 returns "Basic Latin") */ +const char *_utf8_get_block_name(int i) +{ + if ((i < 0) || (i > ARRAY_SIZEOF(unicode_blocks)-1)) + return NULL; + return unicode_blocks[i].name; +} + +/** Get UTF8 block number by name (eg "Basic Latin" returns 0) */ +int _utf8_get_block_number(const char *name) +{ + int i; + for (i = 0; i < ARRAY_SIZEOF(unicode_blocks); i++) + if (!strcasecmp(unicode_blocks[i].name, name)) + return i; + return -1; +}