Add URL API and use it at one place from central-blocklist. Docs at:

https://www.unrealircd.org/docs/Dev:URL_API

include/modules.h

@@ -60,6 +60,7 @@ typedef struct Hooktype Hooktype;
 typedef struct Callback Callback;
 typedef struct Efunction Efunction;
 typedef enum EfunctionType EfunctionType;
+typedef struct APICallback APICallback;
 
 /*
  * Module header that every module must include, with the name of
@@ -108,6 +109,7 @@ typedef enum ModuleObjectType {
 	MOBJ_MTAG = 17,
 	MOBJ_HISTORY_BACKEND = 18,
 	MOBJ_RPC = 19,
+	MOBJ_API_CALLBACK = 20,
 } ModuleObjectType;
 
 typedef struct Umode Umode;
@@ -746,6 +748,7 @@ typedef struct ModuleObject {
 		MessageTagHandler *mtag;
 		HistoryBackend *history_backend;
 		RPCHandler *rpc;
+		APICallback *apicallback;
 	} object;
 } ModuleObject;
 
@@ -832,6 +835,20 @@ struct EventInfo {
 	void *data;
 };
 
+typedef enum APICallbackType {
+	API_CALLBACK_WEB_RESPONSE = 1,
+} APICallbackType;
+
+struct APICallback {
+	APICallback *prev, *next;
+	char *name; /**< Name of the api callback */
+	Module *owner; /**< To which module this object belongs */
+	char unloaded; /**< Set to 1 if this object is marked for deletion */
+	APICallbackType callback_type;
+	union {
+		void (*web_response)(OutgoingWebRequest *request, OutgoingWebResponse *response);
+	} callback; /**< The callback itself, obviously chosen by .callback_type */
+};
 
 extern MODVAR Hook		*Hooks[MAXHOOKTYPES];
 extern MODVAR Hooktype		Hooktypes[MAXCUSTOMHOOKS];
@@ -1016,6 +1033,22 @@ extern void LoadPersistentLongLongX(ModuleInfo *modinfo, const char *varshortnam
 extern void SavePersistentLongLongX(ModuleInfo *modinfo, const char *varshortname, long long var);
 #define SavePersistentLongLong(modinfo, var) SavePersistentLongLongX(modinfo, #var, var)
 
+extern APICallback *APICallbackFind(const char *method, APICallbackType callback_type);
+extern void APICallbackDel(APICallback *m);
+extern APICallback *APICallbackAdd(Module *module, APICallback *mreq);
+
+#define RegisterApiCallback(modhandle, api_callback_type, api_name, api_func) \
+	do { \
+		APICallback req; \
+		memset(&req, 0, sizeof(req)); \
+		req.name = api_name; \
+		req.callback_type = api_callback_type; \
+		\
+		if (api_callback_type == API_CALLBACK_WEB_RESPONSE) \
+			req.callback.web_response = api_func; \
+		APICallbackAdd(modhandle, &req); \
+	} while(0)
+
 /** Hooks trigger on "events", such as a new user connecting or joining a channel,
  * see https://www.unrealircd.org/docs/Dev:Hook_API for background info.
  * You are suggested to use CTRL+F on this page to search for any useful hook,

include/struct.h

@@ -143,6 +143,9 @@ typedef struct CommandOverride CommandOverride;
 typedef struct Member Member;
 typedef struct Membership Membership;
 
+typedef struct OutgoingWebRequest OutgoingWebRequest;
+typedef struct OutgoingWebResponse OutgoingWebResponse;
+
 typedef enum OperClassEntryType { OPERCLASSENTRY_ALLOW=1, OPERCLASSENTRY_DENY=2} OperClassEntryType;
 
 typedef enum OperPermission { OPER_ALLOW=1, OPER_DENY=0} OperPermission;
@@ -1881,13 +1884,12 @@ struct HTTPForwardedHeader
 	char ip[IPLEN+1];
 };
 
-typedef struct OutgoingWebRequest OutgoingWebRequest;
-typedef struct OutgoingWebResponse OutgoingWebResponse;
 
 /** An outgoing web request (eg remote includes download) */
 struct OutgoingWebRequest
 {
-	void (*callback)(OutgoingWebRequest *request, OutgoingWebResponse *response);
+	void (*callback)(OutgoingWebRequest *request, OutgoingWebResponse *response); /**< Either use this for non-modules */
+	char *apicallback; /** Or use an api callback that you registered via RegisterApiCallbackWebResponse() before */
 	void *callback_data;
 	char *url; /**< must be freed by url_do_transfers_async() */
 	char *actual_url; /**< if you actually want to use a different url, mostly for redirects (end-users: don't set this!) */
@@ -1902,7 +1904,7 @@ struct OutgoingWebRequest
 	int transfer_timeout; /**< How many seconds the total transfer may take (connect+reading everything) */
 	// If you are adding allocated fields here:
 	// 1) update duplicate_outgoingwebrequest() in src/misc.c
-	// 2) and update url_free_handle_request_portion() there as well
+	// 2) and update free_outgoingwebrequest() there as well
 };
 
 /** The result of an HTTP(S) call, such as the downloaded file, error, etc. */

src/Makefile.in

@@ -30,7 +30,7 @@ OBJS=ircd_vars.o dns.o auth.o channel.o dbuf.o \
 	version.o whowas.o random.o api-usermode.o api-channelmode.o \
 	api-moddata.o api-extban.o api-isupport.o api-command.o \
 	api-clicap.o api-messagetag.o api-history-backend.o api-efunctions.o \
-	api-event.o api-rpc.o \
+	api-event.o api-rpc.o api-apicallback.o \
 	crypt_blowfish.o unrealdb.o crashreport.o modulemanager.o \
 	utf8.o json.o log.o \
 	openssl_hostname_validation.o $(URL)

src/api-apicallback.c

@@ -0,0 +1,185 @@
+/************************************************************************
+ * UnrealIRCd - Unreal Internet Relay Chat Daemon - src/api-apicallback.c
+ * (c) 2022- Bram Matthys and The UnrealIRCd Team
+ * License: GPLv2 or later
+ */
+
+/** @file
+ * @brief APICallback API
+ */
+#include "unrealircd.h"
+
+/** This is for API callbacks like RegisterWebCallback.
+ */
+
+/** List of API Callbacks */
+MODVAR APICallback *apicallbacks = NULL;
+
+/* Forward declarations */
+static void unload_apicallback_commit(APICallback *m);
+#ifdef DEBUGMODE
+static void print_apicallbacks(void);
+#endif
+
+/** Adds a new API Callback.
+ * @param module The module which owns this API callback.
+ * @param mreq   The details of the request such as the name and callback
+ * @return Returns the handle to the API callback if successful, otherwise NULL.
+ *         The module's error code contains specific information about the
+ *         error.
+ */
+APICallback *APICallbackAdd(Module *module, APICallback *mreq)
+{
+	APICallback *m;
+	ModuleObject *mobj;
+
+	/* Some consistency checks to avoid a headache for module devs later on: */
+	if (!mreq->callback_type)
+	{
+		unreal_log(ULOG_ERROR, "module", "API_CALLBACK_ADD_API_ERROR", NULL,
+			   "APICallbackAdd() from module $module_name: "
+			   "Missing required fields.",
+			   log_data_string("module_name", module->header->name));
+		abort();
+	}
+
+	m = APICallbackFind(mreq->name, mreq->callback_type);
+	if (m)
+	{
+		if (m->unloaded)
+		{
+			m->unloaded = 0;
+		} else {
+			if (module)
+				module->errorcode = MODERR_EXISTS;
+			return NULL;
+		}
+	} else {
+		/* New API callback */
+		m = safe_alloc(sizeof(APICallback));
+		safe_strdup(m->name, mreq->name);
+		AddListItem(m, apicallbacks);
+	}
+	/* Add or update the following fields: */
+	m->owner = module;
+	m->callback_type = mreq->callback_type;
+	memcpy(&m->callback, &mreq->callback, sizeof(m->callback));
+
+	/* Add module object */
+	mobj = safe_alloc(sizeof(ModuleObject));
+	mobj->type = MOBJ_API_CALLBACK;
+	mobj->object.apicallback = m;
+	AddListItem(mobj, module->objects);
+	module->errorcode = MODERR_NOERROR;
+
+#ifdef DEBUGMODE
+	unreal_log(ULOG_DEBUG, "module", "API_CALLBACK_DEBUG", NULL, "APICallbackAdd()");
+	print_apicallbacks();
+#endif
+	return m;
+}
+
+/** Returns the API callback for the given name and callback type.
+ * @param name The method to search for.
+ * @param callback_type To which callback_type this belongs (scope)
+ * @return Returns the handle to the API callback,
+ *         or NULL if not found.
+ */
+APICallback *APICallbackFind(const char *name, APICallbackType callback_type)
+{
+	APICallback *m;
+
+#ifdef DEBUGMODE
+	unreal_log(ULOG_DEBUG, "module", "API_CALLBACK_DEBUG", NULL, "APICallbackFind()");
+	print_apicallbacks();
+#endif
+	for (m = apicallbacks; m; m = m->next)
+	{
+		if ((m->callback_type == callback_type) &&
+		    !strcasecmp(name, m->name))
+		{
+			return m;
+		}
+	}
+	return NULL;
+}
+
+/** Remove the specified API callback - modules should not call this.
+ * This is done automatically for modules on unload, so is only called internally.
+ * @param m The API Callback to remove.
+ */
+void APICallbackDel(APICallback *m)
+{
+	if (m->owner)
+	{
+		ModuleObject *mobj;
+		for (mobj = m->owner->objects; mobj; mobj = mobj->next)
+		{
+			if (mobj->type == MOBJ_API_CALLBACK && mobj->object.apicallback == m)
+			{
+				DelListItem(mobj, m->owner->objects);
+				safe_free(mobj);
+				break;
+			}
+		}
+		m->owner = NULL;
+	}
+
+	if (loop.rehashing)
+		m->unloaded = 1;
+	else
+		unload_apicallback_commit(m);
+}
+
+/** @} */
+
+static void unload_apicallback_commit(APICallback *m)
+{
+	/* This is an unusual operation, I think we should log it. */
+	unreal_log(ULOG_INFO, "module", "UNLOAD_API_CALLBACK", NULL,
+	           "Unloading API callback for '$object_name'",
+	           log_data_string("object_name", m->name));
+
+	/* Destroy the object */
+	DelListItem(m, apicallbacks);
+	safe_free(m->name);
+	safe_free(m);
+}
+
+void unload_all_unused_apicallbacks(void)
+{
+	APICallback *m, *m_next;
+
+#ifdef DEBUGMODE
+	unreal_log(ULOG_DEBUG, "module", "API_CALLBACK_DEBUG", NULL, "unload_all_unused_apicallbacks() BEFORE");
+	print_apicallbacks();
+#endif
+	for (m = apicallbacks; m; m = m_next)
+	{
+		m_next = m->next;
+		if (m->unloaded)
+			unload_apicallback_commit(m);
+	}
+#ifdef DEBUGMODE
+	unreal_log(ULOG_DEBUG, "module", "API_CALLBACK_DEBUG", NULL, "unload_all_unused_apicallbacks() AFTER");
+	print_apicallbacks();
+#endif
+}
+
+#ifdef DEBUGMODE
+void print_apicallbacks(void)
+{
+	APICallback *m, *m_next;
+
+	unreal_log(ULOG_DEBUG, "module", "API_CALLBACK_LIST", NULL, "----");
+	for (m = apicallbacks; m; m = m_next)
+	{
+		m_next = m->next;
+		unreal_log(ULOG_DEBUG, "module", "API_CALLBACK_LIST", NULL,
+			"$name ($deleted)",
+			log_data_string("name", m->name),
+			log_data_string("deleted", m->unloaded ? "deleted" : ""));
+	}
+	unreal_log(ULOG_DEBUG, "module", "API_CALLBACK_LIST", NULL, "----");
+}
+#endif

src/conf.c

@@ -192,6 +192,7 @@ extern void unload_all_unused_extbans(void);
 extern void unload_all_unused_caps(void);
 extern void unload_all_unused_history_backends(void);
 extern void unload_all_unused_rpc_handlers(void);
+extern void unload_all_unused_apicallbacks(void);
 
 int reloadable_perm_module_unloaded(void);
 int tls_tests(void);
@@ -11172,6 +11173,7 @@ int rehash_internal(Client *client)
 	unload_all_unused_caps();
 	unload_all_unused_history_backends();
 	unload_all_unused_rpc_handlers();
+	unload_all_unused_apicallbacks();
 	unload_all_unused_moddata();
 	clicap_check_for_changes();
 	umodes_check_for_changes();

src/misc.c

@@ -3178,6 +3178,7 @@ int valid_operclass_name(const char *str)
 /** Free an OutgoingWebRequest struct - note: use safe_free_outgoingwebrequest() instead (which calls us). */
 void free_outgoingwebrequest(OutgoingWebRequest *r)
 {
+	safe_free(r->apicallback);
 	safe_free(r->url);
 	safe_free(r->actual_url);
         safe_free(r->body);
@@ -3189,7 +3190,9 @@ void free_outgoingwebrequest(OutgoingWebRequest *r)
 OutgoingWebRequest *duplicate_outgoingwebrequest(OutgoingWebRequest *orig)
 {
 	OutgoingWebRequest *e = safe_alloc(sizeof(OutgoingWebRequest));
+
 	e->callback = orig->callback;
+	safe_strdup(e->apicallback, orig->apicallback);
 	e->callback_data = orig->callback_data;
 	safe_strdup(e->url, orig->url);
 	safe_strdup(e->actual_url, orig->actual_url);
@@ -3249,7 +3252,7 @@ void url_callback(OutgoingWebRequest *r, const char *file, const char *memory, i
 {
 	OutgoingWebResponse *response;
 
-	if (!r->callback)
+	if (!r->callback && !r->apicallback)
 		return; /* Nothing to do */
 
 	response = safe_alloc(sizeof(OutgoingWebResponse));
@@ -3259,7 +3262,17 @@ void url_callback(OutgoingWebRequest *r, const char *file, const char *memory, i
 	response->errorbuf = errorbuf;
 	response->cached = cached;
 	response->ptr = ptr;
-	r->callback(r, response);
+
+	if (r->callback)
+	{
+		r->callback(r, response);
+	} else if (r->apicallback)
+	{
+		APICallback *cb = APICallbackFind(r->apicallback, API_CALLBACK_WEB_RESPONSE);
+		if (cb && !cb->unloaded)
+			cb->callback.web_response(r, response);
+	}
+
 	safe_free(response);
 }
 

src/modules.c

@@ -572,6 +572,9 @@ void FreeModObj(ModuleObject *obj, Module *m)
 	else if (obj->type == MOBJ_RPC) {
 		RPCHandlerDel(obj->object.rpc);
 	}
+	else if (obj->type == MOBJ_API_CALLBACK) {
+		APICallbackDel(obj->object.apicallback);
+	}
 	else
 	{
 		unreal_log(ULOG_FATAL, "module", "FREEMODOBJ_UNKNOWN_TYPE", NULL,

src/modules/central-blocklist.c

@@ -3,29 +3,6 @@
  * License: GPLv2
  */
 
-/*** <<<MODULE MANAGER START>>>
-module
-{
-	documentation "https://www.unrealircd.org/docs/Central_Blocklist";
-
-	// This is displayed in './unrealircd module info ..' and also if compilation of the module fails:
-	troubleshooting "Please report at https://bugs.unrealircd.org/ if this module fails to compile";
-
-	// Minimum version necessary for this module to work:
-	min-unrealircd-version "6.1.2";
-
-	// Maximum version
-	max-unrealircd-version "6.*";
-
-	post-install-text {
-		"The module is installed. See https://www.unrealircd.org/docs/Central_Blocklist";
-		"for the configuration that you need to add. One important aspect is getting";
-		"an API Key, which is a process that (as of October 2023) is not open to everyone.";
-	}
-}
-*** <<<MODULE MANAGER END>>>
-*/
-
 #include "unrealircd.h"
 
 ModuleHeader MOD_HEADER
@@ -230,6 +207,7 @@ MOD_INIT()
 	HookAdd(modinfo->handle, HOOKTYPE_CONFIGRUN, 0, cbl_config_run);
 	//HookAdd(modinfo->handle, HOOKTYPE_PRE_LOCAL_CONNECT, 0, cbl_prelocalconnect);
 	HookAdd(modinfo->handle, HOOKTYPE_IS_HANDSHAKE_FINISHED, INT_MAX, cbl_is_handshake_finished);
+	RegisterApiCallback(modinfo->handle, API_CALLBACK_WEB_RESPONSE, "cbl_download_complete", cbl_download_complete);
 	return MOD_SUCCESS;
 }
 
@@ -284,7 +262,7 @@ MOD_LOAD()
 
 MOD_UNLOAD()
 {
-	cbl_cancel_all_transfers();
+	// No longer needed thanks to RegisterApiCallbackXX -- cbl_cancel_all_transfers();
 	free_config();
 	return MOD_SUCCESS;
 }
@@ -1038,7 +1016,8 @@ void send_request_for_pending_clients(void)
 	w->body = json_serialized;
 	w->headers = headers;
 	w->max_redirects = 1;
-	w->callback = cbl_download_complete;
+	//w->callback = cbl_download_complete;
+	safe_strdup(w->apicallback, "cbl_download_complete");
 	w->callback_data = c;
 	url_start_async(w);
 }

src/tls.c

@@ -190,10 +190,9 @@ static int ssl_hostname_callback(SSL *ssl, int *unk, void *arg)
 	ConfigItem_sni *sni;
 
 	if (name && (sni = find_sni(name)))
-	{
 		SSL_set_SSL_CTX(ssl, sni->ssl_ctx);
-		set_client_sni_name(ssl, name);
-	}
+
+	set_client_sni_name(ssl, name);
 
 	return SSL_TLSEXT_ERR_OK;
 }

src/url_curl.c

@@ -180,10 +180,6 @@ static void url_check_multi_handles(void)
 				handle->file_fd = NULL;
 			}
 
-			if (handle->request->callback == NULL)
-			{
-				/* Request is already canceled, we don't care about the result */
-			} else
 			if (msg->data.result == CURLE_OK)
 			{
 				if (code == 304 || (last_mod != -1 && last_mod <= handle->request->cachetime))

src/url_unreal.c

@@ -134,8 +134,7 @@ int https_cancel(Download *handle, FORMAT_STRING(const char *pattern), ...)
 	va_start(vl, pattern);
 	vsnprintf(handle->errorbuf, sizeof(handle->errorbuf), pattern, vl);
 	va_end(vl);
-	if (handle->request->callback)
-		url_callback(handle->request, NULL, NULL, 0, handle->errorbuf, 0, handle->request->callback_data);
+	url_callback(handle->request, NULL, NULL, 0, handle->errorbuf, 0, handle->request->callback_data);
 	url_free_handle(handle);
 	return -1;
 }
@@ -934,8 +933,6 @@ void https_done(Download *handle)
 		handle->file_fd = NULL;
 	}
 
-	if (!handle->request->callback)
-		; /* No special action, request was cancelled */
 	else if (!handle->got_response)
 		url_callback(handle->request, NULL, NULL, 0, "HTTPS response not received", 0, handle->request->callback_data);
 	else
@@ -955,33 +952,25 @@ void https_done_cached(Download *handle)
 		fclose(handle->file_fd);
 		handle->file_fd = NULL;
 	}
-	if (handle->request->callback)
-		url_callback(handle->request, NULL, NULL, 0, NULL, 1, handle->request->callback_data);
+	url_callback(handle->request, NULL, NULL, 0, NULL, 1, handle->request->callback_data);
 	url_free_handle(handle);
 }
 
 void https_redirect(Download *handle)
 {
+	OutgoingWebRequest *r;
+
 	if (handle->request->max_redirects == 0)
 	{
 		https_cancel(handle, "Too many HTTP redirects (%d)", DOWNLOAD_MAX_REDIRECTS);
 		return;
 	}
 
-	/* If still an outstanding request (not cancelled), follow the redirect.. */
-	if (handle->request->callback)
-	{
-		OutgoingWebRequest *r = duplicate_outgoingwebrequest(handle->request);
-		safe_strdup(r->actual_url, handle->redirect_new_location); // override actual url
-		r->max_redirects--; // safe, checked to be >0 a few lines up
-		url_free_handle(handle); // free old handle
-		url_start_async(r); // create new one
-	} else {
-		/* The callback is NULL, so we can just free without
-		 * following redirects and any error reporting
-		 */
-		url_free_handle(handle); // free old handle
-	}
+	r = duplicate_outgoingwebrequest(handle->request);
+	safe_strdup(r->actual_url, handle->redirect_new_location); // override actual url
+	r->max_redirects--; // safe, checked to be >0 a few lines up
+	url_free_handle(handle); // free old handle
+	url_start_async(r); // create new one
 }
 
 /** Helper function to parse the HTTP header consisting of multiple 'Key: value' pairs */