- Added special caching of remote includes. When a remote include fails to

load (for example when the webserver is down), then the most recent
  version of that remote include will be used, and the ircd will still boot
  and be able to rehash. Even though this is quite a simple feature, it
  can make a key difference when deciding to roll out remote includes on
  your network. Previously, servers would be unable to boot or rehash when
  the webserver was down, which would be a big problem (often unacceptable).
  The latest version of fetched urls are cached in the cache/ directory as
  cache/<md5 hash of url>.
  Obviously, if there's no 'latest version' and an url fails, the ircd will
  still not be able to boot. This would be the case if you added or changed
  the path of a remote include and it's trying to fetch it for the first time.
  To disable this new behavior, check out REMOTEINC_SPECIALCACHE in
  include/config.h.

Changes

@@ -1810,3 +1810,17 @@
 - When an incorrect command line argument is passed, the IRCd will no longer
   boot. Previously it said 'Server not started' but started anyway.
   Reported and patch provided by ohnobinki (#0003870).
+- Added special caching of remote includes. When a remote include fails to
+  load (for example when the webserver is down), then the most recent
+  version of that remote include will be used, and the ircd will still boot
+  and be able to rehash. Even though this is quite a simple feature, it
+  can make a key difference when deciding to roll out remote includes on
+  your network. Previously, servers would be unable to boot or rehash when
+  the webserver was down, which would be a big problem (often unacceptable).
+  The latest version of fetched urls are cached in the cache/ directory as
+  cache/<md5 hash of url>.
+  Obviously, if there's no 'latest version' and an url fails, the ircd will
+  still not be able to boot. This would be the case if you added or changed
+  the path of a remote include and it's trying to fetch it for the first time.
+  To disable this new behavior, check out REMOTEINC_SPECIALCACHE in
+  include/config.h.

include/h.h

@@ -635,6 +635,8 @@ extern int count_oper_sessions(char *);
 extern char *unreal_mktemp(char *dir, char *suffix);
 extern char *unreal_getpathname(char *filepath, char *path);
 extern char *unreal_getfilename(char *path);
+extern char *unreal_mkcache(char *url);
+extern int has_cached_version(char *url);
 extern int unreal_copyfile(char *src, char *dest);
 extern int unreal_copyfileex(char *src, char *dest, int tryhardlink);
 extern time_t unreal_getfilemodtime(char *filename);
@@ -752,6 +754,7 @@ extern char *clean_ban_mask(char *, int, aClient *);
 extern void chanfloodtimer_stopchantimers(aChannel *chptr);
 extern int find_invex(aChannel *chptr, aClient *sptr);
 extern void DoMD5(unsigned char *mdout, unsigned char *src, unsigned long n);
+extern char *md5hash(unsigned char *dst, unsigned char *src, unsigned long n);
 #ifdef JOINTHROTTLE
 aJFlood *cmodej_addentry(aClient *cptr, aChannel *chptr);
 void cmodej_delentry(aJFlood *e);

src/ircd.c

@@ -1370,8 +1370,14 @@ int InitwIRCD(int argc, char *argv[])
 #endif
 #ifndef _WIN32
 	mkdir("tmp", S_IRUSR|S_IWUSR|S_IXUSR); /* Create the tmp dir, if it doesn't exist */
+ #if defined(USE_LIBCURL) && defined(REMOTEINC_SPECIALCACHE)
+ 	mkdir("cache", S_IRUSR|S_IWUSR|S_IXUSR); /* Create the cache dir, if using curl and it doesn't exist */
+ #endif
 #else
 	mkdir("tmp");
+ #if defined(USE_LIBCURL) && defined(REMOTEINC_SPECIALCACHE)
+ 	mkdir("cache");
+ #endif
 #endif
 #ifndef _WIN32
 	/*

src/md5.c

@@ -16,6 +16,7 @@
  */
 
 #include "config.h"
+#include <stdio.h>
 #if !defined(USE_SSL)
 
 #include <string.h>
@@ -292,3 +293,20 @@ MD5_CTX hash;
 	MD5_Update(&hash, src, n);
 	MD5_Final(mdout, &hash);
 }
+
+/** Generates an MD5 checksum - ASCII printable string (0011223344..etc..).
+ * @param dst[out]  Buffer to store result in, this will be the result will be
+ *                  32 characters + nul terminator, so needs to be at least 33 characters.
+ * @param src[in]   The input data used to generate the checksum.
+ * @param n[in]     Length of data.
+ */
+char *md5hash(unsigned char *dst, unsigned char *src, unsigned long n)
+{
+unsigned char tmp[16];
+	
+	DoMD5(tmp, src, n);
+	sprintf(dst, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x",
+		tmp[0], tmp[1], tmp[2], tmp[3], tmp[4], tmp[5], tmp[6], tmp[7], tmp[8],
+		tmp[9], tmp[10], tmp[11], tmp[12], tmp[13], tmp[14], tmp[15]);
+	return dst;
+}

src/s_conf.c

@@ -9438,7 +9438,7 @@ static void conf_download_complete(char *url, char *file, char *errorbuf, int ca
 		}
 	}
 	if (!file && !cached)
-		add_remote_include(file, url, 0, errorbuf);
+		add_remote_include(file, url, 0, errorbuf); /* DOWNLOAD FAILED */
 	else
 	{
 		if (cached)
@@ -9447,11 +9447,18 @@ static void conf_download_complete(char *url, char *file, char *errorbuf, int ca
 			char *file = unreal_getfilename(urlfile);
 			char *tmp = unreal_mktemp("tmp", file);
 			unreal_copyfileex(inc->file, tmp, 1);
+#ifdef REMOTEINC_SPECIALCACHE
+			unreal_copyfileex(inc->file, unreal_mkcache(url), 0);
+#endif
 			add_remote_include(tmp, url, 0, NULL);
 			free(urlfile);
 		}
-		else
+		else {
 			add_remote_include(file, url, 0, NULL);
+#ifdef REMOTEINC_SPECIALCACHE
+			unreal_copyfileex(file, unreal_mkcache(url), 0);
+#endif
+		}
 	}
 	for (inc = conf_include; inc; inc = (ConfigItem_include *)inc->next)
 	{
@@ -9648,27 +9655,50 @@ int remote_include(ConfigEntry *ce)
 		file = download_file(ce->ce_vardata, &error);
 		if (!file)
 		{
-			config_error("%s:%i: include: error downloading '%s': %s",
-				ce->ce_fileptr->cf_filename, ce->ce_varlinenum,
-				 ce->ce_vardata, error);
-			return -1;
-		}
-		else
-		{
-			if ((ret = load_conf(file)) >= 0)
-				add_remote_include(file, ce->ce_vardata, INCLUDE_USED, NULL);
-			free(file);
-			return ret;
+#ifdef REMOTEINC_SPECIALCACHE
+			if (has_cached_version(ce->ce_vardata))
+			{
+				config_warn("%s:%i: include: error downloading '%s': %s -- using cached version instead.",
+					ce->ce_fileptr->cf_filename, ce->ce_varlinenum,
+					ce->ce_vardata, error);
+				file = strdup(unreal_mkcache(ce->ce_vardata));
+				/* Let it pass to load_conf()... */
+			} else {
+#endif
+				config_error("%s:%i: include: error downloading '%s': %s",
+					ce->ce_fileptr->cf_filename, ce->ce_varlinenum,
+					 ce->ce_vardata, error);
+				return -1;
+#ifdef REMOTEINC_SPECIALCACHE
+			}
+#endif
 		}
+		if ((ret = load_conf(file)) >= 0)
+			add_remote_include(file, ce->ce_vardata, INCLUDE_USED, NULL);
+		free(file);
+		return ret;
 	}
 	else
 	{
 		if (errorbuf)
 		{
-			config_error("%s:%i: include: error downloading '%s': %s",
-                                ce->ce_fileptr->cf_filename, ce->ce_varlinenum,
-                                ce->ce_vardata, errorbuf);
-			return -1;
+#ifdef REMOTEINC_SPECIALCACHE
+			if (has_cached_version(ce->ce_vardata))
+			{
+				config_warn("%s:%i: include: error downloading '%s': %s -- using cached version instead.",
+					ce->ce_fileptr->cf_filename, ce->ce_varlinenum,
+					ce->ce_vardata, errorbuf);
+				/* Let it pass to load_conf()... */
+				file = strdup(unreal_mkcache(ce->ce_vardata));
+			} else {
+#endif
+				config_error("%s:%i: include: error downloading '%s': %s",
+					ce->ce_fileptr->cf_filename, ce->ce_varlinenum,
+					ce->ce_vardata, errorbuf);
+				return -1;
+#ifdef REMOTEINC_SPECIALCACHE
+			}
+#endif
 		}
 		if (config_verbose > 0)
 			config_status("Loading %s from download", ce->ce_vardata);

src/support.c

@@ -1760,6 +1760,24 @@ char *unreal_getfilename(char *path)
         return end;
 }
 
+/* Returns a consistent filename for the cache/ directory.
+ * Returned value will be like: cache/<hash of url>
+ */
+char *unreal_mkcache(char *url)
+{
+static char tempbuf[PATH_MAX+1];
+char tmp2[33];
+	
+	snprintf(tempbuf, PATH_MAX, "cache/%s", md5hash(tmp2, url, strlen(url)));
+	return tempbuf;
+}
+
+/* Returns 1 if a cached version of the url exists, otherwise 0. */
+int has_cached_version(char *url)
+{
+	return file_exists(unreal_mkcache(url));
+}
+
 /* Copys the contents of the src file to the dest file.
  * The dest file will have permissions r-x------
  */