diff --git a/Makefile.windows b/Makefile.windows index 340fbc7d3..385d98e86 100644 --- a/Makefile.windows +++ b/Makefile.windows @@ -430,6 +430,7 @@ DLL_FILES=\ src/modules/usermodes/servicebot.dll \ src/modules/usermodes/showwhois.dll \ src/modules/usermodes/wallops.dll \ + src/modules/utf8functions.dll \ src/modules/vhost.dll \ src/modules/watch-backend.dll \ src/modules/watch.dll \ @@ -1403,6 +1404,9 @@ src/modules/usermodes/showwhois.dll: src/modules/usermodes/showwhois.c $(INCLUDE src/modules/usermodes/wallops.dll: src/modules/usermodes/wallops.c $(INCLUDES) $(CC) $(MODCFLAGS) src/modules/usermodes/wallops.c /Fesrc/modules/usermodes/ /Fosrc/modules/usermodes/ /Fdsrc/modules/usermodes/wallops.pdb $(MODLFLAGS) +src/modules/utf8functions.dll: src/modules/utf8functions.c $(INCLUDES) + $(CC) $(MODCFLAGS) src/modules/utf8functions.c /Fesrc/modules/ /Fosrc/modules/ /Fdsrc/modules/utf8functions.pdb $(MODLFLAGS) + src/modules/vhost.dll: src/modules/vhost.c $(INCLUDES) $(CC) $(MODCFLAGS) src/modules/vhost.c /Fesrc/modules/ /Fosrc/modules/ /Fdsrc/modules/vhost.pdb $(MODLFLAGS) diff --git a/doc/conf/modules.default.conf b/doc/conf/modules.default.conf index 34a06ed9e..664063793 100644 --- a/doc/conf/modules.default.conf +++ b/doc/conf/modules.default.conf @@ -299,6 +299,7 @@ loadmodule "websocket_common"; /* helper functions for websocket (internal) */ loadmodule "spamreport"; /* Spam reporting to a blacklist */ loadmodule "crule"; /* Rules in spamfilter::rule and deny link::rule */ loadmodule "maxperip"; /* allow::maxperip restrictions */ +loadmodule "utf8functions"; /* Various UTF8 helper functions */ loadmodule "geoip_classic"; @if module-loaded("geoip_classic") diff --git a/include/h.h b/include/h.h index 1fb4bccc5..259aca668 100644 --- a/include/h.h +++ b/include/h.h @@ -936,6 +936,7 @@ extern MODVAR void (*exit_client_fmt)(Client *client, MessageTag *recv_mtags, FO extern MODVAR void (*exit_client_ex)(Client *client, Client *origin, MessageTag *recv_mtags, const char *comment); extern MODVAR void (*banned_client)(Client *client, const char *bantype, const char *reason, int global, int noexit); extern MODVAR char (*unreal_expand_string)(const char *str, char *buf, size_t buflen, NameValuePrioList *nvp, int buildvarstring_options, Client *client); +extern MODVAR char *(*utf8_convert_confusables)(const char *i, char *obuf, int olen); /* /Efuncs */ /* TLS functions */ @@ -996,6 +997,7 @@ extern int central_spamreport_enabled_default_handler(void); extern void sasl_succeeded_default_handler(Client *client); extern void sasl_failed_default_handler(Client *client); extern int decode_authenticate_plain_default_handler(const char *param, char **authorization_id, char **authentication_id, char **passwd); +extern char *utf8_convert_confusables_default_handler(const char *i, char *obuf, int olen); /* End of default handlers for efunctions */ extern MODVAR MOTDFile opermotd, svsmotd, motd, botmotd, smotd, rules; diff --git a/include/modules.h b/include/modules.h index e9cabb7d5..3885a0066 100644 --- a/include/modules.h +++ b/include/modules.h @@ -2735,6 +2735,7 @@ enum EfunctionType { EFUNC_EXIT_CLIENT_EX, EFUNC_BANNED_CLIENT, EFUNC_UNREAL_EXPAND_STRING, + EFUNC_UTF8_CONVERT_CONFUSABLES, }; /* Module flags */ diff --git a/src/api-efunctions.c b/src/api-efunctions.c index 875ac7ece..9706b416d 100644 --- a/src/api-efunctions.c +++ b/src/api-efunctions.c @@ -185,6 +185,7 @@ void (*exit_client_fmt)(Client *client, MessageTag *recv_mtags, FORMAT_STRING(co void (*exit_client_ex)(Client *client, Client *origin, MessageTag *recv_mtags, const char *comment); void (*banned_client)(Client *client, const char *bantype, const char *reason, int global, int noexit); char (*unreal_expand_string)(const char *str, char *buf, size_t buflen, NameValuePrioList *nvp, int buildvarstring_options, Client *client); +char *(*utf8_convert_confusables)(const char *i, char *obuf, int olen); Efunction *EfunctionAddMain(Module *module, EfunctionType eftype, int (*func)(), void (*vfunc)(), void *(*pvfunc)(), char *(*stringfunc)(), const char *(*conststringfunc)()) { @@ -512,4 +513,5 @@ void efunctions_init(void) efunc_init_function(EFUNC_EXIT_CLIENT_EX, exit_client_ex, NULL, 0); efunc_init_function(EFUNC_BANNED_CLIENT, banned_client, NULL, 0); efunc_init_function(EFUNC_UNREAL_EXPAND_STRING, unreal_expand_string, NULL, 0); + efunc_init_function(EFUNC_UTF8_CONVERT_CONFUSABLES, utf8_convert_confusables, utf8_convert_confusables_default_handler, 0); } diff --git a/src/misc.c b/src/misc.c index a187fbcf0..afb0f6b24 100644 --- a/src/misc.c +++ b/src/misc.c @@ -1536,6 +1536,12 @@ int decode_authenticate_plain_default_handler(const char *param, char **authoriz return 0; } +char *utf8_convert_confusables_default_handler(const char *i, char *obuf, int olen) +{ + strlcpy(obuf, i, olen); + return obuf; +} + /** my_timegm: mktime()-like function which will use GMT/UTC. * Strangely enough there is no standard function for this. diff --git a/src/modules/Makefile.in b/src/modules/Makefile.in index 91664e2ca..7ac34cabf 100644 --- a/src/modules/Makefile.in +++ b/src/modules/Makefile.in @@ -85,7 +85,7 @@ MODULES= \ real-quit-reason.so \ spamreport.so crule.so \ central-api.so central-blocklist.so \ - no-implicit-names.so maxperip.so \ + no-implicit-names.so maxperip.so utf8functions.so \ $(GEOIP_CLASSIC_OBJECTS) $(GEOIP_MAXMIND_OBJECTS) MODULEFLAGS=@MODULEFLAGS@ diff --git a/src/modules/utf8functions.c b/src/modules/utf8functions.c new file mode 100644 index 000000000..6342e615f --- /dev/null +++ b/src/modules/utf8functions.c @@ -0,0 +1,1982 @@ +/** utf8 helper functions + * (C) 2018-.. Bram Matthys and the UnrealIRCd team + * License: GPLv2 or newer + */ + +/* This contains functions that came from antimixedutf8 + * and from confusables.txt + */ + +#include "unrealircd.h" + +ModuleHeader MOD_HEADER += { + "utf8functions", + "1.0.0", + "UTF8 helper functions", + "UnrealIRCd Team", + "unrealircd-6", +}; + +/* https://unicode.org/Public/UNIDATA/Blocks.txt */ +typedef struct UnicodeBlocks { + uint32_t start; + uint32_t end; + const char *name; + int score; +} UnicodeBlocks; + +typedef struct ConfusablesConversionTable { + uint32_t from; + uint32_t to; +} ConfusablesConversionTable; + +/* This is the list of all the unicode blocks. + * If you want to ignore transition to/from a block, then + * you can comment it out by putting // in front, + * transitions to/from that code block will then not lead to a score. + */ +UnicodeBlocks unicode_blocks[] = +{ + {0x0000, 0x007F, "Basic Latin", 1}, + {0x0080, 0x00FF, "Latin-1 Supplement", 1}, + {0x0100, 0x017F, "Latin Extended-A", 1}, + {0x0180, 0x024F, "Latin Extended-B", 1}, + {0x0250, 0x02AF, "IPA Extensions", 1}, + {0x02B0, 0x02FF, "Spacing Modifier Letters", 1}, + {0x0300, 0x036F, "Combining Diacritical Marks", 1}, + {0x0370, 0x03FF, "Greek and Coptic", 1}, + {0x0400, 0x04FF, "Cyrillic", 1}, + {0x0500, 0x052F, "Cyrillic Supplement", 1}, + {0x0530, 0x058F, "Armenian", 1}, + {0x0590, 0x05FF, "Hebrew", 1}, + {0x0600, 0x06FF, "Arabic", 1}, + {0x0700, 0x074F, "Syriac", 1}, + {0x0750, 0x077F, "Arabic Supplement", 1}, + {0x0780, 0x07BF, "Thaana", 1}, + {0x07C0, 0x07FF, "NKo", 1}, + {0x0800, 0x083F, "Samaritan", 1}, + {0x0840, 0x085F, "Mandaic", 1}, + {0x0860, 0x086F, "Syriac Supplement", 1}, + {0x0870, 0x089F, "Arabic Extended-B", 1}, + {0x08A0, 0x08FF, "Arabic Extended-A", 1}, + {0x0900, 0x097F, "Devanagari", 1}, + {0x0980, 0x09FF, "Bengali", 1}, + {0x0A00, 0x0A7F, "Gurmukhi", 1}, + {0x0A80, 0x0AFF, "Gujarati", 1}, + {0x0B00, 0x0B7F, "Oriya", 1}, + {0x0B80, 0x0BFF, "Tamil", 1}, + {0x0C00, 0x0C7F, "Telugu", 1}, + {0x0C80, 0x0CFF, "Kannada", 1}, + {0x0D00, 0x0D7F, "Malayalam", 1}, + {0x0D80, 0x0DFF, "Sinhala", 1}, + {0x0E00, 0x0E7F, "Thai", 1}, + {0x0E80, 0x0EFF, "Lao", 1}, + {0x0F00, 0x0FFF, "Tibetan", 1}, + {0x1000, 0x109F, "Myanmar", 1}, + {0x10A0, 0x10FF, "Georgian", 1}, + {0x1100, 0x11FF, "Hangul Jamo", 1}, + {0x1200, 0x137F, "Ethiopic", 1}, + {0x1380, 0x139F, "Ethiopic Supplement", 1}, + {0x13A0, 0x13FF, "Cherokee", 1}, + {0x1400, 0x167F, "Unified Canadian Aboriginal Syllabics", 1}, + {0x1680, 0x169F, "Ogham", 1}, + {0x16A0, 0x16FF, "Runic", 1}, + {0x1700, 0x171F, "Tagalog", 1}, + {0x1720, 0x173F, "Hanunoo", 1}, + {0x1740, 0x175F, "Buhid", 1}, + {0x1760, 0x177F, "Tagbanwa", 1}, + {0x1780, 0x17FF, "Khmer", 1}, + {0x1800, 0x18AF, "Mongolian", 1}, + {0x18B0, 0x18FF, "Unified Canadian Aboriginal Syllabics Extended", 1}, + {0x1900, 0x194F, "Limbu", 1}, + {0x1950, 0x197F, "Tai Le", 1}, + {0x1980, 0x19DF, "New Tai Lue", 1}, + {0x19E0, 0x19FF, "Khmer Symbols", 1}, + {0x1A00, 0x1A1F, "Buginese", 1}, + {0x1A20, 0x1AAF, "Tai Tham", 1}, + {0x1AB0, 0x1AFF, "Combining Diacritical Marks Extended", 1}, + {0x1B00, 0x1B7F, "Balinese", 1}, + {0x1B80, 0x1BBF, "Sundanese", 1}, + {0x1BC0, 0x1BFF, "Batak", 1}, + {0x1C00, 0x1C4F, "Lepcha", 1}, + {0x1C50, 0x1C7F, "Ol Chiki", 1}, + {0x1C80, 0x1C8F, "Cyrillic Extended-C", 1}, + {0x1C90, 0x1CBF, "Georgian Extended", 1}, + {0x1CC0, 0x1CCF, "Sundanese Supplement", 1}, + {0x1CD0, 0x1CFF, "Vedic Extensions", 1}, + {0x1D00, 0x1D7F, "Phonetic Extensions", 1}, + {0x1D80, 0x1DBF, "Phonetic Extensions Supplement", 1}, + {0x1DC0, 0x1DFF, "Combining Diacritical Marks Supplement", 1}, + {0x1E00, 0x1EFF, "Latin Extended Additional", 1}, + {0x1F00, 0x1FFF, "Greek Extended", 1}, + {0x2000, 0x206F, "General Punctuation", 1}, + {0x2070, 0x209F, "Superscripts and Subscripts", 1}, + {0x20A0, 0x20CF, "Currency Symbols", 1}, + {0x20D0, 0x20FF, "Combining Diacritical Marks for Symbols", 1}, + {0x2100, 0x214F, "Letterlike Symbols", 1}, + {0x2150, 0x218F, "Number Forms", 1}, + {0x2190, 0x21FF, "Arrows", 1}, + {0x2200, 0x22FF, "Mathematical Operators", 1}, + {0x2300, 0x23FF, "Miscellaneous Technical", 1}, + {0x2400, 0x243F, "Control Pictures", 1}, + {0x2440, 0x245F, "Optical Character Recognition", 1}, + {0x2460, 0x24FF, "Enclosed Alphanumerics", 1}, + {0x2500, 0x257F, "Box Drawing", 1}, + {0x2580, 0x259F, "Block Elements", 1}, + {0x25A0, 0x25FF, "Geometric Shapes", 1}, + {0x2600, 0x26FF, "Miscellaneous Symbols", 1}, + {0x2700, 0x27BF, "Dingbats", 1}, + {0x27C0, 0x27EF, "Miscellaneous Mathematical Symbols-A", 1}, + {0x27F0, 0x27FF, "Supplemental Arrows-A", 1}, + {0x2800, 0x28FF, "Braille Patterns", 1}, + {0x2900, 0x297F, "Supplemental Arrows-B", 1}, + {0x2980, 0x29FF, "Miscellaneous Mathematical Symbols-B", 1}, + {0x2A00, 0x2AFF, "Supplemental Mathematical Operators", 1}, + {0x2B00, 0x2BFF, "Miscellaneous Symbols and Arrows", 1}, + {0x2C00, 0x2C5F, "Glagolitic", 1}, + {0x2C60, 0x2C7F, "Latin Extended-C", 1}, + {0x2C80, 0x2CFF, "Coptic", 1}, + {0x2D00, 0x2D2F, "Georgian Supplement", 1}, + {0x2D30, 0x2D7F, "Tifinagh", 1}, + {0x2D80, 0x2DDF, "Ethiopic Extended", 1}, + {0x2DE0, 0x2DFF, "Cyrillic Extended-A", 1}, + {0x2E00, 0x2E7F, "Supplemental Punctuation", 1}, + {0x2E80, 0x2EFF, "CJK Radicals Supplement", 1}, + {0x2F00, 0x2FDF, "Kangxi Radicals", 1}, + {0x2FF0, 0x2FFF, "Ideographic Description Characters", 1}, + {0x3000, 0x303F, "CJK Symbols and Punctuation", 1}, + {0x3040, 0x309F, "Hiragana", 1}, + {0x30A0, 0x30FF, "Katakana", 1}, + {0x3100, 0x312F, "Bopomofo", 1}, + {0x3130, 0x318F, "Hangul Compatibility Jamo", 1}, + {0x3190, 0x319F, "Kanbun", 1}, + {0x31A0, 0x31BF, "Bopomofo Extended", 1}, + {0x31C0, 0x31EF, "CJK Strokes", 1}, + {0x31F0, 0x31FF, "Katakana Phonetic Extensions", 1}, + {0x3200, 0x32FF, "Enclosed CJK Letters and Months", 1}, + {0x3300, 0x33FF, "CJK Compatibility", 1}, + {0x3400, 0x4DBF, "CJK Unified Ideographs Extension A", 1}, + {0x4DC0, 0x4DFF, "Yijing Hexagram Symbols", 1}, + {0x4E00, 0x9FFF, "CJK Unified Ideographs", 1}, + {0xA000, 0xA48F, "Yi Syllables", 1}, + {0xA490, 0xA4CF, "Yi Radicals", 1}, + {0xA4D0, 0xA4FF, "Lisu", 1}, + {0xA500, 0xA63F, "Vai", 1}, + {0xA640, 0xA69F, "Cyrillic Extended-B", 1}, + {0xA6A0, 0xA6FF, "Bamum", 1}, + {0xA700, 0xA71F, "Modifier Tone Letters", 1}, + {0xA720, 0xA7FF, "Latin Extended-D", 1}, + {0xA800, 0xA82F, "Syloti Nagri", 1}, + {0xA830, 0xA83F, "Common Indic Number Forms", 1}, + {0xA840, 0xA87F, "Phags-pa", 1}, + {0xA880, 0xA8DF, "Saurashtra", 1}, + {0xA8E0, 0xA8FF, "Devanagari Extended", 1}, + {0xA900, 0xA92F, "Kayah Li", 1}, + {0xA930, 0xA95F, "Rejang", 1}, + {0xA960, 0xA97F, "Hangul Jamo Extended-A", 1}, + {0xA980, 0xA9DF, "Javanese", 1}, + {0xA9E0, 0xA9FF, "Myanmar Extended-B", 1}, + {0xAA00, 0xAA5F, "Cham", 1}, + {0xAA60, 0xAA7F, "Myanmar Extended-A", 1}, + {0xAA80, 0xAADF, "Tai Viet", 1}, + {0xAAE0, 0xAAFF, "Meetei Mayek Extensions", 1}, + {0xAB00, 0xAB2F, "Ethiopic Extended-A", 1}, + {0xAB30, 0xAB6F, "Latin Extended-E", 1}, + {0xAB70, 0xABBF, "Cherokee Supplement", 1}, + {0xABC0, 0xABFF, "Meetei Mayek", 1}, + {0xAC00, 0xD7AF, "Hangul Syllables", 1}, + {0xD7B0, 0xD7FF, "Hangul Jamo Extended-B", 1}, + {0xD800, 0xDB7F, "High Surrogates", 1}, + {0xDB80, 0xDBFF, "High Private Use Surrogates", 1}, + {0xDC00, 0xDFFF, "Low Surrogates", 1}, + {0xE000, 0xF8FF, "Private Use Area", 1}, + {0xF900, 0xFAFF, "CJK Compatibility Ideographs", 1}, + {0xFB00, 0xFB4F, "Alphabetic Presentation Forms", 1}, + {0xFB50, 0xFDFF, "Arabic Presentation Forms-A", 1}, + {0xFE00, 0xFE0F, "Variation Selectors", 1}, + {0xFE10, 0xFE1F, "Vertical Forms", 1}, + {0xFE20, 0xFE2F, "Combining Half Marks", 1}, + {0xFE30, 0xFE4F, "CJK Compatibility Forms", 1}, + {0xFE50, 0xFE6F, "Small Form Variants", 1}, + {0xFE70, 0xFEFF, "Arabic Presentation Forms-B", 1}, + {0xFF00, 0xFFEF, "Halfwidth and Fullwidth Forms", 1}, + {0xFFF0, 0xFFFF, "Specials", 1}, + {0x10000, 0x1007F, "Linear B Syllabary", 1}, + {0x10080, 0x100FF, "Linear B Ideograms", 1}, + {0x10100, 0x1013F, "Aegean Numbers", 1}, + {0x10140, 0x1018F, "Ancient Greek Numbers", 1}, + {0x10190, 0x101CF, "Ancient Symbols", 1}, + {0x101D0, 0x101FF, "Phaistos Disc", 1}, + {0x10280, 0x1029F, "Lycian", 1}, + {0x102A0, 0x102DF, "Carian", 1}, + {0x102E0, 0x102FF, "Coptic Epact Numbers", 1}, + {0x10300, 0x1032F, "Old Italic", 1}, + {0x10330, 0x1034F, "Gothic", 1}, + {0x10350, 0x1037F, "Old Permic", 1}, + {0x10380, 0x1039F, "Ugaritic", 1}, + {0x103A0, 0x103DF, "Old Persian", 1}, + {0x10400, 0x1044F, "Deseret", 1}, + {0x10450, 0x1047F, "Shavian", 1}, + {0x10480, 0x104AF, "Osmanya", 1}, + {0x104B0, 0x104FF, "Osage", 1}, + {0x10500, 0x1052F, "Elbasan", 1}, + {0x10530, 0x1056F, "Caucasian Albanian", 1}, + {0x10570, 0x105BF, "Vithkuqi", 1}, + {0x10600, 0x1077F, "Linear A", 1}, + {0x10780, 0x107BF, "Latin Extended-F", 1}, + {0x10800, 0x1083F, "Cypriot Syllabary", 1}, + {0x10840, 0x1085F, "Imperial Aramaic", 1}, + {0x10860, 0x1087F, "Palmyrene", 1}, + {0x10880, 0x108AF, "Nabataean", 1}, + {0x108E0, 0x108FF, "Hatran", 1}, + {0x10900, 0x1091F, "Phoenician", 1}, + {0x10920, 0x1093F, "Lydian", 1}, + {0x10980, 0x1099F, "Meroitic Hieroglyphs", 1}, + {0x109A0, 0x109FF, "Meroitic Cursive", 1}, + {0x10A00, 0x10A5F, "Kharoshthi", 1}, + {0x10A60, 0x10A7F, "Old South Arabian", 1}, + {0x10A80, 0x10A9F, "Old North Arabian", 1}, + {0x10AC0, 0x10AFF, "Manichaean", 1}, + {0x10B00, 0x10B3F, "Avestan", 1}, + {0x10B40, 0x10B5F, "Inscriptional Parthian", 1}, + {0x10B60, 0x10B7F, "Inscriptional Pahlavi", 1}, + {0x10B80, 0x10BAF, "Psalter Pahlavi", 1}, + {0x10C00, 0x10C4F, "Old Turkic", 1}, + {0x10C80, 0x10CFF, "Old Hungarian", 1}, + {0x10D00, 0x10D3F, "Hanifi Rohingya", 1}, + {0x10E60, 0x10E7F, "Rumi Numeral Symbols", 1}, + {0x10E80, 0x10EBF, "Yezidi", 1}, + {0x10EC0, 0x10EFF, "Arabic Extended-C", 1}, + {0x10F00, 0x10F2F, "Old Sogdian", 1}, + {0x10F30, 0x10F6F, "Sogdian", 1}, + {0x10F70, 0x10FAF, "Old Uyghur", 1}, + {0x10FB0, 0x10FDF, "Chorasmian", 1}, + {0x10FE0, 0x10FFF, "Elymaic", 1}, + {0x11000, 0x1107F, "Brahmi", 1}, + {0x11080, 0x110CF, "Kaithi", 1}, + {0x110D0, 0x110FF, "Sora Sompeng", 1}, + {0x11100, 0x1114F, "Chakma", 1}, + {0x11150, 0x1117F, "Mahajani", 1}, + {0x11180, 0x111DF, "Sharada", 1}, + {0x111E0, 0x111FF, "Sinhala Archaic Numbers", 1}, + {0x11200, 0x1124F, "Khojki", 1}, + {0x11280, 0x112AF, "Multani", 1}, + {0x112B0, 0x112FF, "Khudawadi", 1}, + {0x11300, 0x1137F, "Grantha", 1}, + {0x11400, 0x1147F, "Newa", 1}, + {0x11480, 0x114DF, "Tirhuta", 1}, + {0x11580, 0x115FF, "Siddham", 1}, + {0x11600, 0x1165F, "Modi", 1}, + {0x11660, 0x1167F, "Mongolian Supplement", 1}, + {0x11680, 0x116CF, "Takri", 1}, + {0x11700, 0x1174F, "Ahom", 1}, + {0x11800, 0x1184F, "Dogra", 1}, + {0x118A0, 0x118FF, "Warang Citi", 1}, + {0x11900, 0x1195F, "Dives Akuru", 1}, + {0x119A0, 0x119FF, "Nandinagari", 1}, + {0x11A00, 0x11A4F, "Zanabazar Square", 1}, + {0x11A50, 0x11AAF, "Soyombo", 1}, + {0x11AB0, 0x11ABF, "Unified Canadian Aboriginal Syllabics Extended-A", 1}, + {0x11AC0, 0x11AFF, "Pau Cin Hau", 1}, + {0x11B00, 0x11B5F, "Devanagari Extended-A", 1}, + {0x11C00, 0x11C6F, "Bhaiksuki", 1}, + {0x11C70, 0x11CBF, "Marchen", 1}, + {0x11D00, 0x11D5F, "Masaram Gondi", 1}, + {0x11D60, 0x11DAF, "Gunjala Gondi", 1}, + {0x11EE0, 0x11EFF, "Makasar", 1}, + {0x11F00, 0x11F5F, "Kawi", 1}, + {0x11FB0, 0x11FBF, "Lisu Supplement", 1}, + {0x11FC0, 0x11FFF, "Tamil Supplement", 1}, + {0x12000, 0x123FF, "Cuneiform", 1}, + {0x12400, 0x1247F, "Cuneiform Numbers and Punctuation", 1}, + {0x12480, 0x1254F, "Early Dynastic Cuneiform", 1}, + {0x12F90, 0x12FFF, "Cypro-Minoan", 1}, + {0x13000, 0x1342F, "Egyptian Hieroglyphs", 1}, + {0x13430, 0x1345F, "Egyptian Hieroglyph Format Controls", 1}, + {0x14400, 0x1467F, "Anatolian Hieroglyphs", 1}, + {0x16800, 0x16A3F, "Bamum Supplement", 1}, + {0x16A40, 0x16A6F, "Mro", 1}, + {0x16A70, 0x16ACF, "Tangsa", 1}, + {0x16AD0, 0x16AFF, "Bassa Vah", 1}, + {0x16B00, 0x16B8F, "Pahawh Hmong", 1}, + {0x16E40, 0x16E9F, "Medefaidrin", 1}, + {0x16F00, 0x16F9F, "Miao", 1}, + {0x16FE0, 0x16FFF, "Ideographic Symbols and Punctuation", 1}, + {0x17000, 0x187FF, "Tangut", 1}, + {0x18800, 0x18AFF, "Tangut Components", 1}, + {0x18B00, 0x18CFF, "Khitan Small Script", 1}, + {0x18D00, 0x18D7F, "Tangut Supplement", 1}, + {0x1AFF0, 0x1AFFF, "Kana Extended-B", 1}, + {0x1B000, 0x1B0FF, "Kana Supplement", 1}, + {0x1B100, 0x1B12F, "Kana Extended-A", 1}, + {0x1B130, 0x1B16F, "Small Kana Extension", 1}, + {0x1B170, 0x1B2FF, "Nushu", 1}, + {0x1BC00, 0x1BC9F, "Duployan", 1}, + {0x1BCA0, 0x1BCAF, "Shorthand Format Controls", 1}, + {0x1CF00, 0x1CFCF, "Znamenny Musical Notation", 1}, + {0x1D000, 0x1D0FF, "Byzantine Musical Symbols", 1}, + {0x1D100, 0x1D1FF, "Musical Symbols", 1}, + {0x1D200, 0x1D24F, "Ancient Greek Musical Notation", 1}, + {0x1D2C0, 0x1D2DF, "Kaktovik Numerals", 1}, + {0x1D2E0, 0x1D2FF, "Mayan Numerals", 1}, + {0x1D300, 0x1D35F, "Tai Xuan Jing Symbols", 1}, + {0x1D360, 0x1D37F, "Counting Rod Numerals", 1}, + {0x1D400, 0x1D7FF, "Mathematical Alphanumeric Symbols", 3}, + {0x1D800, 0x1DAAF, "Sutton SignWriting", 1}, + {0x1DF00, 0x1DFFF, "Latin Extended-G", 1}, + {0x1E000, 0x1E02F, "Glagolitic Supplement", 1}, + {0x1E030, 0x1E08F, "Cyrillic Extended-D", 1}, + {0x1E100, 0x1E14F, "Nyiakeng Puachue Hmong", 1}, + {0x1E290, 0x1E2BF, "Toto", 1}, + {0x1E2C0, 0x1E2FF, "Wancho", 1}, + {0x1E4D0, 0x1E4FF, "Nag Mundari", 1}, + {0x1E7E0, 0x1E7FF, "Ethiopic Extended-B", 1}, + {0x1E800, 0x1E8DF, "Mende Kikakui", 1}, + {0x1E900, 0x1E95F, "Adlam", 1}, + {0x1EC70, 0x1ECBF, "Indic Siyaq Numbers", 1}, + {0x1ED00, 0x1ED4F, "Ottoman Siyaq Numbers", 1}, + {0x1EE00, 0x1EEFF, "Arabic Mathematical Alphabetic Symbols", 1}, + {0x1F000, 0x1F02F, "Mahjong Tiles", 1}, + {0x1F030, 0x1F09F, "Domino Tiles", 1}, + {0x1F0A0, 0x1F0FF, "Playing Cards", 1}, + {0x1F100, 0x1F1FF, "Enclosed Alphanumeric Supplement", 1}, + {0x1F200, 0x1F2FF, "Enclosed Ideographic Supplement", 1}, + {0x1F300, 0x1F5FF, "Miscellaneous Symbols and Pictographs", 1}, + {0x1F600, 0x1F64F, "Emoticons", 1}, + {0x1F650, 0x1F67F, "Ornamental Dingbats", 1}, + {0x1F680, 0x1F6FF, "Transport and Map Symbols", 1}, + {0x1F700, 0x1F77F, "Alchemical Symbols", 1}, + {0x1F780, 0x1F7FF, "Geometric Shapes Extended", 1}, + {0x1F800, 0x1F8FF, "Supplemental Arrows-C", 1}, + {0x1F900, 0x1F9FF, "Supplemental Symbols and Pictographs", 1}, + {0x1FA00, 0x1FA6F, "Chess Symbols", 1}, + {0x1FA70, 0x1FAFF, "Symbols and Pictographs Extended-A", 1}, + {0x1FB00, 0x1FBFF, "Symbols for Legacy Computing", 1}, + {0x20000, 0x2A6DF, "CJK Unified Ideographs Extension B", 1}, + {0x2A700, 0x2B73F, "CJK Unified Ideographs Extension C", 1}, + {0x2B740, 0x2B81F, "CJK Unified Ideographs Extension D", 1}, + {0x2B820, 0x2CEAF, "CJK Unified Ideographs Extension E", 1}, + {0x2CEB0, 0x2EBEF, "CJK Unified Ideographs Extension F", 1}, + {0x2F800, 0x2FA1F, "CJK Compatibility Ideographs Supplement", 1}, + {0x30000, 0x3134F, "CJK Unified Ideographs Extension G", 1}, + {0x31350, 0x323AF, "CJK Unified Ideographs Extension H", 1}, + {0xE0000, 0xE007F, "Tags", 1}, + {0xE0100, 0xE01EF, "Variation Selectors Supplement", 1}, + {0xF0000, 0xFFFFF, "Supplementary Private Use Area-A", 1}, + {0x100000, 0x10FFFF, "Supplementary Private Use Area-B", 1} +}; + +/* To generate those, i (Syzop) used: + * wget https://www.unicode.org/Public/security/8.0.0/confusables.txt + * egrep '→.*→.*LATIN (SMALL|CAPITAL) LETTER [A-Z]\s' confusables.txt|\ + * egrep -v ';.*[0-9A-Z]+ [0-9A-Z]+.*;.*'|sed 's/\s//g'|\ + * awk -F ';' '{ print "0x" $1 ", 0x" $2 "}," }'|sort -g|awk '{ print "\t{" $0 }' + */ +ConfusablesConversionTable confusables_table[] = +{ + {0x0030, 0x004F}, + {0x0031, 0x006C}, + {0x0049, 0x006C}, + {0x007C, 0x006C}, + {0x00D7, 0x0078}, + {0x011A, 0x0114}, + {0x011B, 0x0115}, + {0x0131, 0x0069}, + {0x0138, 0x006B}, + {0x0146, 0x0272}, + {0x0163, 0x01AB}, + {0x017F, 0x0066}, + {0x0184, 0x0062}, + {0x018D, 0x0067}, + {0x0196, 0x006C}, + {0x01A6, 0x0052}, + {0x01BD, 0x0073}, + {0x01C0, 0x006C}, + {0x01CD, 0x0102}, + {0x01CE, 0x0103}, + {0x01CF, 0x012C}, + {0x01D0, 0x012D}, + {0x01D1, 0x014E}, + {0x01D2, 0x014F}, + {0x01D3, 0x016C}, + {0x01D4, 0x016D}, + {0x01E6, 0x011E}, + {0x01E7, 0x011F}, + {0x01F5, 0x0123}, + {0x021A, 0x0162}, + {0x021B, 0x01AB}, + {0x0226, 0x00C5}, + {0x0227, 0x00E5}, + {0x0251, 0x0061}, + {0x025B, 0xA793}, + {0x0261, 0x0067}, + {0x0263, 0x0079}, + {0x0269, 0x0069}, + {0x026A, 0x0069}, + {0x028B, 0x0075}, + {0x028F, 0x0079}, + {0x02DB, 0x0069}, + {0x037A, 0x0069}, + {0x037F, 0x004A}, + {0x0391, 0x0041}, + {0x0392, 0x0042}, + {0x0395, 0x0045}, + {0x0396, 0x005A}, + {0x0397, 0x0048}, + {0x0399, 0x006C}, + {0x039A, 0x004B}, + {0x039C, 0x004D}, + {0x039D, 0x004E}, + {0x039F, 0x004F}, + {0x03A1, 0x0050}, + {0x03A4, 0x0054}, + {0x03A5, 0x0059}, + {0x03A7, 0x0058}, + {0x03B1, 0x0061}, + {0x03B3, 0x0079}, + {0x03B5, 0xA793}, + {0x03B9, 0x0069}, + {0x03BA, 0x006B}, + {0x03BD, 0x0076}, + {0x03BF, 0x006F}, + {0x03C0, 0x006E}, + {0x03C1, 0x0070}, + {0x03C3, 0x006F}, + {0x03C4, 0x0074}, + {0x03C5, 0x0075}, + {0x03D2, 0x0059}, + {0x03D6, 0x006E}, + {0x03DC, 0x0046}, + {0x03F0, 0x006B}, + {0x03F1, 0x0070}, + {0x03F2, 0x0063}, + {0x03F3, 0x006A}, + {0x03F5, 0xA793}, + {0x03F9, 0x0043}, + {0x03FA, 0x004D}, + {0x0404, 0xA792}, + {0x0405, 0x0053}, + {0x0406, 0x006C}, + {0x0408, 0x004A}, + {0x0410, 0x0041}, + {0x0412, 0x0042}, + {0x0415, 0x0045}, + {0x041A, 0x004B}, + {0x041C, 0x004D}, + {0x041D, 0x0048}, + {0x041E, 0x004F}, + {0x0420, 0x0050}, + {0x0421, 0x0043}, + {0x0422, 0x0054}, + {0x0425, 0x0058}, + {0x042C, 0x0062}, + {0x0430, 0x0061}, + {0x0433, 0x0072}, + {0x0435, 0x0065}, + {0x043A, 0x006B}, + {0x043E, 0x006F}, + {0x043F, 0x006E}, + {0x0440, 0x0070}, + {0x0441, 0x0063}, + {0x0442, 0x0074}, + {0x0443, 0x0079}, + {0x0445, 0x0078}, + {0x0446, 0x0075}, + {0x0454, 0xA793}, + {0x0455, 0x0073}, + {0x0456, 0x0069}, + {0x0458, 0x006A}, + {0x0474, 0x0056}, + {0x0475, 0x0076}, + {0x04AE, 0x0059}, + {0x04AF, 0x0079}, + {0x04BB, 0x0068}, + {0x04BD, 0x0065}, + {0x04C0, 0x006C}, + {0x04CF, 0x0069}, + {0x0501, 0x0064}, + {0x050C, 0x0047}, + {0x0511, 0xA793}, + {0x051B, 0x0071}, + {0x051C, 0x0057}, + {0x054D, 0x0055}, + {0x054F, 0x0053}, + {0x0555, 0x004F}, + {0x0563, 0x0071}, + {0x0566, 0x0071}, + {0x0570, 0x0068}, + {0x0578, 0x006E}, + {0x057C, 0x006E}, + {0x057D, 0x0075}, + {0x0581, 0x0067}, + {0x0584, 0x0066}, + {0x0585, 0x006F}, + {0x05C0, 0x006C}, + {0x05D5, 0x006C}, + {0x05D8, 0x0076}, + {0x05DF, 0x006C}, + {0x05E1, 0x006F}, + {0x0627, 0x006C}, + {0x0647, 0x006F}, + {0x0661, 0x006C}, + {0x0665, 0x006F}, + {0x0667, 0x0056}, + {0x06BE, 0x006F}, + {0x06C1, 0x006F}, + {0x06D5, 0x006F}, + {0x06F1, 0x006C}, + {0x06F5, 0x006F}, + {0x06F7, 0x0056}, + {0x07C0, 0x004F}, + {0x07CA, 0x006C}, + {0x0966, 0x006F}, + {0x09E6, 0x004F}, + {0x0A66, 0x006F}, + {0x0AE6, 0x006F}, + {0x0B20, 0x004F}, + {0x0B66, 0x004F}, + {0x0BE6, 0x006F}, + {0x0C02, 0x006F}, + {0x0C66, 0x006F}, + {0x0C82, 0x006F}, + {0x0CE6, 0x006F}, + {0x0D02, 0x006F}, + {0x0D20, 0x004F}, + {0x0D66, 0x006F}, + {0x0D82, 0x006F}, + {0x0E50, 0x006F}, + {0x0ED0, 0x006F}, + {0x101D, 0x006F}, + {0x1040, 0x006F}, + {0x10E7, 0x0079}, + {0x10FF, 0x006F}, + {0x13A0, 0x0044}, + {0x13A1, 0x0052}, + {0x13A2, 0x0054}, + {0x13A5, 0x0069}, + {0x13A9, 0x0059}, + {0x13AA, 0x0041}, + {0x13AB, 0x004A}, + {0x13AC, 0x0045}, + {0x13B3, 0x0057}, + {0x13B7, 0x004D}, + {0x13BB, 0x0048}, + {0x13BD, 0x0059}, + {0x13BF, 0x01AB}, + {0x13C0, 0x0047}, + {0x13C2, 0x0068}, + {0x13C3, 0x005A}, + {0x13CF, 0x0062}, + {0x13D2, 0x0052}, + {0x13D4, 0x0057}, + {0x13D5, 0x0053}, + {0x13D9, 0x0056}, + {0x13DA, 0x0053}, + {0x13DE, 0x004C}, + {0x13DF, 0x0043}, + {0x13E2, 0x0050}, + {0x13E6, 0x004B}, + {0x13E7, 0x0064}, + {0x13F3, 0x0047}, + {0x13F4, 0x0042}, + {0x142F, 0x0056}, + {0x144C, 0x0055}, + {0x146D, 0x0050}, + {0x146F, 0x0064}, + {0x148D, 0x004A}, + {0x14AA, 0x004C}, + {0x1541, 0x0078}, + {0x157C, 0x0048}, + {0x157D, 0x0078}, + {0x1587, 0x0052}, + {0x15AF, 0x0062}, + {0x15B4, 0x0046}, + {0x15C5, 0x0041}, + {0x15DE, 0x0044}, + {0x15EA, 0x0044}, + {0x15F0, 0x004D}, + {0x15F7, 0x0042}, + {0x166D, 0x0058}, + {0x166E, 0x0078}, + {0x16B7, 0x0058}, + {0x16C1, 0x006C}, + {0x16D5, 0x004B}, + {0x16D6, 0x004D}, + {0x1D00, 0x0041}, + {0x1D04, 0x0063}, + {0x1D0B, 0x006B}, + {0x1D0F, 0x006F}, + {0x1D11, 0x006F}, + {0x1D1B, 0x0074}, + {0x1D1C, 0x0075}, + {0x1D20, 0x0076}, + {0x1D22, 0x007A}, + {0x1D26, 0x0072}, + {0x1D28, 0x006E}, + {0x1D83, 0x0067}, + {0x1D8C, 0x0079}, + {0x1D90, 0x024B}, + {0x1E9A, 0x1EA3}, + {0x1E9D, 0x0066}, + {0x1EFF, 0x0079}, + {0x1FBE, 0x0069}, + {0x20AC, 0xA792}, + {0x2102, 0x0043}, + {0x210A, 0x0067}, + {0x210B, 0x0048}, + {0x210C, 0x0048}, + {0x210D, 0x0048}, + {0x210E, 0x0068}, + {0x2110, 0x006C}, + {0x2111, 0x006C}, + {0x2112, 0x004C}, + {0x2113, 0x006C}, + {0x2115, 0x004E}, + {0x2119, 0x0050}, + {0x211A, 0x0051}, + {0x211B, 0x0052}, + {0x211C, 0x0052}, + {0x211D, 0x0052}, + {0x2124, 0x005A}, + {0x2128, 0x005A}, + {0x212A, 0x004B}, + {0x212C, 0x0042}, + {0x212D, 0x0043}, + {0x212E, 0x0065}, + {0x212F, 0x0065}, + {0x2130, 0x0045}, + {0x2131, 0x0046}, + {0x2133, 0x004D}, + {0x2134, 0x006F}, + {0x2139, 0x0069}, + {0x213C, 0x006E}, + {0x213D, 0x0079}, + {0x2145, 0x0044}, + {0x2146, 0x0064}, + {0x2147, 0x0065}, + {0x2148, 0x0069}, + {0x2149, 0x006A}, + {0x2160, 0x006C}, + {0x2164, 0x0056}, + {0x2169, 0x0058}, + {0x216C, 0x004C}, + {0x216D, 0x0043}, + {0x216E, 0x0044}, + {0x216F, 0x004D}, + {0x2170, 0x0069}, + {0x2174, 0x0076}, + {0x2179, 0x0078}, + {0x217C, 0x006C}, + {0x217D, 0x0063}, + {0x217E, 0x0064}, + {0x2223, 0x006C}, + {0x2228, 0x0076}, + {0x222A, 0x0055}, + {0x22A4, 0x0054}, + {0x22C1, 0x0076}, + {0x22C3, 0x0055}, + {0x22F4, 0xA793}, + {0x22FF, 0x0045}, + {0x2373, 0x0069}, + {0x2374, 0x0070}, + {0x237A, 0x0061}, + {0x24DB, 0x24BE}, + {0x2573, 0x0058}, + {0x27D9, 0x0054}, + {0x292B, 0x0078}, + {0x292C, 0x0078}, + {0x2A2F, 0x0078}, + {0x2C85, 0x0072}, + {0x2C88, 0xA792}, + {0x2C89, 0xA793}, + {0x2C8E, 0x0048}, + {0x2C92, 0x006C}, + {0x2C94, 0x004B}, + {0x2C95, 0x006B}, + {0x2C98, 0x004D}, + {0x2C9A, 0x004E}, + {0x2C9E, 0x004F}, + {0x2C9F, 0x006F}, + {0x2CA2, 0x0050}, + {0x2CA3, 0x0070}, + {0x2CA4, 0x0043}, + {0x2CA5, 0x0063}, + {0x2CA6, 0x0054}, + {0x2CA8, 0x0059}, + {0x2CAC, 0x0058}, + {0x2CD0, 0x004C}, + {0x2D38, 0x0056}, + {0x2D39, 0x0045}, + {0x2D4F, 0x006C}, + {0x2D54, 0x004F}, + {0x2D55, 0x0051}, + {0x2D5D, 0x0058}, + {0x3007, 0x004F}, + {0xA4D0, 0x0042}, + {0xA4D1, 0x0050}, + {0xA4D2, 0x0064}, + {0xA4D3, 0x0044}, + {0xA4D4, 0x0054}, + {0xA4D6, 0x0047}, + {0xA4D7, 0x004B}, + {0xA4D9, 0x004A}, + {0xA4DA, 0x0043}, + {0xA4DC, 0x005A}, + {0xA4DD, 0x0046}, + {0xA4DF, 0x004D}, + {0xA4E0, 0x004E}, + {0xA4E1, 0x004C}, + {0xA4E2, 0x0053}, + {0xA4E3, 0x0052}, + {0xA4E6, 0x0056}, + {0xA4E7, 0x0048}, + {0xA4EA, 0x0057}, + {0xA4EB, 0x0058}, + {0xA4EC, 0x0059}, + {0xA4EE, 0x0041}, + {0xA4F0, 0x0045}, + {0xA4F2, 0x006C}, + {0xA4F3, 0x004F}, + {0xA4F4, 0x0055}, + {0xA647, 0x0069}, + {0xA731, 0x0073}, + {0xA798, 0x0046}, + {0xA799, 0x0066}, + {0xA79F, 0x0075}, + {0xA7B2, 0x004A}, + {0xA7B3, 0x0058}, + {0xA7B4, 0x0042}, + {0xAB32, 0x0065}, + {0xAB35, 0x0066}, + {0xAB3D, 0x006F}, + {0xAB47, 0x0072}, + {0xAB48, 0x0072}, + {0xAB4E, 0x0075}, + {0xAB52, 0x0075}, + {0xAB5A, 0x0079}, + {0xFBA6, 0x006F}, + {0xFBA7, 0x006F}, + {0xFBA8, 0x006F}, + {0xFBA9, 0x006F}, + {0xFBAA, 0x006F}, + {0xFBAB, 0x006F}, + {0xFBAC, 0x006F}, + {0xFBAD, 0x006F}, + {0xFE8D, 0x006C}, + {0xFE8E, 0x006C}, + {0xFEE9, 0x006F}, + {0xFEEA, 0x006F}, + {0xFEEB, 0x006F}, + {0xFEEC, 0x006F}, + {0xFF21, 0x0041}, + {0xFF22, 0x0042}, + {0xFF23, 0x0043}, + {0xFF25, 0x0045}, + {0xFF28, 0x0048}, + {0xFF29, 0x006C}, + {0xFF2A, 0x004A}, + {0xFF2B, 0x004B}, + {0xFF2D, 0x004D}, + {0xFF2E, 0x004E}, + {0xFF2F, 0x004F}, + {0xFF30, 0x0050}, + {0xFF33, 0x0053}, + {0xFF34, 0x0054}, + {0xFF38, 0x0058}, + {0xFF39, 0x0059}, + {0xFF3A, 0x005A}, + {0xFF41, 0x0061}, + {0xFF43, 0x0063}, + {0xFF45, 0x0065}, + {0xFF47, 0x0067}, + {0xFF48, 0x0068}, + {0xFF49, 0x0069}, + {0xFF4A, 0x006A}, + {0xFF4C, 0x006C}, + {0xFF4F, 0x006F}, + {0xFF50, 0x0070}, + {0xFF53, 0x0073}, + {0xFF56, 0x0076}, + {0xFF58, 0x0078}, + {0xFF59, 0x0079}, + {0xFFE8, 0x006C}, + {0x10282, 0x0042}, + {0x10286, 0x0045}, + {0x10287, 0x0046}, + {0x1028A, 0x006C}, + {0x10290, 0x0058}, + {0x10292, 0x004F}, + {0x10295, 0x0050}, + {0x10296, 0x0053}, + {0x10297, 0x0054}, + {0x102A0, 0x0041}, + {0x102A1, 0x0042}, + {0x102A2, 0x0043}, + {0x102A5, 0x0046}, + {0x102AB, 0x004F}, + {0x102B0, 0x004D}, + {0x102B1, 0x0054}, + {0x102B2, 0x0059}, + {0x102B4, 0x0058}, + {0x102CF, 0x0048}, + {0x102F5, 0x005A}, + {0x10301, 0x0042}, + {0x10302, 0x0043}, + {0x10309, 0x006C}, + {0x10311, 0x004D}, + {0x10315, 0x0054}, + {0x10317, 0x0058}, + {0x10320, 0x006C}, + {0x10322, 0x0058}, + {0x10404, 0x004F}, + {0x10415, 0x0043}, + {0x1041B, 0x004C}, + {0x10420, 0x0053}, + {0x10429, 0xA793}, + {0x1042C, 0x006F}, + {0x1043D, 0x0063}, + {0x10448, 0x0073}, + {0x10513, 0x004E}, + {0x10516, 0x004F}, + {0x10518, 0x004B}, + {0x1051C, 0x0043}, + {0x1051D, 0x0056}, + {0x10525, 0x0046}, + {0x10526, 0x004C}, + {0x10527, 0x0058}, + {0x114D0, 0x004F}, + {0x118A0, 0x0056}, + {0x118A2, 0x0046}, + {0x118A3, 0x004C}, + {0x118A4, 0x0059}, + {0x118A6, 0x0045}, + {0x118A9, 0x005A}, + {0x118AE, 0x0045}, + {0x118B2, 0x004C}, + {0x118B5, 0x004F}, + {0x118B8, 0x0055}, + {0x118BC, 0x0054}, + {0x118C0, 0x0076}, + {0x118C1, 0x0073}, + {0x118C2, 0x0046}, + {0x118C3, 0x0069}, + {0x118C4, 0x007A}, + {0x118C8, 0x006F}, + {0x118CE, 0xA793}, + {0x118D7, 0x006F}, + {0x118D8, 0x0075}, + {0x118DC, 0x0079}, + {0x118E0, 0x004F}, + {0x118E5, 0x005A}, + {0x118E6, 0x0057}, + {0x118E9, 0x0043}, + {0x118EC, 0x0058}, + {0x118EF, 0x0057}, + {0x118F2, 0x0043}, + {0x1D400, 0x0041}, + {0x1D401, 0x0042}, + {0x1D402, 0x0043}, + {0x1D403, 0x0044}, + {0x1D404, 0x0045}, + {0x1D405, 0x0046}, + {0x1D406, 0x0047}, + {0x1D407, 0x0048}, + {0x1D408, 0x006C}, + {0x1D409, 0x004A}, + {0x1D40A, 0x004B}, + {0x1D40B, 0x004C}, + {0x1D40C, 0x004D}, + {0x1D40D, 0x004E}, + {0x1D40E, 0x004F}, + {0x1D40F, 0x0050}, + {0x1D410, 0x0051}, + {0x1D411, 0x0052}, + {0x1D412, 0x0053}, + {0x1D413, 0x0054}, + {0x1D414, 0x0055}, + {0x1D415, 0x0056}, + {0x1D416, 0x0057}, + {0x1D417, 0x0058}, + {0x1D418, 0x0059}, + {0x1D419, 0x005A}, + {0x1D41A, 0x0061}, + {0x1D41B, 0x0062}, + {0x1D41C, 0x0063}, + {0x1D41D, 0x0064}, + {0x1D41E, 0x0065}, + {0x1D41F, 0x0066}, + {0x1D420, 0x0067}, + {0x1D421, 0x0068}, + {0x1D422, 0x0069}, + {0x1D423, 0x006A}, + {0x1D424, 0x006B}, + {0x1D425, 0x006C}, + {0x1D427, 0x006E}, + {0x1D428, 0x006F}, + {0x1D429, 0x0070}, + {0x1D42A, 0x0071}, + {0x1D42B, 0x0072}, + {0x1D42C, 0x0073}, + {0x1D42D, 0x0074}, + {0x1D42E, 0x0075}, + {0x1D42F, 0x0076}, + {0x1D431, 0x0078}, + {0x1D432, 0x0079}, + {0x1D433, 0x007A}, + {0x1D434, 0x0041}, + {0x1D435, 0x0042}, + {0x1D436, 0x0043}, + {0x1D437, 0x0044}, + {0x1D438, 0x0045}, + {0x1D439, 0x0046}, + {0x1D43A, 0x0047}, + {0x1D43B, 0x0048}, + {0x1D43C, 0x006C}, + {0x1D43D, 0x004A}, + {0x1D43E, 0x004B}, + {0x1D43F, 0x004C}, + {0x1D440, 0x004D}, + {0x1D441, 0x004E}, + {0x1D442, 0x004F}, + {0x1D443, 0x0050}, + {0x1D444, 0x0051}, + {0x1D445, 0x0052}, + {0x1D446, 0x0053}, + {0x1D447, 0x0054}, + {0x1D448, 0x0055}, + {0x1D449, 0x0056}, + {0x1D44A, 0x0057}, + {0x1D44B, 0x0058}, + {0x1D44C, 0x0059}, + {0x1D44D, 0x005A}, + {0x1D44E, 0x0061}, + {0x1D44F, 0x0062}, + {0x1D450, 0x0063}, + {0x1D451, 0x0064}, + {0x1D452, 0x0065}, + {0x1D453, 0x0066}, + {0x1D454, 0x0067}, + {0x1D456, 0x0069}, + {0x1D457, 0x006A}, + {0x1D458, 0x006B}, + {0x1D459, 0x006C}, + {0x1D45B, 0x006E}, + {0x1D45C, 0x006F}, + {0x1D45D, 0x0070}, + {0x1D45E, 0x0071}, + {0x1D45F, 0x0072}, + {0x1D460, 0x0073}, + {0x1D461, 0x0074}, + {0x1D462, 0x0075}, + {0x1D463, 0x0076}, + {0x1D465, 0x0078}, + {0x1D466, 0x0079}, + {0x1D467, 0x007A}, + {0x1D468, 0x0041}, + {0x1D469, 0x0042}, + {0x1D46A, 0x0043}, + {0x1D46B, 0x0044}, + {0x1D46C, 0x0045}, + {0x1D46D, 0x0046}, + {0x1D46E, 0x0047}, + {0x1D46F, 0x0048}, + {0x1D470, 0x006C}, + {0x1D471, 0x004A}, + {0x1D472, 0x004B}, + {0x1D473, 0x004C}, + {0x1D474, 0x004D}, + {0x1D475, 0x004E}, + {0x1D476, 0x004F}, + {0x1D477, 0x0050}, + {0x1D478, 0x0051}, + {0x1D479, 0x0052}, + {0x1D47A, 0x0053}, + {0x1D47B, 0x0054}, + {0x1D47C, 0x0055}, + {0x1D47D, 0x0056}, + {0x1D47E, 0x0057}, + {0x1D47F, 0x0058}, + {0x1D480, 0x0059}, + {0x1D481, 0x005A}, + {0x1D482, 0x0061}, + {0x1D483, 0x0062}, + {0x1D484, 0x0063}, + {0x1D485, 0x0064}, + {0x1D486, 0x0065}, + {0x1D487, 0x0066}, + {0x1D488, 0x0067}, + {0x1D489, 0x0068}, + {0x1D48A, 0x0069}, + {0x1D48B, 0x006A}, + {0x1D48C, 0x006B}, + {0x1D48D, 0x006C}, + {0x1D48F, 0x006E}, + {0x1D490, 0x006F}, + {0x1D491, 0x0070}, + {0x1D492, 0x0071}, + {0x1D493, 0x0072}, + {0x1D494, 0x0073}, + {0x1D495, 0x0074}, + {0x1D496, 0x0075}, + {0x1D497, 0x0076}, + {0x1D499, 0x0078}, + {0x1D49A, 0x0079}, + {0x1D49B, 0x007A}, + {0x1D49C, 0x0041}, + {0x1D49E, 0x0043}, + {0x1D49F, 0x0044}, + {0x1D4A2, 0x0047}, + {0x1D4A5, 0x004A}, + {0x1D4A6, 0x004B}, + {0x1D4A9, 0x004E}, + {0x1D4AA, 0x004F}, + {0x1D4AB, 0x0050}, + {0x1D4AC, 0x0051}, + {0x1D4AE, 0x0053}, + {0x1D4AF, 0x0054}, + {0x1D4B0, 0x0055}, + {0x1D4B1, 0x0056}, + {0x1D4B2, 0x0057}, + {0x1D4B3, 0x0058}, + {0x1D4B4, 0x0059}, + {0x1D4B5, 0x005A}, + {0x1D4B6, 0x0061}, + {0x1D4B7, 0x0062}, + {0x1D4B8, 0x0063}, + {0x1D4B9, 0x0064}, + {0x1D4BB, 0x0066}, + {0x1D4BD, 0x0068}, + {0x1D4BE, 0x0069}, + {0x1D4BF, 0x006A}, + {0x1D4C0, 0x006B}, + {0x1D4C1, 0x006C}, + {0x1D4C3, 0x006E}, + {0x1D4C5, 0x0070}, + {0x1D4C6, 0x0071}, + {0x1D4C7, 0x0072}, + {0x1D4C8, 0x0073}, + {0x1D4C9, 0x0074}, + {0x1D4CA, 0x0075}, + {0x1D4CB, 0x0076}, + {0x1D4CD, 0x0078}, + {0x1D4CE, 0x0079}, + {0x1D4CF, 0x007A}, + {0x1D4D0, 0x0041}, + {0x1D4D1, 0x0042}, + {0x1D4D2, 0x0043}, + {0x1D4D3, 0x0044}, + {0x1D4D4, 0x0045}, + {0x1D4D5, 0x0046}, + {0x1D4D6, 0x0047}, + {0x1D4D7, 0x0048}, + {0x1D4D8, 0x006C}, + {0x1D4D9, 0x004A}, + {0x1D4DA, 0x004B}, + {0x1D4DB, 0x004C}, + {0x1D4DC, 0x004D}, + {0x1D4DD, 0x004E}, + {0x1D4DE, 0x004F}, + {0x1D4DF, 0x0050}, + {0x1D4E0, 0x0051}, + {0x1D4E1, 0x0052}, + {0x1D4E2, 0x0053}, + {0x1D4E3, 0x0054}, + {0x1D4E4, 0x0055}, + {0x1D4E5, 0x0056}, + {0x1D4E6, 0x0057}, + {0x1D4E7, 0x0058}, + {0x1D4E8, 0x0059}, + {0x1D4E9, 0x005A}, + {0x1D4EA, 0x0061}, + {0x1D4EB, 0x0062}, + {0x1D4EC, 0x0063}, + {0x1D4ED, 0x0064}, + {0x1D4EE, 0x0065}, + {0x1D4EF, 0x0066}, + {0x1D4F0, 0x0067}, + {0x1D4F1, 0x0068}, + {0x1D4F2, 0x0069}, + {0x1D4F3, 0x006A}, + {0x1D4F4, 0x006B}, + {0x1D4F5, 0x006C}, + {0x1D4F7, 0x006E}, + {0x1D4F8, 0x006F}, + {0x1D4F9, 0x0070}, + {0x1D4FA, 0x0071}, + {0x1D4FB, 0x0072}, + {0x1D4FC, 0x0073}, + {0x1D4FD, 0x0074}, + {0x1D4FE, 0x0075}, + {0x1D4FF, 0x0076}, + {0x1D501, 0x0078}, + {0x1D502, 0x0079}, + {0x1D503, 0x007A}, + {0x1D504, 0x0041}, + {0x1D505, 0x0042}, + {0x1D507, 0x0044}, + {0x1D508, 0x0045}, + {0x1D509, 0x0046}, + {0x1D50A, 0x0047}, + {0x1D50D, 0x004A}, + {0x1D50E, 0x004B}, + {0x1D50F, 0x004C}, + {0x1D510, 0x004D}, + {0x1D511, 0x004E}, + {0x1D512, 0x004F}, + {0x1D513, 0x0050}, + {0x1D514, 0x0051}, + {0x1D516, 0x0053}, + {0x1D517, 0x0054}, + {0x1D518, 0x0055}, + {0x1D519, 0x0056}, + {0x1D51A, 0x0057}, + {0x1D51B, 0x0058}, + {0x1D51C, 0x0059}, + {0x1D51E, 0x0061}, + {0x1D51F, 0x0062}, + {0x1D520, 0x0063}, + {0x1D521, 0x0064}, + {0x1D522, 0x0065}, + {0x1D523, 0x0066}, + {0x1D524, 0x0067}, + {0x1D525, 0x0068}, + {0x1D526, 0x0069}, + {0x1D527, 0x006A}, + {0x1D528, 0x006B}, + {0x1D529, 0x006C}, + {0x1D52B, 0x006E}, + {0x1D52C, 0x006F}, + {0x1D52D, 0x0070}, + {0x1D52E, 0x0071}, + {0x1D52F, 0x0072}, + {0x1D530, 0x0073}, + {0x1D531, 0x0074}, + {0x1D532, 0x0075}, + {0x1D533, 0x0076}, + {0x1D535, 0x0078}, + {0x1D536, 0x0079}, + {0x1D537, 0x007A}, + {0x1D538, 0x0041}, + {0x1D539, 0x0042}, + {0x1D53B, 0x0044}, + {0x1D53C, 0x0045}, + {0x1D53D, 0x0046}, + {0x1D53E, 0x0047}, + {0x1D540, 0x006C}, + {0x1D541, 0x004A}, + {0x1D542, 0x004B}, + {0x1D543, 0x004C}, + {0x1D544, 0x004D}, + {0x1D546, 0x004F}, + {0x1D54A, 0x0053}, + {0x1D54B, 0x0054}, + {0x1D54C, 0x0055}, + {0x1D54D, 0x0056}, + {0x1D54E, 0x0057}, + {0x1D54F, 0x0058}, + {0x1D550, 0x0059}, + {0x1D552, 0x0061}, + {0x1D553, 0x0062}, + {0x1D554, 0x0063}, + {0x1D555, 0x0064}, + {0x1D556, 0x0065}, + {0x1D557, 0x0066}, + {0x1D558, 0x0067}, + {0x1D559, 0x0068}, + {0x1D55A, 0x0069}, + {0x1D55B, 0x006A}, + {0x1D55C, 0x006B}, + {0x1D55D, 0x006C}, + {0x1D55F, 0x006E}, + {0x1D560, 0x006F}, + {0x1D561, 0x0070}, + {0x1D562, 0x0071}, + {0x1D563, 0x0072}, + {0x1D564, 0x0073}, + {0x1D565, 0x0074}, + {0x1D566, 0x0075}, + {0x1D567, 0x0076}, + {0x1D569, 0x0078}, + {0x1D56A, 0x0079}, + {0x1D56B, 0x007A}, + {0x1D56C, 0x0041}, + {0x1D56D, 0x0042}, + {0x1D56E, 0x0043}, + {0x1D56F, 0x0044}, + {0x1D570, 0x0045}, + {0x1D571, 0x0046}, + {0x1D572, 0x0047}, + {0x1D573, 0x0048}, + {0x1D574, 0x006C}, + {0x1D575, 0x004A}, + {0x1D576, 0x004B}, + {0x1D577, 0x004C}, + {0x1D578, 0x004D}, + {0x1D579, 0x004E}, + {0x1D57A, 0x004F}, + {0x1D57B, 0x0050}, + {0x1D57C, 0x0051}, + {0x1D57D, 0x0052}, + {0x1D57E, 0x0053}, + {0x1D57F, 0x0054}, + {0x1D580, 0x0055}, + {0x1D581, 0x0056}, + {0x1D582, 0x0057}, + {0x1D583, 0x0058}, + {0x1D584, 0x0059}, + {0x1D585, 0x005A}, + {0x1D586, 0x0061}, + {0x1D587, 0x0062}, + {0x1D588, 0x0063}, + {0x1D589, 0x0064}, + {0x1D58A, 0x0065}, + {0x1D58B, 0x0066}, + {0x1D58C, 0x0067}, + {0x1D58D, 0x0068}, + {0x1D58E, 0x0069}, + {0x1D58F, 0x006A}, + {0x1D590, 0x006B}, + {0x1D591, 0x006C}, + {0x1D593, 0x006E}, + {0x1D594, 0x006F}, + {0x1D595, 0x0070}, + {0x1D596, 0x0071}, + {0x1D597, 0x0072}, + {0x1D598, 0x0073}, + {0x1D599, 0x0074}, + {0x1D59A, 0x0075}, + {0x1D59B, 0x0076}, + {0x1D59D, 0x0078}, + {0x1D59E, 0x0079}, + {0x1D59F, 0x007A}, + {0x1D5A0, 0x0041}, + {0x1D5A1, 0x0042}, + {0x1D5A2, 0x0043}, + {0x1D5A3, 0x0044}, + {0x1D5A4, 0x0045}, + {0x1D5A5, 0x0046}, + {0x1D5A6, 0x0047}, + {0x1D5A7, 0x0048}, + {0x1D5A8, 0x006C}, + {0x1D5A9, 0x004A}, + {0x1D5AA, 0x004B}, + {0x1D5AB, 0x004C}, + {0x1D5AC, 0x004D}, + {0x1D5AD, 0x004E}, + {0x1D5AE, 0x004F}, + {0x1D5AF, 0x0050}, + {0x1D5B0, 0x0051}, + {0x1D5B1, 0x0052}, + {0x1D5B2, 0x0053}, + {0x1D5B3, 0x0054}, + {0x1D5B4, 0x0055}, + {0x1D5B5, 0x0056}, + {0x1D5B6, 0x0057}, + {0x1D5B7, 0x0058}, + {0x1D5B8, 0x0059}, + {0x1D5B9, 0x005A}, + {0x1D5BA, 0x0061}, + {0x1D5BB, 0x0062}, + {0x1D5BC, 0x0063}, + {0x1D5BD, 0x0064}, + {0x1D5BE, 0x0065}, + {0x1D5BF, 0x0066}, + {0x1D5C0, 0x0067}, + {0x1D5C1, 0x0068}, + {0x1D5C2, 0x0069}, + {0x1D5C3, 0x006A}, + {0x1D5C4, 0x006B}, + {0x1D5C5, 0x006C}, + {0x1D5C7, 0x006E}, + {0x1D5C8, 0x006F}, + {0x1D5C9, 0x0070}, + {0x1D5CA, 0x0071}, + {0x1D5CB, 0x0072}, + {0x1D5CC, 0x0073}, + {0x1D5CD, 0x0074}, + {0x1D5CE, 0x0075}, + {0x1D5CF, 0x0076}, + {0x1D5D1, 0x0078}, + {0x1D5D2, 0x0079}, + {0x1D5D3, 0x007A}, + {0x1D5D4, 0x0041}, + {0x1D5D5, 0x0042}, + {0x1D5D6, 0x0043}, + {0x1D5D7, 0x0044}, + {0x1D5D8, 0x0045}, + {0x1D5D9, 0x0046}, + {0x1D5DA, 0x0047}, + {0x1D5DB, 0x0048}, + {0x1D5DC, 0x006C}, + {0x1D5DD, 0x004A}, + {0x1D5DE, 0x004B}, + {0x1D5DF, 0x004C}, + {0x1D5E0, 0x004D}, + {0x1D5E1, 0x004E}, + {0x1D5E2, 0x004F}, + {0x1D5E3, 0x0050}, + {0x1D5E4, 0x0051}, + {0x1D5E5, 0x0052}, + {0x1D5E6, 0x0053}, + {0x1D5E7, 0x0054}, + {0x1D5E8, 0x0055}, + {0x1D5E9, 0x0056}, + {0x1D5EA, 0x0057}, + {0x1D5EB, 0x0058}, + {0x1D5EC, 0x0059}, + {0x1D5ED, 0x005A}, + {0x1D5EE, 0x0061}, + {0x1D5EF, 0x0062}, + {0x1D5F0, 0x0063}, + {0x1D5F1, 0x0064}, + {0x1D5F2, 0x0065}, + {0x1D5F3, 0x0066}, + {0x1D5F4, 0x0067}, + {0x1D5F5, 0x0068}, + {0x1D5F6, 0x0069}, + {0x1D5F7, 0x006A}, + {0x1D5F8, 0x006B}, + {0x1D5F9, 0x006C}, + {0x1D5FB, 0x006E}, + {0x1D5FC, 0x006F}, + {0x1D5FD, 0x0070}, + {0x1D5FE, 0x0071}, + {0x1D5FF, 0x0072}, + {0x1D600, 0x0073}, + {0x1D601, 0x0074}, + {0x1D602, 0x0075}, + {0x1D603, 0x0076}, + {0x1D605, 0x0078}, + {0x1D606, 0x0079}, + {0x1D607, 0x007A}, + {0x1D608, 0x0041}, + {0x1D609, 0x0042}, + {0x1D60A, 0x0043}, + {0x1D60B, 0x0044}, + {0x1D60C, 0x0045}, + {0x1D60D, 0x0046}, + {0x1D60E, 0x0047}, + {0x1D60F, 0x0048}, + {0x1D610, 0x006C}, + {0x1D611, 0x004A}, + {0x1D612, 0x004B}, + {0x1D613, 0x004C}, + {0x1D614, 0x004D}, + {0x1D615, 0x004E}, + {0x1D616, 0x004F}, + {0x1D617, 0x0050}, + {0x1D618, 0x0051}, + {0x1D619, 0x0052}, + {0x1D61A, 0x0053}, + {0x1D61B, 0x0054}, + {0x1D61C, 0x0055}, + {0x1D61D, 0x0056}, + {0x1D61E, 0x0057}, + {0x1D61F, 0x0058}, + {0x1D620, 0x0059}, + {0x1D621, 0x005A}, + {0x1D622, 0x0061}, + {0x1D623, 0x0062}, + {0x1D624, 0x0063}, + {0x1D625, 0x0064}, + {0x1D626, 0x0065}, + {0x1D627, 0x0066}, + {0x1D628, 0x0067}, + {0x1D629, 0x0068}, + {0x1D62A, 0x0069}, + {0x1D62B, 0x006A}, + {0x1D62C, 0x006B}, + {0x1D62D, 0x006C}, + {0x1D62F, 0x006E}, + {0x1D630, 0x006F}, + {0x1D631, 0x0070}, + {0x1D632, 0x0071}, + {0x1D633, 0x0072}, + {0x1D634, 0x0073}, + {0x1D635, 0x0074}, + {0x1D636, 0x0075}, + {0x1D637, 0x0076}, + {0x1D639, 0x0078}, + {0x1D63A, 0x0079}, + {0x1D63B, 0x007A}, + {0x1D63C, 0x0041}, + {0x1D63D, 0x0042}, + {0x1D63E, 0x0043}, + {0x1D63F, 0x0044}, + {0x1D640, 0x0045}, + {0x1D641, 0x0046}, + {0x1D642, 0x0047}, + {0x1D643, 0x0048}, + {0x1D644, 0x006C}, + {0x1D645, 0x004A}, + {0x1D646, 0x004B}, + {0x1D647, 0x004C}, + {0x1D648, 0x004D}, + {0x1D649, 0x004E}, + {0x1D64A, 0x004F}, + {0x1D64B, 0x0050}, + {0x1D64C, 0x0051}, + {0x1D64D, 0x0052}, + {0x1D64E, 0x0053}, + {0x1D64F, 0x0054}, + {0x1D650, 0x0055}, + {0x1D651, 0x0056}, + {0x1D652, 0x0057}, + {0x1D653, 0x0058}, + {0x1D654, 0x0059}, + {0x1D655, 0x005A}, + {0x1D656, 0x0061}, + {0x1D657, 0x0062}, + {0x1D658, 0x0063}, + {0x1D659, 0x0064}, + {0x1D65A, 0x0065}, + {0x1D65B, 0x0066}, + {0x1D65C, 0x0067}, + {0x1D65D, 0x0068}, + {0x1D65E, 0x0069}, + {0x1D65F, 0x006A}, + {0x1D660, 0x006B}, + {0x1D661, 0x006C}, + {0x1D663, 0x006E}, + {0x1D664, 0x006F}, + {0x1D665, 0x0070}, + {0x1D666, 0x0071}, + {0x1D667, 0x0072}, + {0x1D668, 0x0073}, + {0x1D669, 0x0074}, + {0x1D66A, 0x0075}, + {0x1D66B, 0x0076}, + {0x1D66D, 0x0078}, + {0x1D66E, 0x0079}, + {0x1D66F, 0x007A}, + {0x1D670, 0x0041}, + {0x1D671, 0x0042}, + {0x1D672, 0x0043}, + {0x1D673, 0x0044}, + {0x1D674, 0x0045}, + {0x1D675, 0x0046}, + {0x1D676, 0x0047}, + {0x1D677, 0x0048}, + {0x1D678, 0x006C}, + {0x1D679, 0x004A}, + {0x1D67A, 0x004B}, + {0x1D67B, 0x004C}, + {0x1D67C, 0x004D}, + {0x1D67D, 0x004E}, + {0x1D67E, 0x004F}, + {0x1D67F, 0x0050}, + {0x1D680, 0x0051}, + {0x1D681, 0x0052}, + {0x1D682, 0x0053}, + {0x1D683, 0x0054}, + {0x1D684, 0x0055}, + {0x1D685, 0x0056}, + {0x1D686, 0x0057}, + {0x1D687, 0x0058}, + {0x1D688, 0x0059}, + {0x1D689, 0x005A}, + {0x1D68A, 0x0061}, + {0x1D68B, 0x0062}, + {0x1D68C, 0x0063}, + {0x1D68D, 0x0064}, + {0x1D68E, 0x0065}, + {0x1D68F, 0x0066}, + {0x1D690, 0x0067}, + {0x1D691, 0x0068}, + {0x1D692, 0x0069}, + {0x1D693, 0x006A}, + {0x1D694, 0x006B}, + {0x1D695, 0x006C}, + {0x1D697, 0x006E}, + {0x1D698, 0x006F}, + {0x1D699, 0x0070}, + {0x1D69A, 0x0071}, + {0x1D69B, 0x0072}, + {0x1D69C, 0x0073}, + {0x1D69D, 0x0074}, + {0x1D69E, 0x0075}, + {0x1D69F, 0x0076}, + {0x1D6A1, 0x0078}, + {0x1D6A2, 0x0079}, + {0x1D6A3, 0x007A}, + {0x1D6A4, 0x0069}, + {0x1D6A8, 0x0041}, + {0x1D6A9, 0x0042}, + {0x1D6AC, 0x0045}, + {0x1D6AD, 0x005A}, + {0x1D6AE, 0x0048}, + {0x1D6B0, 0x006C}, + {0x1D6B1, 0x004B}, + {0x1D6B3, 0x004D}, + {0x1D6B4, 0x004E}, + {0x1D6B6, 0x004F}, + {0x1D6B8, 0x0050}, + {0x1D6BB, 0x0054}, + {0x1D6BC, 0x0059}, + {0x1D6BE, 0x0058}, + {0x1D6C2, 0x0061}, + {0x1D6C4, 0x0079}, + {0x1D6C6, 0xA793}, + {0x1D6CA, 0x0069}, + {0x1D6CB, 0x006B}, + {0x1D6CE, 0x0076}, + {0x1D6D0, 0x006F}, + {0x1D6D1, 0x006E}, + {0x1D6D2, 0x0070}, + {0x1D6D4, 0x006F}, + {0x1D6D5, 0x0074}, + {0x1D6D6, 0x0075}, + {0x1D6DC, 0xA793}, + {0x1D6DE, 0x006B}, + {0x1D6E0, 0x0070}, + {0x1D6E1, 0x006E}, + {0x1D6E2, 0x0041}, + {0x1D6E3, 0x0042}, + {0x1D6E6, 0x0045}, + {0x1D6E7, 0x005A}, + {0x1D6E8, 0x0048}, + {0x1D6EA, 0x006C}, + {0x1D6EB, 0x004B}, + {0x1D6ED, 0x004D}, + {0x1D6EE, 0x004E}, + {0x1D6F0, 0x004F}, + {0x1D6F2, 0x0050}, + {0x1D6F5, 0x0054}, + {0x1D6F6, 0x0059}, + {0x1D6F8, 0x0058}, + {0x1D6FC, 0x0061}, + {0x1D6FE, 0x0079}, + {0x1D700, 0xA793}, + {0x1D704, 0x0069}, + {0x1D705, 0x006B}, + {0x1D708, 0x0076}, + {0x1D70A, 0x006F}, + {0x1D70B, 0x006E}, + {0x1D70C, 0x0070}, + {0x1D70E, 0x006F}, + {0x1D70F, 0x0074}, + {0x1D710, 0x0075}, + {0x1D716, 0xA793}, + {0x1D718, 0x006B}, + {0x1D71A, 0x0070}, + {0x1D71B, 0x006E}, + {0x1D71C, 0x0041}, + {0x1D71D, 0x0042}, + {0x1D720, 0x0045}, + {0x1D721, 0x005A}, + {0x1D722, 0x0048}, + {0x1D724, 0x006C}, + {0x1D725, 0x004B}, + {0x1D727, 0x004D}, + {0x1D728, 0x004E}, + {0x1D72A, 0x004F}, + {0x1D72C, 0x0050}, + {0x1D72F, 0x0054}, + {0x1D730, 0x0059}, + {0x1D732, 0x0058}, + {0x1D736, 0x0061}, + {0x1D738, 0x0079}, + {0x1D73A, 0xA793}, + {0x1D73E, 0x0069}, + {0x1D73F, 0x006B}, + {0x1D742, 0x0076}, + {0x1D744, 0x006F}, + {0x1D745, 0x006E}, + {0x1D746, 0x0070}, + {0x1D748, 0x006F}, + {0x1D749, 0x0074}, + {0x1D74A, 0x0075}, + {0x1D750, 0xA793}, + {0x1D752, 0x006B}, + {0x1D754, 0x0070}, + {0x1D755, 0x006E}, + {0x1D756, 0x0041}, + {0x1D757, 0x0042}, + {0x1D75A, 0x0045}, + {0x1D75B, 0x005A}, + {0x1D75C, 0x0048}, + {0x1D75E, 0x006C}, + {0x1D75F, 0x004B}, + {0x1D761, 0x004D}, + {0x1D762, 0x004E}, + {0x1D764, 0x004F}, + {0x1D766, 0x0050}, + {0x1D769, 0x0054}, + {0x1D76A, 0x0059}, + {0x1D76C, 0x0058}, + {0x1D770, 0x0061}, + {0x1D772, 0x0079}, + {0x1D774, 0xA793}, + {0x1D778, 0x0069}, + {0x1D779, 0x006B}, + {0x1D77C, 0x0076}, + {0x1D77E, 0x006F}, + {0x1D77F, 0x006E}, + {0x1D780, 0x0070}, + {0x1D782, 0x006F}, + {0x1D783, 0x0074}, + {0x1D784, 0x0075}, + {0x1D78A, 0xA793}, + {0x1D78C, 0x006B}, + {0x1D78E, 0x0070}, + {0x1D78F, 0x006E}, + {0x1D790, 0x0041}, + {0x1D791, 0x0042}, + {0x1D794, 0x0045}, + {0x1D795, 0x005A}, + {0x1D796, 0x0048}, + {0x1D798, 0x006C}, + {0x1D799, 0x004B}, + {0x1D79B, 0x004D}, + {0x1D79C, 0x004E}, + {0x1D79E, 0x004F}, + {0x1D7A0, 0x0050}, + {0x1D7A3, 0x0054}, + {0x1D7A4, 0x0059}, + {0x1D7A6, 0x0058}, + {0x1D7AA, 0x0061}, + {0x1D7AC, 0x0079}, + {0x1D7AE, 0xA793}, + {0x1D7B2, 0x0069}, + {0x1D7B3, 0x006B}, + {0x1D7B6, 0x0076}, + {0x1D7B8, 0x006F}, + {0x1D7B9, 0x006E}, + {0x1D7BA, 0x0070}, + {0x1D7BC, 0x006F}, + {0x1D7BD, 0x0074}, + {0x1D7BE, 0x0075}, + {0x1D7C4, 0xA793}, + {0x1D7C6, 0x006B}, + {0x1D7C8, 0x0070}, + {0x1D7C9, 0x006E}, + {0x1D7CA, 0x0046}, + {0x1D7CE, 0x004F}, + {0x1D7CF, 0x006C}, + {0x1D7D8, 0x004F}, + {0x1D7D9, 0x006C}, + {0x1D7E2, 0x004F}, + {0x1D7E3, 0x006C}, + {0x1D7EC, 0x004F}, + {0x1D7ED, 0x006C}, + {0x1D7F6, 0x004F}, + {0x1D7F7, 0x006C}, + {0x1E8C7, 0x006C}, + {0x1EE00, 0x006C}, + {0x1EE24, 0x006F}, + {0x1EE64, 0x006F}, + {0x1EE80, 0x006C}, + {0x1EE84, 0x006F}, + {0x1F74C, 0x0043}, + {0x1F768, 0x0054}, +}; + +#define IS_IN_RANGE(c, f, l) (((c) >= (f)) && ((c) <= (l))) + +#define SCRIPT_UNDEFINED -1 + +/* Forward declarations */ +char *_utf8_convert_confusables(const char *i, char *obuf, int olen); + +MOD_TEST() +{ + MARK_AS_OFFICIAL_MODULE(modinfo); + EfunctionAddString(modinfo->handle, EFUNC_UTF8_CONVERT_CONFUSABLES, _utf8_convert_confusables); + return MOD_SUCCESS; +} + +MOD_INIT() +{ + MARK_AS_OFFICIAL_MODULE(modinfo); + + return MOD_SUCCESS; +} + +MOD_LOAD() +{ + return MOD_SUCCESS; +} + +MOD_UNLOAD() +{ + return MOD_SUCCESS; +} + +uint32_t utf8_to_utf32(const char *t) +{ + char c1, c2; + const char *ptr = t; + uint32_t uc = 0; + int i; + char seqlen = 0; + + c1 = ptr[0]; + if( (c1 & 0x80) == 0 ) + { + uc = (u_long) (c1 & 0x7F); + seqlen = 1; + } + else if( (c1 & 0xE0) == 0xC0 ) + { + uc = (u_long) (c1 & 0x1F); + seqlen = 2; + } + else if( (c1 & 0xF0) == 0xE0 ) + { + uc = (u_long) (c1 & 0x0F); + seqlen = 3; + } + else if( (c1 & 0xF8) == 0xF0 ) + { + uc = (u_long) (c1 & 0x07); + seqlen = 4; + } else + return -1; /* should be impossible */ + + for (i = 1; i < seqlen; ++i) + { + c1 = ptr[i]; + + if( (c1 & 0xC0) != 0x80 ) + { + // malformed data, do something !!! + return (uint32_t) -1; + } + } + + switch( seqlen ) + { + case 2: + { + c1 = ptr[0]; + + if( !IS_IN_RANGE(c1, 0xC2, 0xDF) ) + { + // malformed data, do something !!! + return (uint32_t) -1; + } + + break; + } + + case 3: + { + c1 = ptr[0]; + c2 = ptr[1]; + + switch (c1) + { + case 0xE0: + if (!IS_IN_RANGE(c2, 0xA0, 0xBF)) + { + // malformed data, do something !!! + return (uint32_t) -1; + } + break; + + case 0xED: + if (!IS_IN_RANGE(c2, 0x80, 0x9F)) + { + // malformed data, do something !!! + return (uint32_t) -1; + } + break; + + default: + if (!IS_IN_RANGE(c1, 0xE1, 0xEC) && !IS_IN_RANGE(c1, 0xEE, 0xEF)) + { + // malformed data, do something !!! + return (uint32_t) -1; + } + break; + } + + break; + } + + case 4: + { + c1 = ptr[0]; + c2 = ptr[1]; + + switch (c1) + { + case 0xF0: + if (!IS_IN_RANGE(c2, 0x90, 0xBF)) + { + // malformed data, do something !!! + return (uint32_t) -1; + } + break; + + case 0xF4: + if (!IS_IN_RANGE(c2, 0x80, 0x8F)) + { + // malformed data, do something !!! + return (uint32_t) -1; + } + break; + + default: + if (!IS_IN_RANGE(c1, 0xF1, 0xF3)) + { + // malformed data, do something !!! + return (uint32_t) -1; + } + break; + } + + break; + } + } + + for (i = 1; i < seqlen; ++i) + { + uc = ((uc << 6) | (uint32_t)(ptr[i] & 0x3F)); + } + return uc; +} + +int utf32_to_utf8(char *buf, uint32_t code) +{ + if (code <= 0x7F) { + buf[0] = code; + return 1; + } + if (code <= 0x7FF) { + buf[0] = 0xC0 | (code >> 6); /* 110xxxxx */ + buf[1] = 0x80 | (code & 0x3F); /* 10xxxxxx */ + return 2; + } + if (code <= 0xFFFF) { + buf[0] = 0xE0 | (code >> 12); /* 1110xxxx */ + buf[1] = 0x80 | ((code >> 6) & 0x3F); /* 10xxxxxx */ + buf[2] = 0x80 | (code & 0x3F); /* 10xxxxxx */ + return 3; + } + if (code <= 0x10FFFF) { + buf[0] = 0xF0 | (code >> 18); /* 11110xxx */ + buf[1] = 0x80 | ((code >> 12) & 0x3F); /* 10xxxxxx */ + buf[2] = 0x80 | ((code >> 6) & 0x3F); /* 10xxxxxx */ + buf[3] = 0x80 | (code & 0x3F); /* 10xxxxxx */ + return 4; + } + return 0; +} + + +/** Detect which script the current character is, + * such as latin script or cyrillic script. + * @retval See SCRIPT_* + */ +int detect_script(uint32_t utfchar) +{ + int i; + + /* Special handling for ASCII: + * - Consider only a-z/A-Z as latin1 + * - All the rest, like spaces or numbers and stuff is + * seen as SCRIPT_UNDEFINED, since it can and will + * likely be used a lot with other unicode blocks too. + */ + if (utfchar <= 127) + { + if (((utfchar >= 'a') && (utfchar <= 'z')) || + ((utfchar >= 'A') && (utfchar <= 'Z'))) + { + return 0; + } + return SCRIPT_UNDEFINED; + } + + for (i=0; i < elementsof(unicode_blocks); i++) + { + if ((utfchar >= unicode_blocks[i].start) && + (utfchar <= unicode_blocks[i].end)) + { + return i; + } + } + + return SCRIPT_UNDEFINED; +} + +/** Returns length of an (UTF8) character. May return <1 for error conditions. + * Made by i + */ +static int utf8_charlen(const char *str) +{ + struct { char mask; char val; } t[4] = + { { 0x80, 0x00 }, { 0xE0, 0xC0 }, { 0xF0, 0xE0 }, { 0xF8, 0xF0 } }; + unsigned k, j; + + for (k = 0; k < 4; k++) + { + if ((*str & t[k].mask) == t[k].val) + { + for (j = 0; j < k; j++) + { + if ((*(++str) & 0xC0) != 0x80) + return -1; + } + return k + 1; + } + } + return 1; +} + +uint32_t utf8_lookup_confusable(uint32_t c) +{ + int start = 0; + int stop = ARRAY_SIZEOF(confusables_table)-1; + int mid; + + while (start <= stop) { + mid = (start+stop)/2; + if (c < confusables_table[mid].from) + stop = mid-1; + else if (c == confusables_table[mid].from) + return confusables_table[mid].to; + else + start = mid+1; + } + return 0; +} + +char *_utf8_convert_confusables(const char *i, char *obuf, int olen) +{ + int len, x; + char *o = obuf; + uint32_t utfchar, conv; + + if ((i == NULL) || (olen <= 0)) + return NULL; + olen--; /* reserve room for \0 right now */ + + for (; *i && olen; i += len) + { + utfchar = utf8_to_utf32(i); + len = utf8_charlen(i); // can't utfchar() set this too? + conv = utf8_lookup_confusable(utfchar); + config_status("char 0x%x to 0x%x", utfchar, conv); // DEBUG + if (conv == 0) + { + /* use as-is */ + if (olen < len) + break; /* cut off */ + for (x=0; x < len; x++) + *o++ = i[x]; + } else { + /* convert */ + int replacelen; + if (olen < 4) // yeah too lazy to lookup replacelen in advance :D + break; /* cut off */ + replacelen = utf32_to_utf8(o, conv); + o += replacelen; + } + } + + *o = '\0'; + return obuf; +} \ No newline at end of file