diff options
author | Eric Erhardt <eric.erhardt@microsoft.com> | 2015-11-25 18:18:20 -0600 |
---|---|---|
committer | Eric Erhardt <eric.erhardt@microsoft.com> | 2015-12-07 09:44:44 -0600 |
commit | cc5de3941217a8bb1e22a5e024da0516a39f0619 (patch) | |
tree | 26375dec0a3de40248e9a485c459b2b5eb64ec85 /src/corefx | |
parent | 0bfacb1f616c3de99aa27f612b27cc5b51a02bef (diff) | |
download | coreclr-cc5de3941217a8bb1e22a5e024da0516a39f0619.tar.gz coreclr-cc5de3941217a8bb1e22a5e024da0516a39f0619.tar.bz2 coreclr-cc5de3941217a8bb1e22a5e024da0516a39f0619.zip |
Adding support for String CompareOptions IgnoreKanaType and IgnoreWidth on Unix.
Diffstat (limited to 'src/corefx')
-rw-r--r-- | src/corefx/System.Globalization.Native/collation.cpp | 163 |
1 files changed, 160 insertions, 3 deletions
diff --git a/src/corefx/System.Globalization.Native/collation.cpp b/src/corefx/System.Globalization.Native/collation.cpp index c305ccffff..f250b63473 100644 --- a/src/corefx/System.Globalization.Native/collation.cpp +++ b/src/corefx/System.Globalization.Native/collation.cpp @@ -6,6 +6,7 @@ #include <assert.h> #include <stdint.h> +#include <vector> #include <map> #include <unicode/uchar.h> #include <unicode/ucol.h> @@ -15,8 +16,8 @@ const int32_t CompareOptionsIgnoreCase = 1; const int32_t CompareOptionsIgnoreNonSpace = 2; const int32_t CompareOptionsIgnoreSymbols = 4; -// const int32_t CompareOptionsIgnoreKanaType = 8; -// const int32_t CompareOptionsIgnoreWidth = 0x10; +const int32_t CompareOptionsIgnoreKanaType = 8; +const int32_t CompareOptionsIgnoreWidth = 0x10; // const int32_t CompareOptionsStringSort = 0x20000000; typedef std::map<int32_t, UCollator*> TCollatorMap; @@ -38,6 +39,138 @@ typedef struct _sort_handle } SortHandle; +// Hiragana character range +const UChar hiraganaStart = 0x3041; +const UChar hiraganaEnd = 0x309e; +const UChar hiraganaToKatakanaOffset = 0x30a1 - 0x3041; + +// Mapping between half- and fullwidth characters. +// LowerChars are the characters that should sort lower than HigherChars +const UChar g_HalfFullLowerChars[] = { + // halfwidth characters + 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, + 0x003f, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, + 0x004e, 0x004f, 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005d, + 0x005e, 0x005f, 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, + 0x006d, 0x006e, 0x006f, 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x00a2, 0x00a3, 0x00ac, 0x00af, 0x00a6, 0x00a5, 0x20a9, + + // fullwidth characters + 0x3002, 0x300c, 0x300d, 0x3001, 0x30fb, 0x30f2, 0x30a1, 0x30a3, 0x30a5, 0x30a7, 0x30a9, 0x30e3, 0x30e5, 0x30e7, 0x30c3, + 0x30fc, 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab, 0x30ad, 0x30af, 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, + 0x30bd, 0x30bf, 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd, 0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, + 0x30db, 0x30de, 0x30df, 0x30e0, 0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, 0x30ea, 0x30eb, 0x30ec, 0x30ed, 0x30ef, + 0x30f3, 0x3099, 0x309a, 0x3164, 0x3131, 0x3132, 0x3133, 0x3134, 0x3135, 0x3136, 0x3137, 0x3138, 0x3139, 0x313a, 0x313b, + 0x313c, 0x313d, 0x313e, 0x313f, 0x3140, 0x3141, 0x3142, 0x3143, 0x3144, 0x3145, 0x3146, 0x3147, 0x3148, 0x3149, 0x314a, + 0x314b, 0x314c, 0x314d, 0x314e, 0x314f, 0x3150, 0x3151, 0x3152, 0x3153, 0x3154, 0x3155, 0x3156, 0x3157, 0x3158, 0x3159, + 0x315a, 0x315b, 0x315c, 0x315d, 0x315e, 0x315f, 0x3160, 0x3161, 0x3162, 0x3163 +}; +const UChar g_HalfFullHigherChars[] = { + // fullwidth characters + 0xff01, 0xff02, 0xff03, 0xff04, 0xff05, 0xff06, 0xff07, 0xff08, 0xff09, 0xff0a, 0xff0b, 0xff0c, 0xff0d, 0xff0e, 0xff0f, + 0xff10, 0xff11, 0xff12, 0xff13, 0xff14, 0xff15, 0xff16, 0xff17, 0xff18, 0xff19, 0xff1a, 0xff1b, 0xff1c, 0xff1d, 0xff1e, + 0xff1f, 0xff20, 0xff21, 0xff22, 0xff23, 0xff24, 0xff25, 0xff26, 0xff27, 0xff28, 0xff29, 0xff2a, 0xff2b, 0xff2c, 0xff2d, + 0xff2e, 0xff2f, 0xff30, 0xff31, 0xff32, 0xff33, 0xff34, 0xff35, 0xff36, 0xff37, 0xff38, 0xff39, 0xff3a, 0xff3b, 0xff3d, + 0xff3e, 0xff3f, 0xff40, 0xff41, 0xff42, 0xff43, 0xff44, 0xff45, 0xff46, 0xff47, 0xff48, 0xff49, 0xff4a, 0xff4b, 0xff4c, + 0xff4d, 0xff4e, 0xff4f, 0xff50, 0xff51, 0xff52, 0xff53, 0xff54, 0xff55, 0xff56, 0xff57, 0xff58, 0xff59, 0xff5a, 0xff5b, + 0xff5c, 0xff5d, 0xff5e, 0xffe0, 0xffe1, 0xffe2, 0xffe3, 0xffe4, 0xffe5, 0xffe6, + + // halfwidth characters + 0xff61, 0xff62, 0xff63, 0xff64, 0xff65, 0xff66, 0xff67, 0xff68, 0xff69, 0xff6a, 0xff6b, 0xff6c, 0xff6d, 0xff6e, 0xff6f, + 0xff70, 0xff71, 0xff72, 0xff73, 0xff74, 0xff75, 0xff76, 0xff77, 0xff78, 0xff79, 0xff7a, 0xff7b, 0xff7c, 0xff7d, 0xff7e, + 0xff7f, 0xff80, 0xff81, 0xff82, 0xff83, 0xff84, 0xff85, 0xff86, 0xff87, 0xff88, 0xff89, 0xff8a, 0xff8b, 0xff8c, 0xff8d, + 0xff8e, 0xff8f, 0xff90, 0xff91, 0xff92, 0xff93, 0xff94, 0xff95, 0xff96, 0xff97, 0xff98, 0xff99, 0xff9a, 0xff9b, 0xff9c, + 0xff9d, 0xff9e, 0xff9f, 0xffa0, 0xffa1, 0xffa2, 0xffa3, 0xffa4, 0xffa5, 0xffa6, 0xffa7, 0xffa8, 0xffa9, 0xffaa, 0xffab, + 0xffac, 0xffad, 0xffae, 0xffaf, 0xffb0, 0xffb1, 0xffb2, 0xffb3, 0xffb4, 0xffb5, 0xffb6, 0xffb7, 0xffb8, 0xffb9, 0xffba, + 0xffbb, 0xffbc, 0xffbd, 0xffbe, 0xffc2, 0xffc3, 0xffc4, 0xffc5, 0xffc6, 0xffc7, 0xffca, 0xffcb, 0xffcc, 0xffcd, 0xffce, + 0xffcf, 0xffd2, 0xffd3, 0xffd4, 0xffd5, 0xffd6, 0xffd7, 0xffda, 0xffdb, 0xffdc +}; +const int32_t g_HalfFullCharsLength = (sizeof(g_HalfFullHigherChars) / sizeof(UChar)); + +/* +ICU collation rules reserve any punctuation and whitespace characters for use in the syntax. +Thus, to use these characters in a rule, they need to be escaped. +*/ +bool NeedsEscape(UChar character) +{ + return ((0x21 <= character && character <= 0x2f) + || (0x3a <= character && character <= 0x40) + || (0x5b <= character && character <= 0x60) + || (0x7b <= character && character <= 0x7e)); +} + +/* +Gets a string of custom collation rules, if necessary. + +Since the CompareOptions flags don't map 1:1 with ICU default functionality, we need to fall back to using +custom rules in order to support IgnoreKanaType and IgnoreWidth CompareOptions correctly. +*/ +std::vector<UChar> GetCustomRules(int32_t options, UColAttributeValue strength) +{ + bool isIgnoreKanaType = (options & CompareOptionsIgnoreKanaType) == CompareOptionsIgnoreKanaType; + bool isIgnoreWidth = (options & CompareOptionsIgnoreWidth) == CompareOptionsIgnoreWidth; + + // kana differs at the quaternary level + bool needsIgnoreKanaTypeCustomRule = isIgnoreKanaType && (strength == UCOL_DEFAULT || strength >= UCOL_QUATERNARY); + bool needsNotIgnoreKanaTypeCustomRule = !isIgnoreKanaType && (strength != UCOL_DEFAULT && strength < UCOL_QUATERNARY); + + // character width differs at the tertiary/default level + bool needsIgnoreWidthCustomRule = isIgnoreWidth && (strength == UCOL_DEFAULT || strength >= UCOL_TERTIARY); + bool needsNotIgnoreWidthCustomRule = !isIgnoreWidth && (strength != UCOL_DEFAULT && strength < UCOL_TERTIARY); + + std::vector<UChar> customRules; + if (needsIgnoreKanaTypeCustomRule || needsNotIgnoreKanaTypeCustomRule || needsIgnoreWidthCustomRule || needsNotIgnoreWidthCustomRule) + { + // If we need to create customRules, the KanaType custom rule will be 88 kana characters * 4 = 352 chars long + // and the Width custom rule will be at least 215 halfwidth characters * 4 = 860 chars long. + // Use 512 as the starting size, so the customRules won't have to grow if we are just + // doing the KanaType custom rule. + customRules.reserve(512); + + if (needsIgnoreKanaTypeCustomRule || needsNotIgnoreKanaTypeCustomRule) + { + UChar compareChar = needsIgnoreKanaTypeCustomRule ? '=' : '<'; + + for (UChar hiraganaChar = hiraganaStart; hiraganaChar <= hiraganaEnd; hiraganaChar++) + { + // Hiragana is the range 3041 to 3096 & 309D & 309E + if (hiraganaChar <= 0x3096 || hiraganaChar >= 0x309D) // characters between 3096 and 309D are not mapped to katakana + { + customRules.push_back('&'); + customRules.push_back(hiraganaChar); + customRules.push_back(compareChar); + customRules.push_back(hiraganaChar + hiraganaToKatakanaOffset); + } + } + } + + if (needsIgnoreWidthCustomRule || needsNotIgnoreWidthCustomRule) + { + UChar compareChar = needsIgnoreWidthCustomRule ? '=' : '<'; + + UChar lowerChar; + for (int i = 0; i < g_HalfFullCharsLength; i++) + { + customRules.push_back('&'); + + // the lower chars need to be checked for escaping since they contain ASCII punctuation + lowerChar = g_HalfFullLowerChars[i]; + if (NeedsEscape(lowerChar)) + { + customRules.push_back('\\'); + } + customRules.push_back(lowerChar); + + customRules.push_back(compareChar); + customRules.push_back(g_HalfFullHigherChars[i]); + } + } + } + + return customRules; +} + /* * The collator returned by this function is owned by the callee and must be * closed when this method returns with a U_SUCCESS UErrorCode. @@ -62,7 +195,31 @@ UCollator* CloneCollatorWithOptions(const UCollator* pCollator, int32_t options, strength = UCOL_PRIMARY; } - UCollator* pClonedCollator = ucol_safeClone(pCollator, nullptr, nullptr, pErr); + UCollator* pClonedCollator; + std::vector<UChar> customRules = GetCustomRules(options, strength); + if (customRules.empty()) + { + pClonedCollator = ucol_safeClone(pCollator, nullptr, nullptr, pErr); + } + else + { + int32_t customRuleLength = customRules.size(); + + int32_t localeRulesLength; + const UChar* localeRules = ucol_getRules(pCollator, &localeRulesLength); + + std::vector<UChar> completeRules(localeRulesLength + customRuleLength + 1, '\0'); + for (int i = 0; i < localeRulesLength; i++) + { + completeRules[i] = localeRules[i]; + } + for (int i = 0; i < customRuleLength; i++) + { + completeRules[localeRulesLength + i] = customRules[i]; + } + + pClonedCollator = ucol_openRules(completeRules.data(), completeRules.size(), UCOL_DEFAULT, strength, NULL, pErr); + } if (isIgnoreSymbols) { |