< Summary

Line coverage
0%
Covered lines: 0
Uncovered lines: 193
Coverable lines: 193
Total lines: 591
Line coverage: 0%
Branch coverage
0%
Covered branches: 0
Total branches: 186
Branch coverage: 0%
Method coverage

Feature is only available for sponsors

Upgrade to PRO version

Metrics

File(s)

https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Globalization/Normalization.cs

#LineLine coverage
 1// Licensed to the .NET Foundation under one or more agreements.
 2// The .NET Foundation licenses this file to you under the MIT license.
 3
 4using System.Runtime.CompilerServices;
 5using System.Text;
 6
 7namespace System.Globalization
 8{
 9    internal static partial class Normalization
 10    {
 11        internal static bool IsNormalized(ReadOnlySpan<char> source, NormalizationForm normalizationForm = Normalization
 12        {
 013            CheckNormalizationForm(normalizationForm);
 14
 15            // In Invariant mode we assume all characters are normalized because we don't support any linguistic operati
 16            // If it's ASCII && one of the 4 main forms, then it's already normalized.
 017            if (GlobalizationMode.Invariant || Ascii.IsValid(source))
 18            {
 019                return true;
 20            }
 21
 022            return GlobalizationMode.UseNls ?
 023                NlsIsNormalized(source, normalizationForm) :
 024                IcuIsNormalized(source, normalizationForm);
 25        }
 26
 27        internal static string Normalize(string strInput, NormalizationForm normalizationForm)
 28        {
 029            CheckNormalizationForm(normalizationForm);
 30
 31            // In Invariant mode we assume all characters are normalized because we don't support any linguistic operati
 32            // If it's ASCII && one of the 4 main forms, then it's already normalized.
 033            if (GlobalizationMode.Invariant || Ascii.IsValid(strInput))
 34            {
 035                return strInput;
 36            }
 37
 038            return GlobalizationMode.UseNls ?
 039                NlsNormalize(strInput, normalizationForm) :
 040                IcuNormalize(strInput, normalizationForm);
 41        }
 42
 43        internal static bool TryNormalize(ReadOnlySpan<char> source, Span<char> destination, out int charsWritten, Norma
 44        {
 045            CheckNormalizationForm(normalizationForm);
 46
 047            if (source.Overlaps(destination))
 48            {
 049                ThrowHelper.ThrowArgumentException(ExceptionResource.InvalidOperation_SpanOverlappedOperation);
 50            }
 51
 52            // In Invariant mode we assume all characters are normalized because we don't support any linguistic operati
 53            // If it's ASCII && one of the 4 main forms, then it's already normalized.
 054            if (GlobalizationMode.Invariant || Ascii.IsValid(source))
 55            {
 056                if (source.TryCopyTo(destination))
 57                {
 058                    charsWritten = source.Length;
 059                    return true;
 60                }
 61
 062                charsWritten = 0;
 063                return false;
 64            }
 65
 066            return GlobalizationMode.UseNls ?
 067                NlsTryNormalize(source, destination, out charsWritten, normalizationForm) :
 068                IcuTryNormalize(source, destination, out charsWritten, normalizationForm);
 69        }
 70
 71        internal static int GetNormalizedLength(this ReadOnlySpan<char> source, NormalizationForm normalizationForm = No
 72        {
 073            CheckNormalizationForm(normalizationForm);
 74
 75            // In Invariant mode we assume all characters are normalized because we don't support any linguistic operati
 76            // If it's ASCII && one of the 4 main forms, then it's already normalized.
 077            if (GlobalizationMode.Invariant || Ascii.IsValid(source))
 78            {
 079                return source.Length;
 80            }
 81
 082            return GlobalizationMode.UseNls ?
 083                NlsGetNormalizedLength(source, normalizationForm) :
 084                IcuGetNormalizedLength(source, normalizationForm);
 85        }
 86
 87        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 88        private static void CheckNormalizationForm(NormalizationForm normalizationForm)
 89        {
 90            if (normalizationForm != NormalizationForm.FormC &&
 91                normalizationForm != NormalizationForm.FormD &&
 92                normalizationForm != NormalizationForm.FormKC &&
 93                normalizationForm != NormalizationForm.FormKD)
 94            {
 095                throw new ArgumentException(SR.Argument_InvalidNormalizationForm, nameof(normalizationForm));
 96            }
 97
 98            if ((OperatingSystem.IsBrowser() || OperatingSystem.IsWasi()) &&
 99                !GlobalizationMode.Invariant &&
 100                !GlobalizationMode.UseNls &&
 101                (normalizationForm == NormalizationForm.FormKC || normalizationForm == NormalizationForm.FormKD))
 102            {
 103                // Browser/WASI builds ship without compatibility normalization data.
 104                throw new PlatformNotSupportedException(SR.Argument_UnsupportedNormalizationFormInBrowser);
 105            }
 0106        }
 107    }
 108}
 109

https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Globalization/Normalization.Icu.cs

#LineLine coverage
 1// Licensed to the .NET Foundation under one or more agreements.
 2// The .NET Foundation licenses this file to you under the MIT license.
 3
 4using System.Buffers;
 5using System.Diagnostics;
 6using System.Runtime.InteropServices;
 7using System.Text;
 8
 9namespace System.Globalization
 10{
 11    internal static partial class Normalization
 12    {
 13        private static unsafe bool IcuIsNormalized(ReadOnlySpan<char> source, NormalizationForm normalizationForm)
 14        {
 015            Debug.Assert(!GlobalizationMode.Invariant);
 016            Debug.Assert(!GlobalizationMode.UseNls);
 017            Debug.Assert(!source.IsEmpty);
 18#pragma warning disable CA1416 // FormKC and FormKD are unsupported on browser, CheckNormalizationForm is throwing Platf
 019            Debug.Assert(normalizationForm is NormalizationForm.FormC or NormalizationForm.FormD or NormalizationForm.Fo
 20#pragma warning restore CA1416
 21
 022            ValidateArguments(source, nameof(source));
 23
 24            int ret;
 025            fixed (char* pInput = source)
 26            {
 27#if TARGET_MACCATALYST || TARGET_IOS || TARGET_TVOS
 28                if (GlobalizationMode.Hybrid)
 29                {
 30                    ret = Interop.Globalization.IsNormalizedNative(normalizationForm, pInput, source.Length);
 31                }
 32                else
 33#endif
 34                {
 035                    ret = Interop.Globalization.IsNormalized(normalizationForm, pInput, source.Length);
 36                }
 37            }
 38
 039            if (ret == -1)
 40            {
 041                throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(source));
 42            }
 43
 044            return ret == 1;
 45        }
 46
 47        private static unsafe string IcuNormalize(string strInput, NormalizationForm normalizationForm)
 48        {
 049            Debug.Assert(!GlobalizationMode.Invariant);
 050            Debug.Assert(!GlobalizationMode.UseNls);
 051            Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD ||
 52
 053            ValidateArguments(strInput);
 54
 055            char[]? toReturn = null;
 56            try
 57            {
 58                const int StackallocThreshold = 512;
 59
 060                Span<char> buffer = strInput.Length <= StackallocThreshold
 061                    ? stackalloc char[StackallocThreshold]
 062                    : (toReturn = ArrayPool<char>.Shared.Rent(strInput.Length));
 63
 064                for (int attempt = 0; attempt < 2; attempt++)
 65                {
 66                    int realLen;
 067                    fixed (char* pInput = strInput)
 068                    fixed (char* pDest = &MemoryMarshal.GetReference(buffer))
 69                    {
 70#if TARGET_MACCATALYST || TARGET_IOS || TARGET_TVOS
 71                        if (GlobalizationMode.Hybrid)
 72                        {
 73                            realLen = Interop.Globalization.NormalizeStringNative(normalizationForm, pInput, strInput.Le
 74                        }
 75                        else
 76#endif
 77                        {
 078                            realLen = Interop.Globalization.NormalizeString(normalizationForm, pInput, strInput.Length, 
 79                        }
 80                    }
 81
 082                    if (realLen == -1)
 83                    {
 084                        throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(strInput));
 85                    }
 86
 087                    if (realLen <= buffer.Length)
 88                    {
 089                        ReadOnlySpan<char> result = buffer.Slice(0, realLen);
 090                        return result.SequenceEqual(strInput)
 091                            ? strInput
 092                            : new string(result);
 93                    }
 94
 095                    Debug.Assert(realLen > StackallocThreshold);
 96
 097                    if (attempt == 0)
 98                    {
 099                        if (toReturn != null)
 100                        {
 101                            // Clear toReturn first to ensure we don't return the same buffer twice
 0102                            char[] temp = toReturn;
 0103                            toReturn = null;
 0104                            ArrayPool<char>.Shared.Return(temp);
 105                        }
 106
 0107                        buffer = toReturn = ArrayPool<char>.Shared.Rent(realLen);
 108                    }
 109                }
 110
 0111                throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(strInput));
 112            }
 113            finally
 114            {
 0115                if (toReturn != null)
 116                {
 0117                    ArrayPool<char>.Shared.Return(toReturn);
 118                }
 0119            }
 0120        }
 121
 122        private static unsafe bool IcuTryNormalize(ReadOnlySpan<char> source, Span<char> destination, out int charsWritt
 123        {
 0124            Debug.Assert(!GlobalizationMode.Invariant);
 0125            Debug.Assert(!GlobalizationMode.UseNls);
 0126            Debug.Assert(!source.IsEmpty);
 0127            Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD ||
 128
 0129            if (destination.IsEmpty)
 130            {
 0131                charsWritten = 0;
 0132                return false;
 133            }
 134
 0135            ValidateArguments(source, nameof(source));
 136
 137            int realLen;
 0138            fixed (char* pInput = source)
 0139            fixed (char* pDest = destination)
 140            {
 141#if TARGET_MACCATALYST || TARGET_IOS || TARGET_TVOS
 142                if (GlobalizationMode.Hybrid)
 143                {
 144                    realLen = Interop.Globalization.NormalizeStringNative(normalizationForm, pInput, source.Length, pDes
 145                }
 146                else
 147#endif
 148                {
 0149                    realLen = Interop.Globalization.NormalizeString(normalizationForm, pInput, source.Length, pDest, des
 150                }
 151            }
 152
 0153            if (realLen < 0)
 154            {
 0155                throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(source));
 156            }
 157
 0158            if (realLen <= destination.Length)
 159            {
 0160                charsWritten = realLen;
 0161                return true;
 162            }
 163
 0164            charsWritten = 0;
 0165            return false;
 166        }
 167
 168        private static unsafe int IcuGetNormalizedLength(ReadOnlySpan<char> source, NormalizationForm normalizationForm)
 169        {
 0170            Debug.Assert(!GlobalizationMode.Invariant);
 0171            Debug.Assert(!GlobalizationMode.UseNls);
 0172            Debug.Assert(!source.IsEmpty);
 0173            Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD ||
 174
 0175            ValidateArguments(source, nameof(source));
 176
 177            int realLen;
 0178            fixed (char* pInput = source)
 179            {
 180#if TARGET_MACCATALYST || TARGET_IOS || TARGET_TVOS
 181                if (GlobalizationMode.Hybrid)
 182                {
 183                    realLen = Interop.Globalization.NormalizeStringNative(normalizationForm, pInput, source.Length, null
 184                }
 185                else
 186#endif
 187                {
 0188                    realLen = Interop.Globalization.NormalizeString(normalizationForm, pInput, source.Length, null, 0);
 189                }
 190            }
 191
 0192            if (realLen < 0)
 193            {
 0194                throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(source));
 195            }
 196
 0197            return realLen;
 198        }
 199
 200        private static void ValidateArguments(ReadOnlySpan<char> strInput, string paramName = "strInput")
 201        {
 0202            if (HasInvalidUnicodeSequence(strInput))
 203            {
 0204                throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, paramName);
 205            }
 0206        }
 207
 208        /// <summary>
 209        /// ICU does not signal an error during normalization if the input string has invalid unicode,
 210        /// unlike Windows (which uses the ERROR_NO_UNICODE_TRANSLATION error value to signal an error).
 211        ///
 212        /// We walk the string ourselves looking for these bad sequences so we can continue to throw
 213        /// ArgumentException in these cases.
 214        /// </summary>
 215        private static bool HasInvalidUnicodeSequence(ReadOnlySpan<char> s)
 216        {
 217            const char Noncharacter = '\uFFFE';
 218
 0219            int i = s.IndexOfAnyInRange(CharUnicodeInfo.HIGH_SURROGATE_START, Noncharacter);
 220
 0221            for (; (uint)i < (uint)s.Length; i++)
 222            {
 0223                char c = s[i];
 224
 0225                if (c < CharUnicodeInfo.HIGH_SURROGATE_START)
 226                {
 227                    continue;
 228                }
 229
 0230                if (c == Noncharacter)
 231                {
 0232                    return true;
 233                }
 234
 235                // If we see low surrogate before a high one, the string is invalid.
 0236                if (char.IsLowSurrogate(c))
 237                {
 0238                    return true;
 239                }
 240
 0241                if (char.IsHighSurrogate(c))
 242                {
 0243                    if ((uint)(i + 1) >= (uint)s.Length || !char.IsLowSurrogate(s[i + 1]))
 244                    {
 245                        // A high surrogate at the end of the string or a high surrogate
 246                        // not followed by a low surrogate
 0247                        return true;
 248                    }
 249
 0250                    i++; // consume the low surrogate.
 251                }
 252            }
 253
 0254            return false;
 255        }
 256    }
 257}
 258

https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Globalization/Normalization.Nls.cs

#LineLine coverage
 1// Licensed to the .NET Foundation under one or more agreements.
 2// The .NET Foundation licenses this file to you under the MIT license.
 3
 4using System.Buffers;
 5using System.Diagnostics;
 6using System.Runtime.CompilerServices;
 7using System.Runtime.InteropServices;
 8using System.Text;
 9
 10namespace System.Globalization
 11{
 12    internal static partial class Normalization
 13    {
 14        private static unsafe bool NlsIsNormalized(ReadOnlySpan<char> source, NormalizationForm normalizationForm)
 15        {
 016            Debug.Assert(!GlobalizationMode.Invariant);
 017            Debug.Assert(GlobalizationMode.UseNls);
 018            Debug.Assert(!source.IsEmpty);
 019            Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD ||
 20
 21            Interop.BOOL result;
 022            fixed (char* pInput = source)
 23            {
 024                result = Interop.Normaliz.IsNormalizedString(normalizationForm, pInput, source.Length);
 25            }
 26
 27            // The only way to know if IsNormalizedString failed is through checking the Win32 last error
 28            // IsNormalizedString pinvoke has SetLastError attribute property which will set the last error
 29            // to 0 (ERROR_SUCCESS) before executing the calls.
 030            CheckLastErrorAndThrowIfFailed(nameof(source));
 31
 032            return result != Interop.BOOL.FALSE;
 33        }
 34
 35        private static unsafe string NlsNormalize(string strInput, NormalizationForm normalizationForm)
 36        {
 037            Debug.Assert(!GlobalizationMode.Invariant);
 038            Debug.Assert(GlobalizationMode.UseNls);
 039            Debug.Assert(strInput != null);
 040            Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD ||
 41
 042            if (strInput.Length == 0)
 43            {
 044                return string.Empty;
 45            }
 46
 047            char[]? toReturn = null;
 48            try
 49            {
 50                const int StackallocThreshold = 512;
 51
 052                Span<char> buffer = strInput.Length <= StackallocThreshold
 053                    ? stackalloc char[StackallocThreshold]
 054                    : (toReturn = ArrayPool<char>.Shared.Rent(strInput.Length));
 55
 56                while (true)
 57                {
 58                    // we depend on Win32 last error when calling NormalizeString
 59                    // NormalizeString pinvoke has SetLastError attribute property which will set the last error
 60                    // to 0 (ERROR_SUCCESS) before executing the calls.
 61                    int realLength;
 062                    fixed (char* pInput = strInput)
 063                    fixed (char* pDest = &MemoryMarshal.GetReference(buffer))
 64                    {
 065                        realLength = Interop.Normaliz.NormalizeString(normalizationForm, pInput, strInput.Length, pDest,
 66                    }
 067                    int lastError = Marshal.GetLastPInvokeError();
 68
 69                    switch (lastError)
 70                    {
 71                        case Interop.Errors.ERROR_SUCCESS:
 072                            ReadOnlySpan<char> result = buffer.Slice(0, realLength);
 073                            return result.SequenceEqual(strInput)
 074                                ? strInput
 075                                : new string(result);
 76
 77                        // Do appropriate stuff for the individual errors:
 78                        case Interop.Errors.ERROR_INSUFFICIENT_BUFFER:
 079                            realLength = Math.Abs(realLength);
 080                            Debug.Assert(realLength > buffer.Length, "Buffer overflow should have iLength > cBuffer.Leng
 081                            if (toReturn != null)
 82                            {
 83                                // Clear toReturn first to ensure we don't return the same buffer twice
 084                                char[] temp = toReturn;
 085                                toReturn = null;
 086                                ArrayPool<char>.Shared.Return(temp);
 87                            }
 088                            Debug.Assert(realLength > StackallocThreshold);
 089                            buffer = toReturn = ArrayPool<char>.Shared.Rent(realLength);
 090                            continue;
 91
 92                        case Interop.Errors.ERROR_INVALID_PARAMETER:
 93                        case Interop.Errors.ERROR_NO_UNICODE_TRANSLATION:
 94                            // Illegal code point or order found.  Ie: FFFE or D800 D800, etc.
 095                            throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(strInput));
 96
 97                        case Interop.Errors.ERROR_NOT_ENOUGH_MEMORY:
 098                            throw new OutOfMemoryException();
 99
 100                        default:
 101                            // We shouldn't get here...
 0102                            throw new InvalidOperationException(SR.Format(SR.UnknownError_Num, lastError));
 103                    }
 104                }
 105            }
 106            finally
 107            {
 0108                if (toReturn != null)
 109                {
 0110                    ArrayPool<char>.Shared.Return(toReturn);
 111                }
 0112            }
 0113        }
 114
 115        private static unsafe bool NlsTryNormalize(ReadOnlySpan<char> source, Span<char> destination, out int charsWritt
 116        {
 0117            Debug.Assert(!GlobalizationMode.Invariant);
 0118            Debug.Assert(GlobalizationMode.UseNls);
 0119            Debug.Assert(!source.IsEmpty);
 0120            Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD ||
 121
 0122            if (destination.IsEmpty)
 123            {
 0124                charsWritten = 0;
 0125                return false;
 126            }
 127
 128            // we depend on Win32 last error when calling NormalizeString
 129            // NormalizeString pinvoke has SetLastError attribute property which will set the last error
 130            // to 0 (ERROR_SUCCESS) before executing the calls.
 131
 132            int realLength;
 0133            fixed (char* pInput = source)
 0134            fixed (char* pDest = destination)
 135            {
 0136                realLength = Interop.Normaliz.NormalizeString(normalizationForm, pInput, source.Length, pDest, destinati
 137            }
 138
 0139            int lastError = Marshal.GetLastPInvokeError();
 140            switch (lastError)
 141            {
 142                case Interop.Errors.ERROR_SUCCESS:
 0143                    charsWritten = realLength;
 0144                    return true;
 145
 146                // Do appropriate stuff for the individual errors:
 147                case Interop.Errors.ERROR_INSUFFICIENT_BUFFER:
 0148                    charsWritten = 0;
 0149                    return false;
 150
 151                case Interop.Errors.ERROR_INVALID_PARAMETER:
 152                case Interop.Errors.ERROR_NO_UNICODE_TRANSLATION:
 153                    // Illegal code point or order found.  Ie: FFFE or D800 D800, etc.
 0154                    throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(source));
 155
 156                case Interop.Errors.ERROR_NOT_ENOUGH_MEMORY:
 0157                    throw new OutOfMemoryException();
 158
 159                default:
 160                    // We shouldn't get here...
 0161                    throw new InvalidOperationException(SR.Format(SR.UnknownError_Num, lastError));
 162            }
 163        }
 164
 165        private static unsafe int NlsGetNormalizedLength(ReadOnlySpan<char> source, NormalizationForm normalizationForm)
 166        {
 0167            Debug.Assert(!GlobalizationMode.Invariant);
 0168            Debug.Assert(GlobalizationMode.UseNls);
 0169            Debug.Assert(!source.IsEmpty);
 0170            Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD ||
 171
 172            // we depend on Win32 last error when calling NormalizeString
 173            // NormalizeString pinvoke has SetLastError attribute property which will set the last error
 174            // to 0 (ERROR_SUCCESS) before executing the calls.
 175
 176            int realLength;
 0177            fixed (char* pInput = source)
 178            {
 0179                realLength = Interop.Normaliz.NormalizeString(normalizationForm, pInput, source.Length, null, 0);
 180            }
 181
 0182            int lastError = Marshal.GetLastPInvokeError();
 183            switch (lastError)
 184            {
 185                case Interop.Errors.ERROR_SUCCESS:
 0186                    return realLength;
 187
 188                case Interop.Errors.ERROR_INVALID_PARAMETER:
 189                case Interop.Errors.ERROR_NO_UNICODE_TRANSLATION:
 190                    // Illegal code point or order found.  Ie: FFFE or D800 D800, etc.
 0191                    throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(source));
 192
 193                case Interop.Errors.ERROR_NOT_ENOUGH_MEMORY:
 0194                    throw new OutOfMemoryException();
 195
 196                default:
 197                    // We shouldn't get here...
 0198                    throw new InvalidOperationException(SR.Format(SR.UnknownError_Num, lastError));
 199            }
 200        }
 201
 202        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 203        private static void CheckLastErrorAndThrowIfFailed(string inputName)
 204        {
 0205            int lastError = Marshal.GetLastPInvokeError();
 206            switch (lastError)
 207            {
 208                case Interop.Errors.ERROR_SUCCESS:
 209                    break;
 210
 211                case Interop.Errors.ERROR_INVALID_PARAMETER:
 212                case Interop.Errors.ERROR_NO_UNICODE_TRANSLATION:
 0213                    throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, inputName);
 214
 215                case Interop.Errors.ERROR_NOT_ENOUGH_MEMORY:
 0216                    throw new OutOfMemoryException();
 217
 218                default:
 0219                    throw new InvalidOperationException(SR.Format(SR.UnknownError_Num, lastError));
 220            }
 0221        }
 222    }
 223}
 224