| | | 1 | | // Licensed to the .NET Foundation under one or more agreements. |
| | | 2 | | // The .NET Foundation licenses this file to you under the MIT license. |
| | | 3 | | |
| | | 4 | | using System.Diagnostics; |
| | | 5 | | using System.Runtime.CompilerServices; |
| | | 6 | | using System.Runtime.InteropServices; |
| | | 7 | | using System.Text; |
| | | 8 | | using System.Diagnostics.CodeAnalysis; |
| | | 9 | | |
| | | 10 | | #if NET |
| | | 11 | | using System.Runtime.Intrinsics.Arm; |
| | | 12 | | using System.Runtime.Intrinsics.Wasm; |
| | | 13 | | using System.Runtime.Intrinsics.X86; |
| | | 14 | | using System.Runtime.Intrinsics; |
| | | 15 | | #endif |
| | | 16 | | |
| | | 17 | | namespace System.Buffers.Text |
| | | 18 | | { |
| | | 19 | | // AVX2 version based on https://github.com/aklomp/base64/tree/e516d769a2a432c08404f1981e73b431566057be/lib/arch/avx |
| | | 20 | | // Vector128 version based on https://github.com/aklomp/base64/tree/e516d769a2a432c08404f1981e73b431566057be/lib/arc |
| | | 21 | | internal static partial class Base64Helper |
| | | 22 | | { |
| | | 23 | | internal static unsafe OperationStatus DecodeFrom<TBase64Decoder, T>(TBase64Decoder decoder, ReadOnlySpan<T> sou |
| | | 24 | | out int bytesConsumed, out int bytesWritten, bool isFinalBlock, bool ignoreWhiteSpace) |
| | | 25 | | where TBase64Decoder : IBase64Decoder<T> |
| | | 26 | | where T : unmanaged |
| | | 27 | | { |
| | | 28 | | if (source.IsEmpty) |
| | | 29 | | { |
| | 43 | 30 | | bytesConsumed = 0; |
| | 43 | 31 | | bytesWritten = 0; |
| | 43 | 32 | | return OperationStatus.Done; |
| | | 33 | | } |
| | | 34 | | |
| | 74682 | 35 | | fixed (T* srcBytes = &MemoryMarshal.GetReference(source)) |
| | 74682 | 36 | | fixed (byte* destBytes = &MemoryMarshal.GetReference(bytes)) |
| | | 37 | | { |
| | 74682 | 38 | | int srcLength = decoder.SrcLength(isFinalBlock, source.Length); |
| | 74682 | 39 | | int destLength = bytes.Length; |
| | 74682 | 40 | | int maxSrcLength = srcLength; |
| | 74682 | 41 | | int decodedLength = decoder.GetMaxDecodedLength(srcLength); |
| | | 42 | | |
| | | 43 | | // max. 2 padding chars |
| | 74682 | 44 | | if (destLength < decodedLength - 2) |
| | | 45 | | { |
| | | 46 | | // For overflow see comment below |
| | 0 | 47 | | maxSrcLength = destLength / 3 * 4; |
| | | 48 | | } |
| | | 49 | | |
| | 74682 | 50 | | T* src = srcBytes; |
| | 74682 | 51 | | byte* dest = destBytes; |
| | 74682 | 52 | | T* srcEnd = srcBytes + (uint)srcLength; |
| | 74682 | 53 | | T* srcMax = srcBytes + (uint)maxSrcLength; |
| | | 54 | | |
| | | 55 | | #if NET |
| | 74682 | 56 | | if (maxSrcLength >= 24) |
| | | 57 | | { |
| | 32054 | 58 | | T* end = srcMax - 88; |
| | 32054 | 59 | | if (Vector512.IsHardwareAccelerated && Avx512Vbmi.IsSupported && (end >= src)) |
| | | 60 | | { |
| | 27869 | 61 | | Avx512Decode(decoder, ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); |
| | | 62 | | |
| | 27869 | 63 | | if (src == srcEnd) |
| | | 64 | | { |
| | | 65 | | goto DoneExit; |
| | | 66 | | } |
| | | 67 | | } |
| | | 68 | | |
| | 32054 | 69 | | end = srcMax - 45; |
| | 32054 | 70 | | if (Avx2.IsSupported && (end >= src)) |
| | | 71 | | { |
| | 29949 | 72 | | Avx2Decode(decoder, ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); |
| | | 73 | | |
| | 29949 | 74 | | if (src == srcEnd) |
| | | 75 | | { |
| | | 76 | | goto DoneExit; |
| | | 77 | | } |
| | | 78 | | } |
| | | 79 | | |
| | 32054 | 80 | | end = srcMax - 66; |
| | | 81 | | if (AdvSimd.Arm64.IsSupported && (end >= src)) |
| | | 82 | | { |
| | | 83 | | AdvSimdDecode(decoder, ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); |
| | | 84 | | |
| | | 85 | | if (src == srcEnd) |
| | | 86 | | { |
| | | 87 | | goto DoneExit; |
| | | 88 | | } |
| | | 89 | | } |
| | | 90 | | |
| | 32054 | 91 | | end = srcMax - 24; |
| | 32054 | 92 | | if ((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported || PackedSimd.IsSupported) && BitConverter.IsLit |
| | | 93 | | { |
| | 31314 | 94 | | Vector128Decode(decoder, ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); |
| | | 95 | | |
| | 31314 | 96 | | if (src == srcEnd) |
| | | 97 | | { |
| | | 98 | | goto DoneExit; |
| | | 99 | | } |
| | | 100 | | } |
| | | 101 | | } |
| | | 102 | | #endif |
| | | 103 | | |
| | | 104 | | // Last bytes could have padding characters, so process them separately and treat them as valid only if |
| | | 105 | | // if isFinalBlock is false, padding characters are considered invalid |
| | 74682 | 106 | | int skipLastChunk = isFinalBlock ? 4 : 0; |
| | | 107 | | |
| | 74682 | 108 | | if (destLength >= decodedLength) |
| | | 109 | | { |
| | 74682 | 110 | | maxSrcLength = srcLength - skipLastChunk; |
| | | 111 | | } |
| | | 112 | | else |
| | | 113 | | { |
| | | 114 | | // This should never overflow since destLength here is less than int.MaxValue / 4 * 3 (i.e. 16106127 |
| | | 115 | | // Therefore, (destLength / 3) * 4 will always be less than 2147483641 |
| | 0 | 116 | | Debug.Assert(destLength < (int.MaxValue / 4 * 3)); |
| | | 117 | | #if NET |
| | 0 | 118 | | (maxSrcLength, int remainder) = int.DivRem(destLength, 3); |
| | 0 | 119 | | maxSrcLength *= 4; |
| | | 120 | | #else |
| | | 121 | | maxSrcLength = (destLength / 3) * 4; |
| | | 122 | | int remainder = (int)((uint)destLength % 3); |
| | | 123 | | #endif |
| | 0 | 124 | | if (isFinalBlock && remainder > 0) |
| | | 125 | | { |
| | 0 | 126 | | srcLength &= ~0x3; // In case of Base64UrlDecoder source can be not a multiple of 4, round down |
| | | 127 | | } |
| | | 128 | | } |
| | | 129 | | |
| | 74682 | 130 | | ref sbyte decodingMap = ref MemoryMarshal.GetReference(decoder.DecodingMap); |
| | 74682 | 131 | | srcMax = srcBytes + maxSrcLength; |
| | | 132 | | |
| | 152906 | 133 | | while (src < srcMax) |
| | | 134 | | { |
| | 108999 | 135 | | int result = decoder.DecodeFourElements(src, ref decodingMap); |
| | | 136 | | |
| | 108999 | 137 | | if (result < 0) |
| | | 138 | | { |
| | | 139 | | goto InvalidDataExit; |
| | | 140 | | } |
| | | 141 | | |
| | 78224 | 142 | | WriteThreeLowOrderBytes(dest, result); |
| | 78224 | 143 | | src += 4; |
| | 78224 | 144 | | dest += 3; |
| | | 145 | | } |
| | | 146 | | |
| | 43907 | 147 | | if (maxSrcLength != srcLength - skipLastChunk) |
| | | 148 | | { |
| | | 149 | | goto DestinationTooSmallExit; |
| | | 150 | | } |
| | | 151 | | |
| | 43907 | 152 | | if (src == srcEnd) |
| | | 153 | | { |
| | 39912 | 154 | | if (isFinalBlock) |
| | | 155 | | { |
| | | 156 | | goto InvalidDataExit; |
| | | 157 | | } |
| | | 158 | | |
| | 39912 | 159 | | if (src == srcBytes + source.Length) |
| | | 160 | | { |
| | 39912 | 161 | | goto DoneExit; |
| | | 162 | | } |
| | | 163 | | |
| | | 164 | | goto NeedMoreDataExit; |
| | | 165 | | } |
| | | 166 | | |
| | | 167 | | // if isFinalBlock is false, we will never reach this point |
| | | 168 | | // Handle remaining bytes, for Base64 its always 4 bytes, for Base64Url up to 8 bytes left. |
| | | 169 | | // If more than 4 bytes remained it will end up in DestinationTooSmallExit or InvalidDataExit (might suc |
| | 3995 | 170 | | long remaining = srcEnd - src; |
| | 3995 | 171 | | Debug.Assert(typeof(TBase64Decoder) == typeof(Base64DecoderByte) ? remaining == 4 : remaining < 8); |
| | 3995 | 172 | | int i0 = decoder.DecodeRemaining(srcEnd, ref decodingMap, remaining, out uint t2, out uint t3); |
| | | 173 | | |
| | 3995 | 174 | | if (i0 < 0) |
| | | 175 | | { |
| | | 176 | | goto InvalidDataExit; |
| | | 177 | | } |
| | | 178 | | |
| | 3370 | 179 | | byte* destMax = destBytes + (uint)destLength; |
| | | 180 | | |
| | 3370 | 181 | | if (!decoder.IsValidPadding(t3)) |
| | | 182 | | { |
| | 1161 | 183 | | int i2 = Unsafe.Add(ref decodingMap, (IntPtr)t2); |
| | 1161 | 184 | | int i3 = Unsafe.Add(ref decodingMap, (IntPtr)t3); |
| | | 185 | | |
| | 1161 | 186 | | i2 <<= 6; |
| | | 187 | | |
| | 1161 | 188 | | i0 |= i3; |
| | 1161 | 189 | | i0 |= i2; |
| | | 190 | | |
| | 1161 | 191 | | if (i0 < 0) |
| | | 192 | | { |
| | | 193 | | goto InvalidDataExit; |
| | | 194 | | } |
| | 1114 | 195 | | if (dest + 3 > destMax) |
| | | 196 | | { |
| | | 197 | | goto DestinationTooSmallExit; |
| | | 198 | | } |
| | | 199 | | |
| | 1114 | 200 | | WriteThreeLowOrderBytes(dest, i0); |
| | 1114 | 201 | | dest += 3; |
| | 1114 | 202 | | src += 4; |
| | | 203 | | } |
| | 2209 | 204 | | else if (!decoder.IsValidPadding(t2)) |
| | | 205 | | { |
| | 997 | 206 | | int i2 = Unsafe.Add(ref decodingMap, (IntPtr)t2); |
| | | 207 | | |
| | 997 | 208 | | i2 <<= 6; |
| | | 209 | | |
| | 997 | 210 | | i0 |= i2; |
| | | 211 | | |
| | 997 | 212 | | if ((i0 & 0x800000c0) != 0) // if negative or 2 unused bits are not 0. |
| | | 213 | | { |
| | | 214 | | goto InvalidDataExit; |
| | | 215 | | } |
| | 924 | 216 | | if (dest + 2 > destMax) |
| | | 217 | | { |
| | | 218 | | goto DestinationTooSmallExit; |
| | | 219 | | } |
| | | 220 | | |
| | 924 | 221 | | dest[0] = (byte)(i0 >> 16); |
| | 924 | 222 | | dest[1] = (byte)(i0 >> 8); |
| | 924 | 223 | | dest += 2; |
| | 924 | 224 | | src += remaining; |
| | | 225 | | } |
| | | 226 | | else |
| | | 227 | | { |
| | 1212 | 228 | | if ((i0 & 0x8000F000) != 0) // if negative or 4 unused bits are not 0. |
| | | 229 | | { |
| | | 230 | | goto InvalidDataExit; |
| | | 231 | | } |
| | 1111 | 232 | | if (dest + 1 > destMax) |
| | | 233 | | { |
| | | 234 | | goto DestinationTooSmallExit; |
| | | 235 | | } |
| | | 236 | | |
| | 1111 | 237 | | dest[0] = (byte)(i0 >> 16); |
| | 1111 | 238 | | dest += 1; |
| | 1111 | 239 | | src += remaining; |
| | | 240 | | } |
| | | 241 | | |
| | 3149 | 242 | | if (srcLength != source.Length) |
| | | 243 | | { |
| | | 244 | | goto InvalidDataExit; |
| | | 245 | | } |
| | | 246 | | |
| | | 247 | | DoneExit: |
| | 43061 | 248 | | bytesConsumed = (int)(src - srcBytes); |
| | 43061 | 249 | | bytesWritten = (int)(dest - destBytes); |
| | 43061 | 250 | | return OperationStatus.Done; |
| | | 251 | | |
| | | 252 | | DestinationTooSmallExit: |
| | 0 | 253 | | if (srcLength != source.Length && isFinalBlock) |
| | | 254 | | { |
| | | 255 | | goto InvalidDataExit; // if input is not a multiple of 4, and there is no more data, return invalid |
| | | 256 | | } |
| | | 257 | | |
| | 0 | 258 | | if (ignoreWhiteSpace) |
| | | 259 | | { |
| | | 260 | | // Fall through to InvalidDataFallback which strips whitespace and re-evaluates destination size req |
| | | 261 | | goto InvalidDataExit; |
| | | 262 | | } |
| | | 263 | | |
| | 0 | 264 | | bytesConsumed = (int)(src - srcBytes); |
| | 0 | 265 | | bytesWritten = (int)(dest - destBytes); |
| | 0 | 266 | | return OperationStatus.DestinationTooSmall; |
| | | 267 | | |
| | | 268 | | NeedMoreDataExit: |
| | 0 | 269 | | bytesConsumed = (int)(src - srcBytes); |
| | 0 | 270 | | bytesWritten = (int)(dest - destBytes); |
| | 0 | 271 | | return OperationStatus.NeedMoreData; |
| | | 272 | | |
| | | 273 | | InvalidDataExit: |
| | 31621 | 274 | | bytesConsumed = (int)(src - srcBytes); |
| | 31621 | 275 | | bytesWritten = (int)(dest - destBytes); |
| | 31621 | 276 | | return ignoreWhiteSpace ? |
| | 31621 | 277 | | InvalidDataFallback(decoder, source, bytes, ref bytesConsumed, ref bytesWritten, isFinalBlock) : |
| | 31621 | 278 | | OperationStatus.InvalidData; |
| | | 279 | | } |
| | | 280 | | |
| | | 281 | | static OperationStatus InvalidDataFallback(TBase64Decoder decoder, ReadOnlySpan<T> source, Span<byte> bytes, |
| | | 282 | | { |
| | 2417 | 283 | | source = source.Slice(bytesConsumed); |
| | 2417 | 284 | | bytes = bytes.Slice(bytesWritten); |
| | | 285 | | |
| | | 286 | | OperationStatus status; |
| | | 287 | | do |
| | | 288 | | { |
| | 30861 | 289 | | int localConsumed = decoder.IndexOfAnyExceptWhiteSpace(source); |
| | 30861 | 290 | | if (localConsumed < 0) |
| | | 291 | | { |
| | | 292 | | // The remainder of the input is all whitespace. Mark it all as having been consumed, |
| | | 293 | | // and mark the operation as being done. |
| | 66 | 294 | | bytesConsumed += source.Length; |
| | 66 | 295 | | status = OperationStatus.Done; |
| | 66 | 296 | | break; |
| | | 297 | | } |
| | | 298 | | |
| | 30795 | 299 | | if (localConsumed == 0) |
| | | 300 | | { |
| | | 301 | | // Non-whitespace was found at the beginning of the input. Since it wasn't consumed |
| | | 302 | | // by the previous call to DecodeFromUtf8, it must be part of a Base64 sequence |
| | | 303 | | // that was interrupted by whitespace or something else considered invalid. |
| | | 304 | | // Fall back to block-wise decoding. This is very slow, but it's also very non-standard |
| | | 305 | | // formatting of the input; whitespace is typically only found between blocks, such as |
| | | 306 | | // when Convert.ToBase64String inserts a line break every 76 output characters. |
| | 2121 | 307 | | return decoder.DecodeWithWhiteSpaceBlockwiseWrapper(decoder, source, bytes, ref bytesConsumed, r |
| | | 308 | | } |
| | | 309 | | |
| | | 310 | | // Skip over the starting whitespace and continue. |
| | 28674 | 311 | | bytesConsumed += localConsumed; |
| | 28674 | 312 | | source = source.Slice(localConsumed); |
| | | 313 | | |
| | | 314 | | // Try again after consumed whitespace |
| | 28674 | 315 | | status = DecodeFrom(decoder, source, bytes, out localConsumed, out int localWritten, isFinalBlock, i |
| | 28674 | 316 | | bytesConsumed += localConsumed; |
| | 28674 | 317 | | bytesWritten += localWritten; |
| | | 318 | | |
| | 28674 | 319 | | if (status is OperationStatus.Done or OperationStatus.NeedMoreData) |
| | | 320 | | { |
| | | 321 | | break; |
| | | 322 | | } |
| | | 323 | | |
| | | 324 | | // The DecodeFrom helper will return DestinationTooSmall if the destination is too small, |
| | | 325 | | // regardless of whether it's actually too small once you skip whitespace characters. |
| | | 326 | | // In that case we loop again and fall back to block-wise decoding if we can't make progress. |
| | | 327 | | |
| | 28444 | 328 | | source = source.Slice(localConsumed); |
| | 28444 | 329 | | bytes = bytes.Slice(localWritten); |
| | | 330 | | } |
| | 28444 | 331 | | while (!source.IsEmpty); |
| | | 332 | | |
| | 296 | 333 | | return status; |
| | | 334 | | } |
| | | 335 | | } |
| | | 336 | | |
| | | 337 | | internal static unsafe OperationStatus DecodeFromUtf8InPlace<TBase64Decoder>(TBase64Decoder decoder, Span<byte> |
| | | 338 | | where TBase64Decoder : IBase64Decoder<byte> |
| | | 339 | | { |
| | | 340 | | if (buffer.IsEmpty) |
| | | 341 | | { |
| | 0 | 342 | | bytesWritten = 0; |
| | 0 | 343 | | return OperationStatus.Done; |
| | | 344 | | } |
| | | 345 | | |
| | 69005 | 346 | | fixed (byte* bufferBytes = &MemoryMarshal.GetReference(buffer)) |
| | | 347 | | { |
| | 69005 | 348 | | uint bufferLength = (uint)buffer.Length; |
| | 69005 | 349 | | uint sourceIndex = 0; |
| | 69005 | 350 | | uint destIndex = 0; |
| | | 351 | | |
| | 69005 | 352 | | if (decoder.IsInvalidLength(buffer.Length)) |
| | | 353 | | { |
| | | 354 | | goto InvalidExit; |
| | | 355 | | } |
| | | 356 | | |
| | 68665 | 357 | | ref sbyte decodingMap = ref MemoryMarshal.GetReference(decoder.DecodingMap); |
| | | 358 | | |
| | | 359 | | #if NET |
| | | 360 | | // Decode in place using the same vectorized helpers as DecodeFrom. This is safe because the |
| | | 361 | | // write cursor (dest) always trails the read cursor (src) by 25%, so each vector store -- including |
| | | 362 | | // its zero-padded overshoot -- ends at or before the next vector load and never clobbers source |
| | | 363 | | // that hasn't been read yet. |
| | 68665 | 364 | | if (bufferLength >= 24) |
| | | 365 | | { |
| | 2012 | 366 | | byte* src = bufferBytes; |
| | 2012 | 367 | | byte* dest = bufferBytes; |
| | 2012 | 368 | | int length = (int)bufferLength; |
| | 2012 | 369 | | byte* srcMax = bufferBytes + length; |
| | | 370 | | |
| | 2012 | 371 | | byte* end = srcMax - 88; |
| | 2012 | 372 | | if (Vector512.IsHardwareAccelerated && Avx512Vbmi.IsSupported && (end >= src)) |
| | | 373 | | { |
| | 1330 | 374 | | Avx512Decode(decoder, ref src, ref dest, end, length, length, bufferBytes, bufferBytes); |
| | | 375 | | } |
| | | 376 | | |
| | 2012 | 377 | | end = srcMax - 45; |
| | 2012 | 378 | | if (Avx2.IsSupported && (end >= src)) |
| | | 379 | | { |
| | 1390 | 380 | | Avx2Decode(decoder, ref src, ref dest, end, length, length, bufferBytes, bufferBytes); |
| | | 381 | | } |
| | | 382 | | |
| | 2012 | 383 | | end = srcMax - 66; |
| | | 384 | | if (AdvSimd.Arm64.IsSupported && (end >= src)) |
| | | 385 | | { |
| | | 386 | | AdvSimdDecode(decoder, ref src, ref dest, end, length, length, bufferBytes, bufferBytes); |
| | | 387 | | } |
| | | 388 | | |
| | 2012 | 389 | | end = srcMax - 24; |
| | 2012 | 390 | | if ((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported || PackedSimd.IsSupported) && BitConverter.IsLit |
| | | 391 | | { |
| | 1637 | 392 | | Vector128Decode(decoder, ref src, ref dest, end, length, length, bufferBytes, bufferBytes); |
| | | 393 | | } |
| | | 394 | | |
| | 2012 | 395 | | sourceIndex = (uint)(src - bufferBytes); |
| | 2012 | 396 | | destIndex = (uint)(dest - bufferBytes); |
| | | 397 | | } |
| | | 398 | | #endif |
| | | 399 | | |
| | 68665 | 400 | | if (bufferLength > 4) |
| | | 401 | | { |
| | 7085 | 402 | | while (sourceIndex < bufferLength - 4) |
| | | 403 | | { |
| | 5529 | 404 | | int result = decoder.DecodeFourElements(bufferBytes + sourceIndex, ref decodingMap); |
| | 5529 | 405 | | if (result < 0) |
| | | 406 | | { |
| | | 407 | | goto InvalidExit; |
| | | 408 | | } |
| | | 409 | | |
| | 4441 | 410 | | WriteThreeLowOrderBytes(bufferBytes + destIndex, result); |
| | 4441 | 411 | | destIndex += 3; |
| | 4441 | 412 | | sourceIndex += 4; |
| | | 413 | | } |
| | | 414 | | } |
| | | 415 | | |
| | | 416 | | uint t0; |
| | | 417 | | uint t1; |
| | | 418 | | uint t2; |
| | | 419 | | uint t3; |
| | | 420 | | |
| | 67577 | 421 | | switch (bufferLength - sourceIndex) |
| | | 422 | | { |
| | | 423 | | case 2: |
| | 678 | 424 | | t0 = bufferBytes[bufferLength - 2]; |
| | 678 | 425 | | t1 = bufferBytes[bufferLength - 1]; |
| | 678 | 426 | | t2 = EncodingPad; |
| | 678 | 427 | | t3 = EncodingPad; |
| | 678 | 428 | | break; |
| | | 429 | | case 3: |
| | 567 | 430 | | t0 = bufferBytes[bufferLength - 3]; |
| | 567 | 431 | | t1 = bufferBytes[bufferLength - 2]; |
| | 567 | 432 | | t2 = bufferBytes[bufferLength - 1]; |
| | 567 | 433 | | t3 = EncodingPad; |
| | 567 | 434 | | break; |
| | | 435 | | case 4: |
| | 66332 | 436 | | t0 = bufferBytes[bufferLength - 4]; |
| | 66332 | 437 | | t1 = bufferBytes[bufferLength - 3]; |
| | 66332 | 438 | | t2 = bufferBytes[bufferLength - 2]; |
| | 66332 | 439 | | t3 = bufferBytes[bufferLength - 1]; |
| | | 440 | | break; |
| | | 441 | | default: |
| | | 442 | | goto InvalidExit; |
| | | 443 | | } |
| | | 444 | | |
| | 67577 | 445 | | int i0 = Unsafe.Add(ref decodingMap, (int)t0); |
| | 67577 | 446 | | int i1 = Unsafe.Add(ref decodingMap, (int)t1); |
| | | 447 | | |
| | 67577 | 448 | | i0 <<= 18; |
| | 67577 | 449 | | i1 <<= 12; |
| | | 450 | | |
| | 67577 | 451 | | i0 |= i1; |
| | | 452 | | |
| | 67577 | 453 | | if (!decoder.IsValidPadding(t3)) |
| | | 454 | | { |
| | 65620 | 455 | | int i2 = Unsafe.Add(ref decodingMap, (int)t2); |
| | 65620 | 456 | | int i3 = Unsafe.Add(ref decodingMap, (int)t3); |
| | | 457 | | |
| | 65620 | 458 | | i2 <<= 6; |
| | | 459 | | |
| | 65620 | 460 | | i0 |= i3; |
| | 65620 | 461 | | i0 |= i2; |
| | | 462 | | |
| | 65620 | 463 | | if (i0 < 0) |
| | | 464 | | { |
| | | 465 | | goto InvalidExit; |
| | | 466 | | } |
| | | 467 | | |
| | 65176 | 468 | | WriteThreeLowOrderBytes(bufferBytes + destIndex, i0); |
| | 65176 | 469 | | destIndex += 3; |
| | | 470 | | } |
| | 1957 | 471 | | else if (!decoder.IsValidPadding(t2)) |
| | | 472 | | { |
| | 848 | 473 | | int i2 = Unsafe.Add(ref decodingMap, (int)t2); |
| | | 474 | | |
| | 848 | 475 | | i2 <<= 6; |
| | | 476 | | |
| | 848 | 477 | | i0 |= i2; |
| | | 478 | | |
| | 848 | 479 | | if ((i0 & 0x800000c0) != 0) // if negative or 2 unused bits are not 0. |
| | | 480 | | { |
| | | 481 | | goto InvalidExit; |
| | | 482 | | } |
| | | 483 | | |
| | 700 | 484 | | bufferBytes[destIndex] = (byte)(i0 >> 16); |
| | 700 | 485 | | bufferBytes[destIndex + 1] = (byte)(i0 >> 8); |
| | 700 | 486 | | destIndex += 2; |
| | | 487 | | } |
| | | 488 | | else |
| | | 489 | | { |
| | 1109 | 490 | | if ((i0 & 0x8000F000) != 0) // if negative or 4 unused bits are not 0. |
| | | 491 | | { |
| | | 492 | | goto InvalidExit; |
| | | 493 | | } |
| | | 494 | | |
| | 865 | 495 | | bufferBytes[destIndex] = (byte)(i0 >> 16); |
| | 865 | 496 | | destIndex += 1; |
| | | 497 | | } |
| | | 498 | | |
| | 66741 | 499 | | bytesWritten = (int)destIndex; |
| | 66741 | 500 | | return OperationStatus.Done; |
| | | 501 | | |
| | | 502 | | InvalidExit: |
| | 2264 | 503 | | bytesWritten = (int)destIndex; |
| | 2264 | 504 | | return ignoreWhiteSpace ? |
| | 2264 | 505 | | DecodeWithWhiteSpaceFromUtf8InPlace<TBase64Decoder>(decoder, buffer, ref bytesWritten, sourceIndex) |
| | 2264 | 506 | | OperationStatus.InvalidData; |
| | | 507 | | } |
| | | 508 | | } |
| | | 509 | | |
| | | 510 | | internal static unsafe OperationStatus DecodeWithWhiteSpaceBlockwise<TBase64Decoder>(TBase64Decoder decoder, Rea |
| | | 511 | | where TBase64Decoder : IBase64Decoder<byte> |
| | | 512 | | { |
| | | 513 | | const int BlockSize = 4; |
| | 2121 | 514 | | Span<byte> buffer = stackalloc byte[BlockSize]; |
| | 2121 | 515 | | OperationStatus status = OperationStatus.Done; |
| | | 516 | | |
| | 50591 | 517 | | while (!source.IsEmpty) |
| | | 518 | | { |
| | | 519 | | // Skip over any leading whitespace |
| | 50479 | 520 | | if (IsWhiteSpace(source[0])) |
| | | 521 | | { |
| | 10153 | 522 | | source = source.Slice(1); |
| | 10153 | 523 | | bytesConsumed++; |
| | 10153 | 524 | | continue; |
| | | 525 | | } |
| | | 526 | | |
| | 40326 | 527 | | int encodedIdx = 0; |
| | 40326 | 528 | | int bufferIdx = 0; |
| | 40326 | 529 | | int skipped = 0; |
| | | 530 | | |
| | 401662 | 531 | | for (; encodedIdx < source.Length && (uint)bufferIdx < (uint)buffer.Length; ++encodedIdx) |
| | | 532 | | { |
| | 180668 | 533 | | if (IsWhiteSpace(source[encodedIdx])) |
| | | 534 | | { |
| | 21060 | 535 | | skipped++; |
| | | 536 | | } |
| | | 537 | | else |
| | | 538 | | { |
| | 159608 | 539 | | buffer[bufferIdx] = source[encodedIdx]; |
| | 159608 | 540 | | bufferIdx++; |
| | | 541 | | } |
| | | 542 | | } |
| | | 543 | | |
| | 40326 | 544 | | source = source.Slice(encodedIdx); |
| | 40326 | 545 | | Debug.Assert(bufferIdx > 0); |
| | | 546 | | |
| | | 547 | | bool hasAnotherBlock; |
| | | 548 | | |
| | 40326 | 549 | | if (typeof(TBase64Decoder) == typeof(Base64DecoderByte)) |
| | | 550 | | { |
| | 0 | 551 | | hasAnotherBlock = source.Length >= BlockSize; |
| | | 552 | | } |
| | | 553 | | else |
| | | 554 | | { |
| | 40326 | 555 | | hasAnotherBlock = source.Length > 1; |
| | | 556 | | } |
| | | 557 | | |
| | 40326 | 558 | | bool localIsFinalBlock = !hasAnotherBlock; |
| | | 559 | | |
| | | 560 | | // If this block contains padding and there's another block, then only whitespace may follow for being v |
| | 40326 | 561 | | if (hasAnotherBlock) |
| | | 562 | | { |
| | 38799 | 563 | | int paddingCount = GetPaddingCount(decoder, ref buffer[BlockSize - 1]); |
| | 38799 | 564 | | if (paddingCount > 0) |
| | | 565 | | { |
| | 169 | 566 | | hasAnotherBlock = false; |
| | 169 | 567 | | localIsFinalBlock = true; |
| | | 568 | | } |
| | | 569 | | } |
| | | 570 | | |
| | 40326 | 571 | | if (localIsFinalBlock && !isFinalBlock) |
| | | 572 | | { |
| | 0 | 573 | | localIsFinalBlock = false; |
| | | 574 | | } |
| | | 575 | | |
| | 40326 | 576 | | status = DecodeFrom<TBase64Decoder, byte>(decoder, buffer.Slice(0, bufferIdx), bytes, out int localConsu |
| | | 577 | | |
| | 40326 | 578 | | if (status != OperationStatus.Done) |
| | | 579 | | { |
| | 760 | 580 | | Debug.Assert(localConsumed == 0 && localWritten == 0, "On failure, should not have consumed or writt |
| | 760 | 581 | | return status; |
| | | 582 | | } |
| | | 583 | | |
| | 39566 | 584 | | bytesConsumed += skipped; |
| | 39566 | 585 | | bytesConsumed += localConsumed; |
| | 39566 | 586 | | bytesWritten += localWritten; |
| | | 587 | | |
| | | 588 | | // The remaining data must all be whitespace in order to be valid. |
| | 39566 | 589 | | if (!hasAnotherBlock) |
| | | 590 | | { |
| | 7274 | 591 | | for (int i = 0; i < source.Length; ++i) |
| | | 592 | | { |
| | 2455 | 593 | | if (!IsWhiteSpace(source[i])) |
| | | 594 | | { |
| | | 595 | | // Revert previous dest increment, since an invalid state followed. |
| | 67 | 596 | | bytesConsumed -= localConsumed; |
| | 67 | 597 | | bytesWritten -= localWritten; |
| | | 598 | | |
| | 67 | 599 | | return OperationStatus.InvalidData; |
| | | 600 | | } |
| | | 601 | | |
| | 2388 | 602 | | bytesConsumed++; |
| | | 603 | | } |
| | | 604 | | |
| | 1182 | 605 | | break; |
| | | 606 | | } |
| | | 607 | | |
| | 38317 | 608 | | bytes = bytes.Slice(localWritten); |
| | 38317 | 609 | | Debug.Assert(!source.IsEmpty); |
| | | 610 | | } |
| | | 611 | | |
| | 1294 | 612 | | return status; |
| | | 613 | | } |
| | | 614 | | |
| | | 615 | | internal static unsafe OperationStatus DecodeWithWhiteSpaceBlockwise<TBase64Decoder>(TBase64Decoder decoder, Rea |
| | | 616 | | where TBase64Decoder : IBase64Decoder<ushort> |
| | | 617 | | { |
| | | 618 | | const int BlockSize = 4; |
| | 0 | 619 | | Span<ushort> buffer = stackalloc ushort[BlockSize]; |
| | 0 | 620 | | OperationStatus status = OperationStatus.Done; |
| | | 621 | | |
| | 0 | 622 | | while (!source.IsEmpty) |
| | | 623 | | { |
| | | 624 | | // Skip over any leading whitespace |
| | 0 | 625 | | if (IsWhiteSpace(source[0])) |
| | | 626 | | { |
| | 0 | 627 | | source = source.Slice(1); |
| | 0 | 628 | | bytesConsumed++; |
| | 0 | 629 | | continue; |
| | | 630 | | } |
| | | 631 | | |
| | 0 | 632 | | int encodedIdx = 0; |
| | 0 | 633 | | int bufferIdx = 0; |
| | 0 | 634 | | int skipped = 0; |
| | | 635 | | |
| | 0 | 636 | | for (; encodedIdx < source.Length && (uint)bufferIdx < (uint)buffer.Length; ++encodedIdx) |
| | | 637 | | { |
| | 0 | 638 | | if (IsWhiteSpace(source[encodedIdx])) |
| | | 639 | | { |
| | 0 | 640 | | skipped++; |
| | | 641 | | } |
| | | 642 | | else |
| | | 643 | | { |
| | 0 | 644 | | buffer[bufferIdx] = source[encodedIdx]; |
| | 0 | 645 | | bufferIdx++; |
| | | 646 | | } |
| | | 647 | | } |
| | | 648 | | |
| | 0 | 649 | | source = source.Slice(encodedIdx); |
| | 0 | 650 | | Debug.Assert(bufferIdx > 0); |
| | | 651 | | |
| | | 652 | | bool hasAnotherBlock; |
| | | 653 | | |
| | 0 | 654 | | if (decoder is Base64DecoderChar) |
| | | 655 | | { |
| | 0 | 656 | | hasAnotherBlock = source.Length >= BlockSize; |
| | | 657 | | } |
| | | 658 | | else |
| | | 659 | | { |
| | 0 | 660 | | hasAnotherBlock = source.Length > 1; |
| | | 661 | | } |
| | | 662 | | |
| | 0 | 663 | | bool localIsFinalBlock = !hasAnotherBlock; |
| | | 664 | | |
| | | 665 | | // If this block contains padding and there's another block, then only whitespace may follow for being v |
| | 0 | 666 | | if (hasAnotherBlock) |
| | | 667 | | { |
| | 0 | 668 | | int paddingCount = GetPaddingCount(decoder, ref buffer[BlockSize - 1]); |
| | 0 | 669 | | if (paddingCount > 0) |
| | | 670 | | { |
| | 0 | 671 | | hasAnotherBlock = false; |
| | 0 | 672 | | localIsFinalBlock = true; |
| | | 673 | | } |
| | | 674 | | } |
| | | 675 | | |
| | 0 | 676 | | if (localIsFinalBlock && !isFinalBlock) |
| | | 677 | | { |
| | 0 | 678 | | localIsFinalBlock = false; |
| | | 679 | | } |
| | | 680 | | |
| | 0 | 681 | | status = DecodeFrom(decoder, buffer.Slice(0, bufferIdx), bytes, out int localConsumed, out int localWrit |
| | | 682 | | |
| | 0 | 683 | | if (status != OperationStatus.Done) |
| | | 684 | | { |
| | 0 | 685 | | Debug.Assert(localConsumed == 0 && localWritten == 0, "On failure, should not have consumed or writt |
| | 0 | 686 | | return status; |
| | | 687 | | } |
| | | 688 | | |
| | 0 | 689 | | bytesConsumed += skipped; |
| | 0 | 690 | | bytesConsumed += localConsumed; |
| | 0 | 691 | | bytesWritten += localWritten; |
| | | 692 | | |
| | | 693 | | // The remaining data must all be whitespace in order to be valid. |
| | 0 | 694 | | if (!hasAnotherBlock) |
| | | 695 | | { |
| | 0 | 696 | | for (int i = 0; i < source.Length; ++i) |
| | | 697 | | { |
| | 0 | 698 | | if (!IsWhiteSpace(source[i])) |
| | | 699 | | { |
| | | 700 | | // Revert previous dest increment, since an invalid state followed. |
| | 0 | 701 | | bytesConsumed -= localConsumed; |
| | 0 | 702 | | bytesWritten -= localWritten; |
| | | 703 | | |
| | 0 | 704 | | return OperationStatus.InvalidData; |
| | | 705 | | } |
| | | 706 | | } |
| | | 707 | | |
| | 0 | 708 | | bytesConsumed += source.Length; |
| | 0 | 709 | | break; |
| | | 710 | | } |
| | | 711 | | |
| | 0 | 712 | | bytes = bytes.Slice(localWritten); |
| | 0 | 713 | | Debug.Assert(!source.IsEmpty); |
| | | 714 | | } |
| | | 715 | | |
| | 0 | 716 | | return status; |
| | | 717 | | } |
| | | 718 | | |
| | | 719 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 720 | | private static int GetPaddingCount<TBase64Decoder>(TBase64Decoder decoder, ref byte ptrToLastElement) |
| | | 721 | | where TBase64Decoder : IBase64Decoder<byte> |
| | | 722 | | { |
| | 38799 | 723 | | int padding = 0; |
| | | 724 | | |
| | 38799 | 725 | | if (decoder.IsValidPadding(ptrToLastElement)) |
| | | 726 | | { |
| | 158 | 727 | | padding++; |
| | | 728 | | } |
| | | 729 | | |
| | 38799 | 730 | | if (decoder.IsValidPadding(Unsafe.Subtract(ref ptrToLastElement, 1))) |
| | | 731 | | { |
| | 91 | 732 | | padding++; |
| | | 733 | | } |
| | | 734 | | |
| | 38799 | 735 | | return padding; |
| | | 736 | | } |
| | | 737 | | |
| | | 738 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 739 | | private static int GetPaddingCount<TBase64Decoder>(TBase64Decoder decoder, ref ushort ptrToLastElement) |
| | | 740 | | where TBase64Decoder : IBase64Decoder<ushort> |
| | | 741 | | { |
| | 0 | 742 | | int padding = 0; |
| | | 743 | | |
| | 0 | 744 | | if (decoder.IsValidPadding(ptrToLastElement)) |
| | | 745 | | { |
| | 0 | 746 | | padding++; |
| | | 747 | | } |
| | | 748 | | |
| | 0 | 749 | | if (decoder.IsValidPadding(Unsafe.Subtract(ref ptrToLastElement, 1))) |
| | | 750 | | { |
| | 0 | 751 | | padding++; |
| | | 752 | | } |
| | | 753 | | |
| | 0 | 754 | | return padding; |
| | | 755 | | } |
| | | 756 | | |
| | | 757 | | private static unsafe OperationStatus DecodeWithWhiteSpaceFromUtf8InPlace<TBase64Decoder>(TBase64Decoder decoder |
| | | 758 | | where TBase64Decoder : IBase64Decoder<byte> |
| | | 759 | | { |
| | 1622 | 760 | | int BlockSize = Math.Min(source.Length - (int)sourceIndex, 4); |
| | 1622 | 761 | | Span<byte> buffer = stackalloc byte[BlockSize]; |
| | | 762 | | |
| | 1622 | 763 | | OperationStatus status = OperationStatus.Done; |
| | 1622 | 764 | | int localDestIndex = destIndex; |
| | 1622 | 765 | | bool hasPaddingBeenProcessed = false; |
| | 1622 | 766 | | int localBytesWritten = 0; |
| | | 767 | | |
| | 66863 | 768 | | while (sourceIndex < (uint)source.Length) |
| | | 769 | | { |
| | 66068 | 770 | | int bufferIdx = 0; |
| | | 771 | | |
| | 372313 | 772 | | while (bufferIdx < BlockSize && sourceIndex < (uint)source.Length) |
| | | 773 | | { |
| | 306245 | 774 | | if (!IsWhiteSpace(source[(int)sourceIndex])) |
| | | 775 | | { |
| | 262147 | 776 | | buffer[bufferIdx] = source[(int)sourceIndex]; |
| | 262147 | 777 | | bufferIdx++; |
| | | 778 | | } |
| | | 779 | | |
| | 306245 | 780 | | sourceIndex++; |
| | | 781 | | } |
| | | 782 | | |
| | 66068 | 783 | | if (bufferIdx == 0) |
| | | 784 | | { |
| | | 785 | | continue; |
| | | 786 | | } |
| | | 787 | | |
| | 65914 | 788 | | if (bufferIdx != 4) |
| | | 789 | | { |
| | | 790 | | // Base64 require 4 bytes, for Base64Url it can be less than 4 bytes but not 1 byte. |
| | 818 | 791 | | if (decoder is Base64DecoderByte || bufferIdx == 1) |
| | | 792 | | { |
| | 164 | 793 | | status = OperationStatus.InvalidData; |
| | 164 | 794 | | break; |
| | | 795 | | } |
| | | 796 | | else // For Base64Url fill empty slots in last block with padding |
| | | 797 | | { |
| | 1571 | 798 | | while (bufferIdx < BlockSize) // Can happen only for last block |
| | | 799 | | { |
| | 917 | 800 | | Debug.Assert(source.Length == sourceIndex); |
| | 917 | 801 | | buffer[bufferIdx++] = (byte)EncodingPad; |
| | | 802 | | } |
| | | 803 | | } |
| | | 804 | | } |
| | | 805 | | |
| | 65750 | 806 | | if (hasPaddingBeenProcessed) |
| | | 807 | | { |
| | | 808 | | // Padding has already been processed, a new valid block cannot be processed. |
| | | 809 | | // Revert previous dest increment, since an invalid state followed. |
| | 21 | 810 | | localDestIndex -= localBytesWritten; |
| | 21 | 811 | | status = OperationStatus.InvalidData; |
| | 21 | 812 | | break; |
| | | 813 | | } |
| | | 814 | | |
| | 65729 | 815 | | status = DecodeFromUtf8InPlace<TBase64Decoder>(decoder, buffer, out localBytesWritten, ignoreWhiteSpace: |
| | 65729 | 816 | | localDestIndex += localBytesWritten; |
| | 65729 | 817 | | hasPaddingBeenProcessed = localBytesWritten < 3; |
| | | 818 | | |
| | 65729 | 819 | | if (status != OperationStatus.Done) |
| | | 820 | | { |
| | | 821 | | break; |
| | | 822 | | } |
| | | 823 | | |
| | | 824 | | // Write result to source span in place. |
| | 519204 | 825 | | for (int i = 0; i < localBytesWritten; i++) |
| | | 826 | | { |
| | 194515 | 827 | | source[localDestIndex - localBytesWritten + i] = buffer[i]; |
| | | 828 | | } |
| | | 829 | | } |
| | | 830 | | |
| | 1622 | 831 | | destIndex = localDestIndex; |
| | 1622 | 832 | | return status; |
| | | 833 | | } |
| | | 834 | | |
| | | 835 | | #if NET |
| | | 836 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 837 | | [CompExactlyDependsOn(typeof(Avx512BW))] |
| | | 838 | | [CompExactlyDependsOn(typeof(Avx512Vbmi))] |
| | | 839 | | private static unsafe void Avx512Decode<TBase64Decoder, T>(TBase64Decoder decoder, ref T* srcBytes, ref byte* de |
| | | 840 | | where TBase64Decoder : IBase64Decoder<T> |
| | | 841 | | where T : unmanaged |
| | | 842 | | { |
| | | 843 | | // Reference for VBMI implementation : https://github.com/WojciechMula/base64simd/tree/master/decode |
| | | 844 | | // If we have AVX512 support, pick off 64 bytes at a time for as long as we can, |
| | | 845 | | // but make sure that we quit before seeing any == markers at the end of the |
| | | 846 | | // string. Also, because we write 16 zeroes at the end of the output, ensure |
| | | 847 | | // that there are at least 22 valid bytes of input data remaining to close the |
| | | 848 | | // gap. 64 + 2 + 22 = 88 bytes. |
| | 29199 | 849 | | T* src = srcBytes; |
| | 29199 | 850 | | byte* dest = destBytes; |
| | | 851 | | |
| | | 852 | | // The JIT won't hoist these "constants", so help it |
| | 29199 | 853 | | Vector512<sbyte> vbmiLookup0 = Vector512.Create(decoder.VbmiLookup0).AsSByte(); |
| | 29199 | 854 | | Vector512<sbyte> vbmiLookup1 = Vector512.Create(decoder.VbmiLookup1).AsSByte(); |
| | 29199 | 855 | | Vector512<byte> vbmiPackedLanesControl = Vector512.Create( |
| | 29199 | 856 | | 0x06000102, 0x090a0405, 0x0c0d0e08, 0x16101112, |
| | 29199 | 857 | | 0x191a1415, 0x1c1d1e18, 0x26202122, 0x292a2425, |
| | 29199 | 858 | | 0x2c2d2e28, 0x36303132, 0x393a3435, 0x3c3d3e38, |
| | 29199 | 859 | | 0x00000000, 0x00000000, 0x00000000, 0x00000000).AsByte(); |
| | | 860 | | |
| | 29199 | 861 | | Vector512<sbyte> mergeConstant0 = Vector512.Create(0x01400140).AsSByte(); |
| | 29199 | 862 | | Vector512<short> mergeConstant1 = Vector512.Create(0x00011000).AsInt16(); |
| | | 863 | | |
| | | 864 | | // This algorithm requires AVX512VBMI support. |
| | | 865 | | // Vbmi was first introduced in CannonLake and is available from IceLake on. |
| | | 866 | | do |
| | | 867 | | { |
| | 53725 | 868 | | if (!decoder.TryLoadVector512(src, srcStart, sourceLength, out Vector512<sbyte> str)) |
| | | 869 | | { |
| | | 870 | | break; |
| | | 871 | | } |
| | | 872 | | |
| | | 873 | | // Step 1: Translate encoded Base64 input to their original indices |
| | | 874 | | // This step also checks for invalid inputs and exits. |
| | | 875 | | // After this, we have indices which are verified to have upper 2 bits set to 0 in each byte. |
| | | 876 | | // origIndex = [...|00dddddd|00cccccc|00bbbbbb|00aaaaaa] |
| | 53725 | 877 | | Vector512<sbyte> origIndex = Avx512Vbmi.PermuteVar64x8x2(vbmiLookup0, str, vbmiLookup1); |
| | 53725 | 878 | | Vector512<sbyte> errorVec = (origIndex.AsInt32() | str.AsInt32()).AsSByte(); |
| | 53725 | 879 | | if (errorVec.ExtractMostSignificantBits() != 0) |
| | | 880 | | { |
| | | 881 | | break; |
| | | 882 | | } |
| | | 883 | | |
| | | 884 | | // Step 2: Now we need to reshuffle bits to remove the 0 bits. |
| | | 885 | | // multiAdd1: [...|0000cccc|ccdddddd|0000aaaa|aabbbbbb] |
| | 27045 | 886 | | Vector512<short> multiAdd1 = Avx512BW.MultiplyAddAdjacent(origIndex.AsByte(), mergeConstant0); |
| | | 887 | | // multiAdd1: [...|00000000|aaaaaabb|bbbbcccc|ccdddddd] |
| | 27045 | 888 | | Vector512<int> multiAdd2 = Avx512BW.MultiplyAddAdjacent(multiAdd1, mergeConstant1); |
| | | 889 | | |
| | | 890 | | // Step 3: Pack 48 bytes |
| | 27045 | 891 | | str = Avx512Vbmi.PermuteVar64x8(multiAdd2.AsByte(), vbmiPackedLanesControl).AsSByte(); |
| | | 892 | | |
| | 27045 | 893 | | AssertWrite<Vector512<sbyte>>(dest, destStart, destLength); |
| | 27045 | 894 | | str.Store((sbyte*)dest); |
| | 27045 | 895 | | src += 64; |
| | 27045 | 896 | | dest += 48; |
| | | 897 | | } |
| | 27045 | 898 | | while (src <= srcEnd); |
| | | 899 | | |
| | 29199 | 900 | | srcBytes = src; |
| | 29199 | 901 | | destBytes = dest; |
| | 29199 | 902 | | } |
| | | 903 | | |
| | | 904 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 905 | | [CompExactlyDependsOn(typeof(Avx2))] |
| | | 906 | | private static unsafe void Avx2Decode<TBase64Decoder, T>(TBase64Decoder decoder, ref T* srcBytes, ref byte* dest |
| | | 907 | | where TBase64Decoder : IBase64Decoder<T> |
| | | 908 | | where T : unmanaged |
| | | 909 | | { |
| | | 910 | | // If we have AVX2 support, pick off 32 bytes at a time for as long as we can, |
| | | 911 | | // but make sure that we quit before seeing any == markers at the end of the |
| | | 912 | | // string. Also, because we write 8 zeroes at the end of the output, ensure |
| | | 913 | | // that there are at least 11 valid bytes of input data remaining to close the |
| | | 914 | | // gap. 32 + 2 + 11 = 45 bytes. |
| | | 915 | | |
| | | 916 | | // See SSSE3-version below for an explanation of how the code works. |
| | | 917 | | |
| | | 918 | | // The JIT won't hoist these "constants", so help it |
| | 31339 | 919 | | Vector256<sbyte> lutHi = Vector256.Create(decoder.Avx2LutHigh); |
| | | 920 | | |
| | 31339 | 921 | | Vector256<sbyte> lutLo = Vector256.Create(decoder.Avx2LutLow); |
| | | 922 | | |
| | 31339 | 923 | | Vector256<sbyte> lutShift = Vector256.Create(decoder.Avx2LutShift); |
| | | 924 | | |
| | 31339 | 925 | | Vector256<sbyte> packBytesInLaneMask = Vector256.Create( |
| | 31339 | 926 | | 2, 1, 0, 6, |
| | 31339 | 927 | | 5, 4, 10, 9, |
| | 31339 | 928 | | 8, 14, 13, 12, |
| | 31339 | 929 | | -1, -1, -1, -1, |
| | 31339 | 930 | | 2, 1, 0, 6, |
| | 31339 | 931 | | 5, 4, 10, 9, |
| | 31339 | 932 | | 8, 14, 13, 12, |
| | 31339 | 933 | | -1, -1, -1, -1); |
| | | 934 | | |
| | 31339 | 935 | | Vector256<int> packLanesControl = Vector256.Create( |
| | 31339 | 936 | | 0, 0, 0, 0, |
| | 31339 | 937 | | 1, 0, 0, 0, |
| | 31339 | 938 | | 2, 0, 0, 0, |
| | 31339 | 939 | | 4, 0, 0, 0, |
| | 31339 | 940 | | 5, 0, 0, 0, |
| | 31339 | 941 | | 6, 0, 0, 0, |
| | 31339 | 942 | | -1, -1, -1, -1, |
| | 31339 | 943 | | -1, -1, -1, -1).AsInt32(); |
| | | 944 | | |
| | 31339 | 945 | | Vector256<sbyte> maskSlashOrUnderscore = Vector256.Create((sbyte)decoder.MaskSlashOrUnderscore); |
| | 31339 | 946 | | Vector256<sbyte> shiftForUnderscore = Vector256.Create((sbyte)33); |
| | 31339 | 947 | | Vector256<sbyte> mergeConstant0 = Vector256.Create(0x01400140).AsSByte(); |
| | 31339 | 948 | | Vector256<short> mergeConstant1 = Vector256.Create(0x00011000).AsInt16(); |
| | | 949 | | |
| | 31339 | 950 | | T* src = srcBytes; |
| | 31339 | 951 | | byte* dest = destBytes; |
| | | 952 | | |
| | | 953 | | //while (remaining >= 45) |
| | | 954 | | do |
| | | 955 | | { |
| | 33185 | 956 | | if (!decoder.TryLoadAvxVector256(src, srcStart, sourceLength, out Vector256<sbyte> str)) |
| | | 957 | | { |
| | | 958 | | break; |
| | | 959 | | } |
| | | 960 | | |
| | 33185 | 961 | | Vector256<sbyte> hiNibbles = ((str.AsInt32()) >>> 4).AsSByte() & maskSlashOrUnderscore; |
| | | 962 | | |
| | 33185 | 963 | | if (!decoder.TryDecode256Core(str, hiNibbles, maskSlashOrUnderscore, lutLo, lutHi, lutShift, shiftForUnd |
| | | 964 | | { |
| | | 965 | | break; |
| | | 966 | | } |
| | | 967 | | |
| | | 968 | | // in, lower lane, bits, upper case are most significant bits, lower case are least significant bits: |
| | | 969 | | // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ |
| | | 970 | | // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG |
| | | 971 | | // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD |
| | | 972 | | // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA |
| | | 973 | | |
| | 4082 | 974 | | Vector256<short> merge_ab_and_bc = Avx2.MultiplyAddAdjacent(str.AsByte(), mergeConstant0); |
| | | 975 | | // 0000kkkk LLllllll 0000JJJJ JJjjKKKK |
| | | 976 | | // 0000hhhh IIiiiiii 0000GGGG GGggHHHH |
| | | 977 | | // 0000eeee FFffffff 0000DDDD DDddEEEE |
| | | 978 | | // 0000bbbb CCcccccc 0000AAAA AAaaBBBB |
| | | 979 | | |
| | 4082 | 980 | | Vector256<int> output = Avx2.MultiplyAddAdjacent(merge_ab_and_bc, mergeConstant1); |
| | | 981 | | // 00000000 JJJJJJjj KKKKkkkk LLllllll |
| | | 982 | | // 00000000 GGGGGGgg HHHHhhhh IIiiiiii |
| | | 983 | | // 00000000 DDDDDDdd EEEEeeee FFffffff |
| | | 984 | | // 00000000 AAAAAAaa BBBBbbbb CCcccccc |
| | | 985 | | |
| | | 986 | | // Pack bytes together in each lane: |
| | 4082 | 987 | | output = Avx2.Shuffle(output.AsSByte(), packBytesInLaneMask).AsInt32(); |
| | | 988 | | // 00000000 00000000 00000000 00000000 |
| | | 989 | | // LLllllll KKKKkkkk JJJJJJjj IIiiiiii |
| | | 990 | | // HHHHhhhh GGGGGGgg FFffffff EEEEeeee |
| | | 991 | | // DDDDDDdd CCcccccc BBBBbbbb AAAAAAaa |
| | | 992 | | |
| | | 993 | | // Pack lanes |
| | 4082 | 994 | | str = Avx2.PermuteVar8x32(output, packLanesControl).AsSByte(); |
| | | 995 | | |
| | 4082 | 996 | | AssertWrite<Vector256<sbyte>>(dest, destStart, destLength); |
| | 4082 | 997 | | Avx.Store(dest, str.AsByte()); |
| | | 998 | | |
| | 4082 | 999 | | src += 32; |
| | 4082 | 1000 | | dest += 24; |
| | | 1001 | | } |
| | 4082 | 1002 | | while (src <= srcEnd); |
| | | 1003 | | |
| | 31339 | 1004 | | srcBytes = src; |
| | 31339 | 1005 | | destBytes = dest; |
| | 31339 | 1006 | | } |
| | | 1007 | | |
| | | 1008 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1009 | | [CompExactlyDependsOn(typeof(Ssse3))] |
| | | 1010 | | [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] |
| | | 1011 | | [CompExactlyDependsOn(typeof(PackedSimd))] |
| | | 1012 | | internal static Vector128<byte> SimdShuffle(Vector128<byte> left, Vector128<byte> right, Vector128<byte> mask8F) |
| | | 1013 | | { |
| | | 1014 | | Debug.Assert((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported || PackedSimd.IsSupported) && BitConverter.IsLi |
| | | 1015 | | |
| | 98588 | 1016 | | if (Ssse3.IsSupported) |
| | | 1017 | | { |
| | 98588 | 1018 | | return Ssse3.Shuffle(left, right); |
| | | 1019 | | } |
| | | 1020 | | else if (PackedSimd.IsSupported) |
| | | 1021 | | { |
| | | 1022 | | return PackedSimd.Swizzle(left, right & mask8F); |
| | | 1023 | | } |
| | | 1024 | | else |
| | | 1025 | | { |
| | 0 | 1026 | | return AdvSimd.Arm64.VectorTableLookup(left, right & mask8F); |
| | | 1027 | | } |
| | | 1028 | | } |
| | | 1029 | | |
| | | 1030 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1031 | | [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] |
| | | 1032 | | private static unsafe void AdvSimdDecode<TBase64Decoder, T>(TBase64Decoder decoder, ref T* srcBytes, ref byte* d |
| | | 1033 | | where TBase64Decoder : IBase64Decoder<T> |
| | | 1034 | | where T : unmanaged |
| | | 1035 | | { |
| | | 1036 | | // C# implementation of https://github.com/aklomp/base64/blob/3a5add8652076612a8407627a42c768736a4263f/lib/a |
| | | 1037 | | // If we have AdvSimd support, pick off 64 bytes at a time for as long as we can, |
| | | 1038 | | // but make sure that we quit before seeing any == markers at the end of the |
| | | 1039 | | // string. 64 + 2 = 66 bytes. |
| | | 1040 | | |
| | | 1041 | | // In the decoding process, we want to map each byte, representing a Base64 value, to its 6-bit (0-63) repre |
| | | 1042 | | // It uses the following mapping. Values outside the following groups are invalid and, we abort decoding whe |
| | | 1043 | | // |
| | | 1044 | | // # From To Char |
| | | 1045 | | // 1 [43] [62] + |
| | | 1046 | | // 2 [47] [63] / |
| | | 1047 | | // 3 [48..57] [52..61] 0..9 |
| | | 1048 | | // 4 [65..90] [0..25] A..Z |
| | | 1049 | | // 5 [97..122] [26..51] a..z |
| | | 1050 | | // |
| | | 1051 | | // To map an input value to its Base64 representation, we use look-up tables 'decLutOne' and 'decLutTwo'. |
| | | 1052 | | // 'decLutOne' helps to map groups 1, 2 and 3 while 'decLutTwo' maps groups 4 and 5 in the above list. |
| | | 1053 | | // After mapping, each value falls between 0-63. Consequently, the last six bits of each byte now hold a val |
| | | 1054 | | // We then compress four such bytes (with valid 4 * 6 = 24 bits) to three UTF8 bytes (3 * 8 = 24 bits). |
| | | 1055 | | // For faster decoding, we use SIMD operations that allow the processing of multiple bytes together. |
| | | 1056 | | // However, the compress operation on adjacent values of a vector could be slower. Thus, we de-interleave wh |
| | | 1057 | | // the input bytes that store adjacent bytes in separate vectors. This later simplifies the compress step wi |
| | | 1058 | | // of logical operations. This requires interleaving while storing the decoded result. |
| | | 1059 | | |
| | | 1060 | | // Values in 'decLutOne' maps input values from 0 to 63. |
| | | 1061 | | // 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 |
| | | 1062 | | // 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 |
| | | 1063 | | // 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63 |
| | | 1064 | | // 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 255, 255, 255 |
| | | 1065 | | var decLutOne = (Vector128<byte>.AllBitsSet, |
| | | 1066 | | Vector128<byte>.AllBitsSet, |
| | | 1067 | | Vector128.Create(decoder.AdvSimdLutOne3).AsByte(), |
| | | 1068 | | Vector128.Create(0x37363534, 0x3B3A3938, 0xFFFF3D3C, 0xFFFFFFFF).AsByte()); |
| | | 1069 | | |
| | | 1070 | | // Values in 'decLutTwo' maps input values from 63 to 127. |
| | | 1071 | | // 0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 |
| | | 1072 | | // 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255 |
| | | 1073 | | // 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39 |
| | | 1074 | | // 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255 |
| | | 1075 | | var decLutTwo = (Vector128.Create(0x0100FF00, 0x05040302, 0x09080706, 0x0D0C0B0A).AsByte(), |
| | | 1076 | | Vector128.Create(0x11100F0E, 0x15141312, 0x19181716, 0xFFFFFFFF).AsByte(), |
| | | 1077 | | Vector128.Create(decoder.AdvSimdLutTwo3Uint1, 0x1F1E1D1C, 0x23222120, 0x27262524).AsByte(), |
| | | 1078 | | Vector128.Create(0x2B2A2928, 0x2F2E2D2C, 0x33323130, 0xFFFFFFFF).AsByte()); |
| | | 1079 | | |
| | | 1080 | | T* src = srcBytes; |
| | | 1081 | | byte* dest = destBytes; |
| | | 1082 | | Vector128<byte> offset = Vector128.Create<byte>(63); |
| | | 1083 | | |
| | | 1084 | | do |
| | | 1085 | | { |
| | | 1086 | | // Step 1: Load 64 bytes and de-interleave. |
| | | 1087 | | if (!decoder.TryLoadArmVector128x4(src, srcStart, sourceLength, |
| | | 1088 | | out Vector128<byte> str1, out Vector128<byte> str2, out Vector128<byte> str3, out Vector128<byte> st |
| | | 1089 | | { |
| | | 1090 | | break; |
| | | 1091 | | } |
| | | 1092 | | |
| | | 1093 | | // Step 2: Map each valid input to its Base64 value. |
| | | 1094 | | // We use two look-ups to compute partial results and combine them later. |
| | | 1095 | | |
| | | 1096 | | // Step 2.1: Detect valid Base64 values from the first three groups. Maps input as, |
| | | 1097 | | // 0 to 63 (Invalid) => 255 |
| | | 1098 | | // 0 to 63 (Valid) => Their Base64 equivalent |
| | | 1099 | | // 64 to 255 => 0 |
| | | 1100 | | |
| | | 1101 | | // Each input value acts as an index in the look-up table 'decLutOne'. |
| | | 1102 | | // e.g., for group 1: index 43 maps to 62 (Base64 '+'). |
| | | 1103 | | // Group 4 and 5 values are out-of-range (>64), so they are mapped to zero. |
| | | 1104 | | // Other valid indices but invalid values are mapped to 255. |
| | | 1105 | | Vector128<byte> decOne1 = AdvSimd.Arm64.VectorTableLookup(decLutOne, str1); |
| | | 1106 | | Vector128<byte> decOne2 = AdvSimd.Arm64.VectorTableLookup(decLutOne, str2); |
| | | 1107 | | Vector128<byte> decOne3 = AdvSimd.Arm64.VectorTableLookup(decLutOne, str3); |
| | | 1108 | | Vector128<byte> decOne4 = AdvSimd.Arm64.VectorTableLookup(decLutOne, str4); |
| | | 1109 | | |
| | | 1110 | | // Step 2.2: Detect valid Base64 values from groups 4 and 5. Maps input as, |
| | | 1111 | | // 0 to 63 => 0 |
| | | 1112 | | // 64 to 122 (Valid) => Their Base64 equivalent |
| | | 1113 | | // 64 to 122 (Invalid) => 255 |
| | | 1114 | | // 123 to 255 => Remains unchanged |
| | | 1115 | | |
| | | 1116 | | // Subtract/offset each input value by 63 so that it can be used as a valid offset. |
| | | 1117 | | // Subtract saturate makes values from the first three groups set to zero that are |
| | | 1118 | | // then mapped to zero in the subsequent look-up. |
| | | 1119 | | Vector128<byte> decTwo1 = AdvSimd.SubtractSaturate(str1, offset); |
| | | 1120 | | Vector128<byte> decTwo2 = AdvSimd.SubtractSaturate(str2, offset); |
| | | 1121 | | Vector128<byte> decTwo3 = AdvSimd.SubtractSaturate(str3, offset); |
| | | 1122 | | Vector128<byte> decTwo4 = AdvSimd.SubtractSaturate(str4, offset); |
| | | 1123 | | |
| | | 1124 | | // We use VTBX to map values where out-of-range indices are unchanged. |
| | | 1125 | | decTwo1 = AdvSimd.Arm64.VectorTableLookupExtension(decTwo1, decLutTwo, decTwo1); |
| | | 1126 | | decTwo2 = AdvSimd.Arm64.VectorTableLookupExtension(decTwo2, decLutTwo, decTwo2); |
| | | 1127 | | decTwo3 = AdvSimd.Arm64.VectorTableLookupExtension(decTwo3, decLutTwo, decTwo3); |
| | | 1128 | | decTwo4 = AdvSimd.Arm64.VectorTableLookupExtension(decTwo4, decLutTwo, decTwo4); |
| | | 1129 | | |
| | | 1130 | | // Step 3: Combine the partial result. |
| | | 1131 | | // Each look-up above maps valid values to their Base64 equivalent or zero. |
| | | 1132 | | // Thus the intermediate results 'decOne' and 'decTwo' could be OR-ed to get final values. |
| | | 1133 | | str1 = (decOne1 | decTwo1); |
| | | 1134 | | str2 = (decOne2 | decTwo2); |
| | | 1135 | | str3 = (decOne3 | decTwo3); |
| | | 1136 | | str4 = (decOne4 | decTwo4); |
| | | 1137 | | |
| | | 1138 | | // Step 4: Detect an invalid input value. |
| | | 1139 | | // Invalid values < 122 are set to 255 while the ones above 122 are unchanged. |
| | | 1140 | | // Check for invalid input, any value larger than 63. |
| | | 1141 | | Vector128<byte> classified = (Vector128.GreaterThan(str1, offset) |
| | | 1142 | | | Vector128.GreaterThan(str2, offset) |
| | | 1143 | | | Vector128.GreaterThan(str3, offset) |
| | | 1144 | | | Vector128.GreaterThan(str4, offset)); |
| | | 1145 | | |
| | | 1146 | | // Check that all bits are zero. |
| | | 1147 | | if (classified != Vector128<byte>.Zero) |
| | | 1148 | | { |
| | | 1149 | | break; |
| | | 1150 | | } |
| | | 1151 | | |
| | | 1152 | | // Step 5: Compress four bytes into three. |
| | | 1153 | | Vector128<byte> res1 = ((str1 << 2) | (str2 >> 4)); |
| | | 1154 | | Vector128<byte> res2 = ((str2 << 4) | (str3 >> 2)); |
| | | 1155 | | Vector128<byte> res3 = ((str3 << 6) | str4); |
| | | 1156 | | |
| | | 1157 | | // Step 6: Interleave and store decoded results. |
| | | 1158 | | AssertWrite<Vector128<byte>>(dest, destStart, destLength); |
| | | 1159 | | AdvSimd.Arm64.StoreVectorAndZip(dest, (res1, res2, res3)); |
| | | 1160 | | |
| | | 1161 | | src += 64; |
| | | 1162 | | dest += 48; |
| | | 1163 | | } |
| | | 1164 | | while (src <= srcEnd); |
| | | 1165 | | |
| | | 1166 | | srcBytes = src; |
| | | 1167 | | destBytes = dest; |
| | | 1168 | | } |
| | | 1169 | | |
| | | 1170 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1171 | | [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] |
| | | 1172 | | [CompExactlyDependsOn(typeof(Ssse3))] |
| | | 1173 | | [CompExactlyDependsOn(typeof(PackedSimd))] |
| | | 1174 | | private static unsafe void Vector128Decode<TBase64Decoder, T>(TBase64Decoder decoder, ref T* srcBytes, ref byte* |
| | | 1175 | | where TBase64Decoder : IBase64Decoder<T> |
| | | 1176 | | where T : unmanaged |
| | | 1177 | | { |
| | | 1178 | | Debug.Assert((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported || PackedSimd.IsSupported) && BitConverter.IsLi |
| | | 1179 | | |
| | | 1180 | | // If we have Vector128 support, pick off 16 bytes at a time for as long as we can, |
| | | 1181 | | // but make sure that we quit before seeing any == markers at the end of the |
| | | 1182 | | // string. Also, because we write four zeroes at the end of the output, ensure |
| | | 1183 | | // that there are at least 6 valid bytes of input data remaining to close the |
| | | 1184 | | // gap. 16 + 2 + 6 = 24 bytes. |
| | | 1185 | | |
| | | 1186 | | // The input consists of six character sets in the Base64 alphabet, |
| | | 1187 | | // which we need to map back to the 6-bit values they represent. |
| | | 1188 | | // There are three ranges, two singles, and then there's the rest. |
| | | 1189 | | // |
| | | 1190 | | // # From To Add Characters |
| | | 1191 | | // 1 [43] [62] +19 + |
| | | 1192 | | // 2 [47] [63] +16 / |
| | | 1193 | | // 3 [48..57] [52..61] +4 0..9 |
| | | 1194 | | // 4 [65..90] [0..25] -65 A..Z |
| | | 1195 | | // 5 [97..122] [26..51] -71 a..z |
| | | 1196 | | // (6) Everything else => invalid input |
| | | 1197 | | |
| | | 1198 | | // We will use LUTS for character validation & offset computation |
| | | 1199 | | // Remember that 0x2X and 0x0X are the same index for _mm_shuffle_epi8, |
| | | 1200 | | // this allows to mask with 0x2F instead of 0x0F and thus save one constant declaration (register and/or mem |
| | | 1201 | | |
| | | 1202 | | // For offsets: |
| | | 1203 | | // Perfect hash for lut = ((src>>4)&0x2F)+((src==0x2F)?0xFF:0x00) |
| | | 1204 | | // 0000 = garbage |
| | | 1205 | | // 0001 = / |
| | | 1206 | | // 0010 = + |
| | | 1207 | | // 0011 = 0-9 |
| | | 1208 | | // 0100 = A-Z |
| | | 1209 | | // 0101 = A-Z |
| | | 1210 | | // 0110 = a-z |
| | | 1211 | | // 0111 = a-z |
| | | 1212 | | // 1000 >= garbage |
| | | 1213 | | |
| | | 1214 | | // For validation, here's the table. |
| | | 1215 | | // A character is valid if and only if the AND of the 2 lookups equals 0: |
| | | 1216 | | |
| | | 1217 | | // hi \ lo 0000 0001 0010 0011 0100 0101 0110 0111 1000 1001 1010 1011 1100 1101 1110 1111 |
| | | 1218 | | // LUT 0x15 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x13 0x1A 0x1B 0x1B 0x1B 0x1A |
| | | 1219 | | |
| | | 1220 | | // 0000 0X10 char NUL SOH STX ETX EOT ENQ ACK BEL BS HT LF VT FF CR SO SI |
| | | 1221 | | // andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 |
| | | 1222 | | |
| | | 1223 | | // 0001 0x10 char DLE DC1 DC2 DC3 DC4 NAK SYN ETB CAN EM SUB ESC FS GS RS US |
| | | 1224 | | // andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 |
| | | 1225 | | |
| | | 1226 | | // 0010 0x01 char ! " # $ % & ' ( ) * + , - . / |
| | | 1227 | | // andlut 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x00 0x01 0x01 0x01 0x00 |
| | | 1228 | | |
| | | 1229 | | // 0011 0x02 char 0 1 2 3 4 5 6 7 8 9 : ; < = > ? |
| | | 1230 | | // andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x02 0x02 0x02 0x02 0x02 0x02 |
| | | 1231 | | |
| | | 1232 | | // 0100 0x04 char @ A B C D E F G H I J K L M N 0 |
| | | 1233 | | // andlut 0x04 0x00 0x00 0x00 0X00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 |
| | | 1234 | | |
| | | 1235 | | // 0101 0x08 char P Q R S T U V W X Y Z [ \ ] ^ _ |
| | | 1236 | | // andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x08 0x08 0x08 0x08 0x08 |
| | | 1237 | | |
| | | 1238 | | // 0110 0x04 char ` a b c d e f g h i j k l m n o |
| | | 1239 | | // andlut 0x04 0x00 0x00 0x00 0X00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 |
| | | 1240 | | // 0111 0X08 char p q r s t u v w x y z { | } ~ |
| | | 1241 | | // andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x08 0x08 0x08 0x08 0x08 |
| | | 1242 | | |
| | | 1243 | | // 1000 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 |
| | | 1244 | | // 1001 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 |
| | | 1245 | | // 1010 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 |
| | | 1246 | | // 1011 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 |
| | | 1247 | | // 1100 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 |
| | | 1248 | | // 1101 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 |
| | | 1249 | | // 1110 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 |
| | | 1250 | | // 1111 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 |
| | | 1251 | | |
| | | 1252 | | // The JIT won't hoist these "constants", so help it |
| | 32951 | 1253 | | Vector128<byte> lutHi = Vector128.Create(decoder.Vector128LutHigh).AsByte(); |
| | 32951 | 1254 | | Vector128<byte> lutLo = Vector128.Create(decoder.Vector128LutLow).AsByte(); |
| | 32951 | 1255 | | Vector128<sbyte> lutShift = Vector128.Create(decoder.Vector128LutShift).AsSByte(); |
| | 32951 | 1256 | | Vector128<sbyte> packBytesMask = Vector128.Create(0x06000102, 0x090A0405, 0x0C0D0E08, 0xffffffff).AsSByte(); |
| | 32951 | 1257 | | Vector128<byte> mergeConstant0 = Vector128.Create(0x01400140).AsByte(); |
| | 32951 | 1258 | | Vector128<short> mergeConstant1 = Vector128.Create(0x00011000).AsInt16(); |
| | 32951 | 1259 | | Vector128<byte> one = Vector128<byte>.One; |
| | 32951 | 1260 | | Vector128<byte> mask2F = Vector128.Create(decoder.MaskSlashOrUnderscore); |
| | 32951 | 1261 | | Vector128<byte> mask8F = Vector128.Create((byte)0x8F); |
| | 32951 | 1262 | | Vector128<byte> shiftForUnderscore = Vector128.Create((byte)33); |
| | 32951 | 1263 | | T* src = srcBytes; |
| | 32951 | 1264 | | byte* dest = destBytes; |
| | | 1265 | | |
| | | 1266 | | //while (remaining >= 24) |
| | | 1267 | | do |
| | | 1268 | | { |
| | 38515 | 1269 | | if (!decoder.TryLoadVector128(src, srcStart, sourceLength, out Vector128<byte> str)) |
| | | 1270 | | { |
| | | 1271 | | break; |
| | | 1272 | | } |
| | | 1273 | | |
| | | 1274 | | // lookup |
| | 38515 | 1275 | | Vector128<byte> hiNibbles = Vector128.ShiftRightLogical(str.AsInt32(), 4).AsByte() & mask2F; |
| | | 1276 | | |
| | 38515 | 1277 | | if (!decoder.TryDecode128Core(str, hiNibbles, mask2F, mask8F, lutLo, lutHi, lutShift, shiftForUnderscore |
| | | 1278 | | { |
| | | 1279 | | break; |
| | | 1280 | | } |
| | | 1281 | | |
| | | 1282 | | // in, bits, upper case are most significant bits, lower case are least significant bits |
| | | 1283 | | // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ |
| | | 1284 | | // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG |
| | | 1285 | | // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD |
| | | 1286 | | // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA |
| | | 1287 | | |
| | | 1288 | | Vector128<short> merge_ab_and_bc; |
| | 8322 | 1289 | | if (Ssse3.IsSupported) |
| | | 1290 | | { |
| | 8322 | 1291 | | merge_ab_and_bc = Ssse3.MultiplyAddAdjacent(str.AsByte(), mergeConstant0.AsSByte()); |
| | | 1292 | | } |
| | | 1293 | | else if (AdvSimd.Arm64.IsSupported) |
| | | 1294 | | { |
| | | 1295 | | Vector128<ushort> evens = AdvSimd.ShiftLeftLogicalWideningLower(AdvSimd.Arm64.UnzipEven(str, one).Ge |
| | | 1296 | | Vector128<ushort> odds = AdvSimd.Arm64.TransposeOdd(str, Vector128<byte>.Zero).AsUInt16(); |
| | | 1297 | | merge_ab_and_bc = Vector128.Add(evens, odds).AsInt16(); |
| | | 1298 | | } |
| | | 1299 | | else if (PackedSimd.IsSupported) |
| | | 1300 | | { |
| | | 1301 | | // MultiplyAddAdjacent by {64,1,...} is the even byte (low of each u16 lane) times 64 plus the odd b |
| | | 1302 | | Vector128<ushort> u = str.AsUInt16(); |
| | | 1303 | | Vector128<ushort> evens = Vector128.ShiftLeft(u & Vector128.Create((ushort)0x00FF), 6); |
| | | 1304 | | Vector128<ushort> odds = Vector128.ShiftRightLogical(u, 8); |
| | | 1305 | | merge_ab_and_bc = (evens + odds).AsInt16(); |
| | | 1306 | | } |
| | | 1307 | | else |
| | | 1308 | | { |
| | | 1309 | | // We explicitly recheck each IsSupported query to ensure that the trimmer can see which paths are l |
| | 0 | 1310 | | ThrowUnreachableException(); |
| | | 1311 | | merge_ab_and_bc = default; |
| | | 1312 | | } |
| | | 1313 | | // 0000kkkk LLllllll 0000JJJJ JJjjKKKK |
| | | 1314 | | // 0000hhhh IIiiiiii 0000GGGG GGggHHHH |
| | | 1315 | | // 0000eeee FFffffff 0000DDDD DDddEEEE |
| | | 1316 | | // 0000bbbb CCcccccc 0000AAAA AAaaBBBB |
| | | 1317 | | |
| | | 1318 | | Vector128<int> output; |
| | 8322 | 1319 | | if (Ssse3.IsSupported) |
| | | 1320 | | { |
| | 8322 | 1321 | | output = Sse2.MultiplyAddAdjacent(merge_ab_and_bc, mergeConstant1); |
| | | 1322 | | } |
| | | 1323 | | else if (AdvSimd.Arm64.IsSupported) |
| | | 1324 | | { |
| | | 1325 | | Vector128<int> ievens = AdvSimd.ShiftLeftLogicalWideningLower(AdvSimd.Arm64.UnzipEven(merge_ab_and_b |
| | | 1326 | | Vector128<int> iodds = AdvSimd.Arm64.TransposeOdd(merge_ab_and_bc, Vector128<short>.Zero).AsInt32(); |
| | | 1327 | | output = Vector128.Add(ievens, iodds).AsInt32(); |
| | | 1328 | | } |
| | | 1329 | | else if (PackedSimd.IsSupported) |
| | | 1330 | | { |
| | | 1331 | | // MultiplyAddAdjacent by {4096,1,...} is the even i16 (low of each i32 lane) times 4096 plus the od |
| | | 1332 | | Vector128<uint> m = merge_ab_and_bc.AsUInt32(); |
| | | 1333 | | Vector128<uint> ievens = Vector128.ShiftLeft(m & Vector128.Create(0x0000FFFFu), 12); |
| | | 1334 | | Vector128<uint> iodds = Vector128.ShiftRightLogical(m, 16); |
| | | 1335 | | output = (ievens + iodds).AsInt32(); |
| | | 1336 | | } |
| | | 1337 | | else |
| | | 1338 | | { |
| | | 1339 | | // We explicitly recheck each IsSupported query to ensure that the trimmer can see which paths are l |
| | 0 | 1340 | | ThrowUnreachableException(); |
| | | 1341 | | output = default; |
| | | 1342 | | } |
| | | 1343 | | // 00000000 JJJJJJjj KKKKkkkk LLllllll |
| | | 1344 | | // 00000000 GGGGGGgg HHHHhhhh IIiiiiii |
| | | 1345 | | // 00000000 DDDDDDdd EEEEeeee FFffffff |
| | | 1346 | | // 00000000 AAAAAAaa BBBBbbbb CCcccccc |
| | | 1347 | | |
| | | 1348 | | // Pack bytes together: |
| | 8322 | 1349 | | str = SimdShuffle(output.AsByte(), packBytesMask.AsByte(), mask8F); |
| | | 1350 | | // 00000000 00000000 00000000 00000000 |
| | | 1351 | | // LLllllll KKKKkkkk JJJJJJjj IIiiiiii |
| | | 1352 | | // HHHHhhhh GGGGGGgg FFffffff EEEEeeee |
| | | 1353 | | // DDDDDDdd CCcccccc BBBBbbbb AAAAAAaa |
| | | 1354 | | |
| | 8322 | 1355 | | AssertWrite<Vector128<sbyte>>(dest, destStart, destLength); |
| | 8322 | 1356 | | str.Store(dest); |
| | | 1357 | | |
| | 8322 | 1358 | | src += 16; |
| | 8322 | 1359 | | dest += 12; |
| | | 1360 | | } |
| | 8322 | 1361 | | while (src <= srcEnd); |
| | | 1362 | | |
| | 32951 | 1363 | | srcBytes = src; |
| | 32951 | 1364 | | destBytes = dest; |
| | 32951 | 1365 | | } |
| | | 1366 | | #endif |
| | | 1367 | | |
| | | 1368 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1369 | | private static unsafe void WriteThreeLowOrderBytes(byte* destination, int value) |
| | | 1370 | | { |
| | 148955 | 1371 | | destination[0] = (byte)(value >> 16); |
| | 148955 | 1372 | | destination[1] = (byte)(value >> 8); |
| | 148955 | 1373 | | destination[2] = (byte)value; |
| | 148955 | 1374 | | } |
| | | 1375 | | |
| | | 1376 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1377 | | internal static bool IsWhiteSpace(int value) |
| | | 1378 | | { |
| | 680759 | 1379 | | Debug.Assert(value >= 0 && value <= ushort.MaxValue); |
| | | 1380 | | uint charMinusLowUInt32; |
| | 680759 | 1381 | | return (int)((0xC8000100U << (short)(charMinusLowUInt32 = (ushort)(value - '\t'))) & (charMinusLowUInt32 - 3 |
| | | 1382 | | } |
| | | 1383 | | |
| | | 1384 | | internal readonly struct Base64DecoderByte : IBase64Decoder<byte> |
| | | 1385 | | { |
| | | 1386 | | // Pre-computing this table using a custom string(s_characters) and GenerateDecodingMapAndVerify (found in t |
| | | 1387 | | public ReadOnlySpan<sbyte> DecodingMap => |
| | 0 | 1388 | | [ |
| | 0 | 1389 | | -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, |
| | 0 | 1390 | | -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, |
| | 0 | 1391 | | -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, //62 is placed at index 43 ( |
| | 0 | 1392 | | 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, //52-61 are placed at index |
| | 0 | 1393 | | -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, |
| | 0 | 1394 | | 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, //0-25 are placed at index 6 |
| | 0 | 1395 | | -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, |
| | 0 | 1396 | | 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, //26-51 are placed at index |
| | 0 | 1397 | | -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // Bytes over 122 ('z') are |
| | 0 | 1398 | | -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // Hence, padding the map wi |
| | 0 | 1399 | | -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, |
| | 0 | 1400 | | -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, |
| | 0 | 1401 | | -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, |
| | 0 | 1402 | | -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, |
| | 0 | 1403 | | -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, |
| | 0 | 1404 | | -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, |
| | 0 | 1405 | | ]; |
| | | 1406 | | |
| | | 1407 | | public ReadOnlySpan<uint> VbmiLookup0 => |
| | 0 | 1408 | | [ |
| | 0 | 1409 | | 0x80808080, 0x80808080, 0x80808080, 0x80808080, |
| | 0 | 1410 | | 0x80808080, 0x80808080, 0x80808080, 0x80808080, |
| | 0 | 1411 | | 0x80808080, 0x80808080, 0x3e808080, 0x3f808080, |
| | 0 | 1412 | | 0x37363534, 0x3b3a3938, 0x80803d3c, 0x80808080 |
| | 0 | 1413 | | ]; |
| | | 1414 | | |
| | | 1415 | | public ReadOnlySpan<uint> VbmiLookup1 => |
| | 0 | 1416 | | [ |
| | 0 | 1417 | | 0x02010080, 0x06050403, 0x0a090807, 0x0e0d0c0b, |
| | 0 | 1418 | | 0x1211100f, 0x16151413, 0x80191817, 0x80808080, |
| | 0 | 1419 | | 0x1c1b1a80, 0x201f1e1d, 0x24232221, 0x28272625, |
| | 0 | 1420 | | 0x2c2b2a29, 0x302f2e2d, 0x80333231, 0x80808080 |
| | 0 | 1421 | | ]; |
| | | 1422 | | |
| | | 1423 | | public ReadOnlySpan<sbyte> Avx2LutHigh => |
| | 0 | 1424 | | [ |
| | 0 | 1425 | | 0x10, 0x10, 0x01, 0x02, |
| | 0 | 1426 | | 0x04, 0x08, 0x04, 0x08, |
| | 0 | 1427 | | 0x10, 0x10, 0x10, 0x10, |
| | 0 | 1428 | | 0x10, 0x10, 0x10, 0x10, |
| | 0 | 1429 | | 0x10, 0x10, 0x01, 0x02, |
| | 0 | 1430 | | 0x04, 0x08, 0x04, 0x08, |
| | 0 | 1431 | | 0x10, 0x10, 0x10, 0x10, |
| | 0 | 1432 | | 0x10, 0x10, 0x10, 0x10 |
| | 0 | 1433 | | ]; |
| | | 1434 | | |
| | | 1435 | | public ReadOnlySpan<sbyte> Avx2LutLow => |
| | 0 | 1436 | | [ |
| | 0 | 1437 | | 0x15, 0x11, 0x11, 0x11, |
| | 0 | 1438 | | 0x11, 0x11, 0x11, 0x11, |
| | 0 | 1439 | | 0x11, 0x11, 0x13, 0x1A, |
| | 0 | 1440 | | 0x1B, 0x1B, 0x1B, 0x1A, |
| | 0 | 1441 | | 0x15, 0x11, 0x11, 0x11, |
| | 0 | 1442 | | 0x11, 0x11, 0x11, 0x11, |
| | 0 | 1443 | | 0x11, 0x11, 0x13, 0x1A, |
| | 0 | 1444 | | 0x1B, 0x1B, 0x1B, 0x1A |
| | 0 | 1445 | | ]; |
| | | 1446 | | |
| | | 1447 | | public ReadOnlySpan<sbyte> Avx2LutShift => |
| | 0 | 1448 | | [ |
| | 0 | 1449 | | 0, 16, 19, 4, |
| | 0 | 1450 | | -65, -65, -71, -71, |
| | 0 | 1451 | | 0, 0, 0, 0, |
| | 0 | 1452 | | 0, 0, 0, 0, |
| | 0 | 1453 | | 0, 16, 19, 4, |
| | 0 | 1454 | | -65, -65, -71, -71, |
| | 0 | 1455 | | 0, 0, 0, 0, |
| | 0 | 1456 | | 0, 0, 0, 0 |
| | 0 | 1457 | | ]; |
| | | 1458 | | |
| | 0 | 1459 | | public byte MaskSlashOrUnderscore => (byte)'/'; |
| | | 1460 | | |
| | 0 | 1461 | | public ReadOnlySpan<int> Vector128LutHigh => [0x02011010, 0x08040804, 0x10101010, 0x10101010]; |
| | | 1462 | | |
| | 0 | 1463 | | public ReadOnlySpan<int> Vector128LutLow => [0x11111115, 0x11111111, 0x1A131111, 0x1A1B1B1B]; |
| | | 1464 | | |
| | 0 | 1465 | | public ReadOnlySpan<uint> Vector128LutShift => [0x04131000, 0xb9b9bfbf, 0x00000000, 0x00000000]; |
| | | 1466 | | |
| | 0 | 1467 | | public ReadOnlySpan<uint> AdvSimdLutOne3 => [0xFFFFFFFF, 0xFFFFFFFF, 0x3EFFFFFF, 0x3FFFFFFF]; |
| | | 1468 | | |
| | 0 | 1469 | | public uint AdvSimdLutTwo3Uint1 => 0x1B1AFFFF; |
| | | 1470 | | |
| | 0 | 1471 | | public int GetMaxDecodedLength(int utf8Length) => Base64.GetMaxDecodedFromUtf8Length(utf8Length); |
| | | 1472 | | |
| | 0 | 1473 | | public bool IsInvalidLength(int bufferLength) => bufferLength % 4 != 0; // only decode input if it is a mult |
| | | 1474 | | |
| | 0 | 1475 | | public bool IsValidPadding(uint padChar) => padChar == EncodingPad; |
| | | 1476 | | |
| | 0 | 1477 | | public int SrcLength(bool _, int utf8Length) => utf8Length & ~0x3; // only decode input up to the closest m |
| | | 1478 | | |
| | | 1479 | | #if NET |
| | | 1480 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1481 | | [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] |
| | | 1482 | | [CompExactlyDependsOn(typeof(Ssse3))] |
| | | 1483 | | [CompExactlyDependsOn(typeof(PackedSimd))] |
| | | 1484 | | public bool TryDecode128Core( |
| | | 1485 | | Vector128<byte> str, |
| | | 1486 | | Vector128<byte> hiNibbles, |
| | | 1487 | | Vector128<byte> maskSlashOrUnderscore, |
| | | 1488 | | Vector128<byte> mask8F, |
| | | 1489 | | Vector128<byte> lutLow, |
| | | 1490 | | Vector128<byte> lutHigh, |
| | | 1491 | | Vector128<sbyte> lutShift, |
| | | 1492 | | Vector128<byte> _, |
| | | 1493 | | out Vector128<byte> result) |
| | | 1494 | | { |
| | 0 | 1495 | | Vector128<byte> loNibbles = str & maskSlashOrUnderscore; |
| | 0 | 1496 | | Vector128<byte> hi = SimdShuffle(lutHigh, hiNibbles, mask8F); |
| | 0 | 1497 | | Vector128<byte> lo = SimdShuffle(lutLow, loNibbles, mask8F); |
| | | 1498 | | |
| | | 1499 | | // Check for invalid input: if any "and" values from lo and hi are not zero, |
| | | 1500 | | // fall back on bytewise code to do error checking and reporting: |
| | 0 | 1501 | | if ((lo & hi) != Vector128<byte>.Zero) |
| | | 1502 | | { |
| | 0 | 1503 | | result = default; |
| | 0 | 1504 | | return false; |
| | | 1505 | | } |
| | | 1506 | | |
| | 0 | 1507 | | Vector128<byte> eq2F = Vector128.Equals(str, maskSlashOrUnderscore); |
| | 0 | 1508 | | Vector128<byte> shift = SimdShuffle(lutShift.AsByte(), (eq2F + hiNibbles), mask8F); |
| | | 1509 | | |
| | | 1510 | | // Now simply add the delta values to the input: |
| | 0 | 1511 | | result = str + shift; |
| | | 1512 | | |
| | 0 | 1513 | | return true; |
| | | 1514 | | } |
| | | 1515 | | |
| | | 1516 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1517 | | [CompExactlyDependsOn(typeof(Avx2))] |
| | | 1518 | | public bool TryDecode256Core( |
| | | 1519 | | Vector256<sbyte> str, |
| | | 1520 | | Vector256<sbyte> hiNibbles, |
| | | 1521 | | Vector256<sbyte> maskSlashOrUnderscore, |
| | | 1522 | | Vector256<sbyte> lutLow, |
| | | 1523 | | Vector256<sbyte> lutHigh, |
| | | 1524 | | Vector256<sbyte> lutShift, |
| | | 1525 | | Vector256<sbyte> _, |
| | | 1526 | | out Vector256<sbyte> result) |
| | | 1527 | | { |
| | 0 | 1528 | | Vector256<sbyte> loNibbles = str & maskSlashOrUnderscore; |
| | 0 | 1529 | | Vector256<sbyte> hi = Avx2.Shuffle(lutHigh, hiNibbles); |
| | 0 | 1530 | | Vector256<sbyte> lo = Avx2.Shuffle(lutLow, loNibbles); |
| | | 1531 | | |
| | 0 | 1532 | | if ((lo & hi) != Vector256<sbyte>.Zero) |
| | | 1533 | | { |
| | 0 | 1534 | | result = default; |
| | 0 | 1535 | | return false; |
| | | 1536 | | } |
| | | 1537 | | |
| | 0 | 1538 | | Vector256<sbyte> eq2F = Avx2.CompareEqual(str, maskSlashOrUnderscore); |
| | 0 | 1539 | | Vector256<sbyte> shift = Avx2.Shuffle(lutShift, eq2F + hiNibbles); |
| | | 1540 | | |
| | 0 | 1541 | | result = str + shift; |
| | | 1542 | | |
| | 0 | 1543 | | return true; |
| | | 1544 | | } |
| | | 1545 | | |
| | | 1546 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1547 | | public unsafe bool TryLoadVector512(byte* src, byte* srcStart, int sourceLength, out Vector512<sbyte> str) |
| | | 1548 | | { |
| | 36409 | 1549 | | AssertRead<Vector512<sbyte>>(src, srcStart, sourceLength); |
| | 36409 | 1550 | | str = Vector512.Load(src).AsSByte(); |
| | 36409 | 1551 | | return true; |
| | | 1552 | | } |
| | | 1553 | | |
| | | 1554 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1555 | | [CompExactlyDependsOn(typeof(Avx2))] |
| | | 1556 | | public unsafe bool TryLoadAvxVector256(byte* src, byte* srcStart, int sourceLength, out Vector256<sbyte> str |
| | | 1557 | | { |
| | 31513 | 1558 | | AssertRead<Vector256<sbyte>>(src, srcStart, sourceLength); |
| | 31513 | 1559 | | str = Avx.LoadVector256(src).AsSByte(); |
| | 31513 | 1560 | | return true; |
| | | 1561 | | } |
| | | 1562 | | |
| | | 1563 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1564 | | public unsafe bool TryLoadVector128(byte* src, byte* srcStart, int sourceLength, out Vector128<byte> str) |
| | | 1565 | | { |
| | 36293 | 1566 | | AssertRead<Vector128<sbyte>>(src, srcStart, sourceLength); |
| | 36293 | 1567 | | str = Vector128.LoadUnsafe(ref *src); |
| | 36293 | 1568 | | return true; |
| | | 1569 | | } |
| | | 1570 | | |
| | | 1571 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1572 | | [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] |
| | | 1573 | | public unsafe bool TryLoadArmVector128x4(byte* src, byte* srcStart, int sourceLength, |
| | | 1574 | | out Vector128<byte> str1, out Vector128<byte> str2, out Vector128<byte> str3, out Vector128<byte> str4) |
| | | 1575 | | { |
| | 0 | 1576 | | AssertRead<Vector128<byte>>(src, srcStart, sourceLength); |
| | 0 | 1577 | | (str1, str2, str3, str4) = AdvSimd.Arm64.Load4xVector128AndUnzip(src); |
| | | 1578 | | |
| | 0 | 1579 | | return true; |
| | | 1580 | | } |
| | | 1581 | | #endif // NET |
| | | 1582 | | |
| | | 1583 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1584 | | public unsafe int DecodeFourElements(byte* source, ref sbyte decodingMap) |
| | | 1585 | | { |
| | | 1586 | | // The 'source' span expected to have at least 4 elements, and the 'decodingMap' consists 256 sbytes |
| | 105578 | 1587 | | uint t0 = source[0]; |
| | 105578 | 1588 | | uint t1 = source[1]; |
| | 105578 | 1589 | | uint t2 = source[2]; |
| | 105578 | 1590 | | uint t3 = source[3]; |
| | | 1591 | | |
| | 105578 | 1592 | | int i0 = Unsafe.Add(ref decodingMap, (int)t0); |
| | 105578 | 1593 | | int i1 = Unsafe.Add(ref decodingMap, (int)t1); |
| | 105578 | 1594 | | int i2 = Unsafe.Add(ref decodingMap, (int)t2); |
| | 105578 | 1595 | | int i3 = Unsafe.Add(ref decodingMap, (int)t3); |
| | | 1596 | | |
| | 105578 | 1597 | | i0 <<= 18; |
| | 105578 | 1598 | | i1 <<= 12; |
| | 105578 | 1599 | | i2 <<= 6; |
| | | 1600 | | |
| | 105578 | 1601 | | i0 |= i3; |
| | 105578 | 1602 | | i1 |= i2; |
| | | 1603 | | |
| | 105578 | 1604 | | i0 |= i1; |
| | 105578 | 1605 | | return i0; |
| | | 1606 | | } |
| | | 1607 | | |
| | | 1608 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1609 | | public unsafe int DecodeRemaining(byte* srcEnd, ref sbyte decodingMap, long remaining, out uint t2, out uint |
| | | 1610 | | { |
| | | 1611 | | uint t0; |
| | | 1612 | | uint t1; |
| | 2357 | 1613 | | t2 = EncodingPad; |
| | 2357 | 1614 | | t3 = EncodingPad; |
| | | 1615 | | switch (remaining) |
| | | 1616 | | { |
| | | 1617 | | case 2: |
| | 629 | 1618 | | t0 = srcEnd[-2]; |
| | 629 | 1619 | | t1 = srcEnd[-1]; |
| | 629 | 1620 | | break; |
| | | 1621 | | case 3: |
| | 628 | 1622 | | t0 = srcEnd[-3]; |
| | 628 | 1623 | | t1 = srcEnd[-2]; |
| | 628 | 1624 | | t2 = srcEnd[-1]; |
| | 628 | 1625 | | break; |
| | | 1626 | | case 4: |
| | 928 | 1627 | | t0 = srcEnd[-4]; |
| | 928 | 1628 | | t1 = srcEnd[-3]; |
| | 928 | 1629 | | t2 = srcEnd[-2]; |
| | 928 | 1630 | | t3 = srcEnd[-1]; |
| | 928 | 1631 | | break; |
| | | 1632 | | default: |
| | 172 | 1633 | | return -1; |
| | | 1634 | | } |
| | | 1635 | | |
| | 2185 | 1636 | | int i0 = Unsafe.Add(ref decodingMap, (IntPtr)t0); |
| | 2185 | 1637 | | int i1 = Unsafe.Add(ref decodingMap, (IntPtr)t1); |
| | | 1638 | | |
| | 2185 | 1639 | | i0 <<= 18; |
| | 2185 | 1640 | | i1 <<= 12; |
| | | 1641 | | |
| | 2185 | 1642 | | i0 |= i1; |
| | 2185 | 1643 | | return i0; |
| | | 1644 | | } |
| | | 1645 | | |
| | | 1646 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1647 | | public int IndexOfAnyExceptWhiteSpace(ReadOnlySpan<byte> span) |
| | | 1648 | | { |
| | 130248 | 1649 | | for (int i = 0; i < span.Length; i++) |
| | | 1650 | | { |
| | 65058 | 1651 | | if (!IsWhiteSpace(span[i])) |
| | | 1652 | | { |
| | 30795 | 1653 | | return i; |
| | | 1654 | | } |
| | | 1655 | | } |
| | | 1656 | | |
| | 66 | 1657 | | return -1; |
| | | 1658 | | } |
| | | 1659 | | |
| | | 1660 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1661 | | public OperationStatus DecodeWithWhiteSpaceBlockwiseWrapper<TBase64Decoder>(TBase64Decoder decoder, ReadOnly |
| | | 1662 | | Span<byte> bytes, ref int bytesConsumed, ref int bytesWritten, bool isFinalBlock = true) |
| | | 1663 | | where TBase64Decoder : IBase64Decoder<byte> => |
| | 0 | 1664 | | DecodeWithWhiteSpaceBlockwise(decoder, utf8, bytes, ref bytesConsumed, ref bytesWritten, isFinalBlock); |
| | | 1665 | | } |
| | | 1666 | | |
| | | 1667 | | internal readonly struct Base64DecoderChar : IBase64Decoder<ushort> |
| | | 1668 | | { |
| | 0 | 1669 | | public ReadOnlySpan<sbyte> DecodingMap => default(Base64DecoderByte).DecodingMap; |
| | | 1670 | | |
| | 0 | 1671 | | public ReadOnlySpan<uint> VbmiLookup0 => default(Base64DecoderByte).VbmiLookup0; |
| | | 1672 | | |
| | 0 | 1673 | | public ReadOnlySpan<uint> VbmiLookup1 => default(Base64DecoderByte).VbmiLookup1; |
| | | 1674 | | |
| | 0 | 1675 | | public ReadOnlySpan<sbyte> Avx2LutHigh => default(Base64DecoderByte).Avx2LutHigh; |
| | | 1676 | | |
| | 0 | 1677 | | public ReadOnlySpan<sbyte> Avx2LutLow => default(Base64DecoderByte).Avx2LutLow; |
| | | 1678 | | |
| | 0 | 1679 | | public ReadOnlySpan<sbyte> Avx2LutShift => default(Base64DecoderByte).Avx2LutShift; |
| | | 1680 | | |
| | 0 | 1681 | | public byte MaskSlashOrUnderscore => default(Base64DecoderByte).MaskSlashOrUnderscore; |
| | | 1682 | | |
| | 0 | 1683 | | public ReadOnlySpan<int> Vector128LutHigh => default(Base64DecoderByte).Vector128LutHigh; |
| | | 1684 | | |
| | 0 | 1685 | | public ReadOnlySpan<int> Vector128LutLow => default(Base64DecoderByte).Vector128LutLow; |
| | | 1686 | | |
| | 0 | 1687 | | public ReadOnlySpan<uint> Vector128LutShift => default(Base64DecoderByte).Vector128LutShift; |
| | | 1688 | | |
| | 0 | 1689 | | public ReadOnlySpan<uint> AdvSimdLutOne3 => default(Base64DecoderByte).AdvSimdLutOne3; |
| | | 1690 | | |
| | 0 | 1691 | | public uint AdvSimdLutTwo3Uint1 => default(Base64DecoderByte).AdvSimdLutTwo3Uint1; |
| | | 1692 | | |
| | 0 | 1693 | | public int GetMaxDecodedLength(int sourceLength) => Base64.GetMaxDecodedFromUtf8Length(sourceLength); |
| | | 1694 | | |
| | 0 | 1695 | | public bool IsInvalidLength(int bufferLength) => bufferLength % 4 != 0; |
| | | 1696 | | |
| | 0 | 1697 | | public bool IsValidPadding(uint padChar) => padChar == EncodingPad; |
| | | 1698 | | |
| | 0 | 1699 | | public int SrcLength(bool _, int sourceLength) => sourceLength & ~0x3; |
| | | 1700 | | |
| | | 1701 | | #if NET |
| | | 1702 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1703 | | [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] |
| | | 1704 | | [CompExactlyDependsOn(typeof(Ssse3))] |
| | | 1705 | | [CompExactlyDependsOn(typeof(PackedSimd))] |
| | | 1706 | | public bool TryDecode128Core(Vector128<byte> str, Vector128<byte> hiNibbles, Vector128<byte> maskSlashOrUnde |
| | | 1707 | | Vector128<byte> lutLow, Vector128<byte> lutHigh, Vector128<sbyte> lutShift, Vector128<byte> shiftForUnde |
| | 0 | 1708 | | default(Base64DecoderByte).TryDecode128Core(str, hiNibbles, maskSlashOrUnderscore, mask8F, lutLow, lutHi |
| | | 1709 | | |
| | | 1710 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1711 | | [CompExactlyDependsOn(typeof(Avx2))] |
| | | 1712 | | public bool TryDecode256Core(Vector256<sbyte> str, Vector256<sbyte> hiNibbles, Vector256<sbyte> maskSlashOrU |
| | | 1713 | | Vector256<sbyte> lutHigh, Vector256<sbyte> lutShift, Vector256<sbyte> shiftForUnderscore, out Vector256< |
| | 0 | 1714 | | default(Base64DecoderByte).TryDecode256Core(str, hiNibbles, maskSlashOrUnderscore, lutLow, lutHigh, lutS |
| | | 1715 | | |
| | | 1716 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1717 | | public unsafe bool TryLoadVector512(ushort* src, ushort* srcStart, int sourceLength, out Vector512<sbyte> st |
| | | 1718 | | { |
| | 17316 | 1719 | | AssertRead<Vector512<ushort>>(src, srcStart, sourceLength); |
| | 17316 | 1720 | | Vector512<ushort> utf16VectorLower = Vector512.Load(src); |
| | 17316 | 1721 | | Vector512<ushort> utf16VectorUpper = Vector512.Load(src + 32); |
| | 17316 | 1722 | | if (Ascii.VectorContainsNonAsciiChar(utf16VectorLower | utf16VectorUpper)) |
| | | 1723 | | { |
| | 0 | 1724 | | str = default; |
| | 0 | 1725 | | return false; |
| | | 1726 | | } |
| | | 1727 | | |
| | 17316 | 1728 | | str = Ascii.ExtractAsciiVector(utf16VectorLower, utf16VectorUpper).AsSByte(); |
| | 17316 | 1729 | | return true; |
| | | 1730 | | } |
| | | 1731 | | |
| | | 1732 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1733 | | [CompExactlyDependsOn(typeof(Avx2))] |
| | | 1734 | | public unsafe bool TryLoadAvxVector256(ushort* src, ushort* srcStart, int sourceLength, out Vector256<sbyte> |
| | | 1735 | | { |
| | 1672 | 1736 | | AssertRead<Vector256<sbyte>>(src, srcStart, sourceLength); |
| | 1672 | 1737 | | Vector256<ushort> utf16VectorLower = Avx.LoadVector256(src); |
| | 1672 | 1738 | | Vector256<ushort> utf16VectorUpper = Avx.LoadVector256(src + 16); |
| | | 1739 | | |
| | 1672 | 1740 | | if (Ascii.VectorContainsNonAsciiChar(utf16VectorLower | utf16VectorUpper)) |
| | | 1741 | | { |
| | 0 | 1742 | | str = default; |
| | 0 | 1743 | | return false; |
| | | 1744 | | } |
| | | 1745 | | |
| | 1672 | 1746 | | str = Ascii.ExtractAsciiVector(utf16VectorLower, utf16VectorUpper).AsSByte(); |
| | 1672 | 1747 | | return true; |
| | | 1748 | | } |
| | | 1749 | | |
| | | 1750 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1751 | | public unsafe bool TryLoadVector128(ushort* src, ushort* srcStart, int sourceLength, out Vector128<byte> str |
| | | 1752 | | { |
| | 2222 | 1753 | | AssertRead<Vector128<sbyte>>(src, srcStart, sourceLength); |
| | 2222 | 1754 | | Vector128<ushort> utf16VectorLower = Vector128.LoadUnsafe(ref *src); |
| | 2222 | 1755 | | Vector128<ushort> utf16VectorUpper = Vector128.LoadUnsafe(ref *src, 8); |
| | 2222 | 1756 | | if (Ascii.VectorContainsNonAsciiChar(utf16VectorLower | utf16VectorUpper)) |
| | | 1757 | | { |
| | 0 | 1758 | | str = default; |
| | 0 | 1759 | | return false; |
| | | 1760 | | } |
| | | 1761 | | |
| | 2222 | 1762 | | str = Ascii.ExtractAsciiVector(utf16VectorLower, utf16VectorUpper); |
| | 2222 | 1763 | | return true; |
| | | 1764 | | } |
| | | 1765 | | |
| | | 1766 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1767 | | [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] |
| | | 1768 | | public unsafe bool TryLoadArmVector128x4(ushort* src, ushort* srcStart, int sourceLength, |
| | | 1769 | | out Vector128<byte> str1, out Vector128<byte> str2, out Vector128<byte> str3, out Vector128<byte> str4) |
| | | 1770 | | { |
| | 0 | 1771 | | AssertRead<Vector128<sbyte>>(src, srcStart, sourceLength); |
| | 0 | 1772 | | var (s11, s12, s21, s22) = AdvSimd.Arm64.Load4xVector128AndUnzip(src); |
| | 0 | 1773 | | var (s31, s32, s41, s42) = AdvSimd.Arm64.Load4xVector128AndUnzip(src + 32); |
| | | 1774 | | |
| | 0 | 1775 | | if (Ascii.VectorContainsNonAsciiChar(s11 | s12 | s21 | s22 | s31 | s32 | s41 | s42)) |
| | | 1776 | | { |
| | 0 | 1777 | | str1 = str2 = str3 = str4 = default; |
| | 0 | 1778 | | return false; |
| | | 1779 | | } |
| | | 1780 | | |
| | 0 | 1781 | | str1 = Ascii.ExtractAsciiVector(s11, s31); |
| | 0 | 1782 | | str2 = Ascii.ExtractAsciiVector(s12, s32); |
| | 0 | 1783 | | str3 = Ascii.ExtractAsciiVector(s21, s41); |
| | 0 | 1784 | | str4 = Ascii.ExtractAsciiVector(s22, s42); |
| | | 1785 | | |
| | 0 | 1786 | | return true; |
| | | 1787 | | } |
| | | 1788 | | #endif // NET |
| | | 1789 | | |
| | | 1790 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1791 | | public unsafe int DecodeFourElements(ushort* source, ref sbyte decodingMap) |
| | | 1792 | | { |
| | | 1793 | | // The 'source' span expected to have at least 4 elements, and the 'decodingMap' consists 256 sbytes |
| | 8950 | 1794 | | uint t0 = source[0]; |
| | 8950 | 1795 | | uint t1 = source[1]; |
| | 8950 | 1796 | | uint t2 = source[2]; |
| | 8950 | 1797 | | uint t3 = source[3]; |
| | | 1798 | | |
| | 8950 | 1799 | | if (((t0 | t1 | t2 | t3) & 0xffffff00) != 0) |
| | | 1800 | | { |
| | 0 | 1801 | | return -1; // One or more chars falls outside the 00..ff range, invalid Base64 character. |
| | | 1802 | | } |
| | | 1803 | | |
| | 8950 | 1804 | | int i0 = Unsafe.Add(ref decodingMap, (int)t0); |
| | 8950 | 1805 | | int i1 = Unsafe.Add(ref decodingMap, (int)t1); |
| | 8950 | 1806 | | int i2 = Unsafe.Add(ref decodingMap, (int)t2); |
| | 8950 | 1807 | | int i3 = Unsafe.Add(ref decodingMap, (int)t3); |
| | | 1808 | | |
| | 8950 | 1809 | | i0 <<= 18; |
| | 8950 | 1810 | | i1 <<= 12; |
| | 8950 | 1811 | | i2 <<= 6; |
| | | 1812 | | |
| | 8950 | 1813 | | i0 |= i3; |
| | 8950 | 1814 | | i1 |= i2; |
| | | 1815 | | |
| | 8950 | 1816 | | i0 |= i1; |
| | 8950 | 1817 | | return i0; |
| | | 1818 | | } |
| | | 1819 | | |
| | | 1820 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1821 | | public unsafe int DecodeRemaining(ushort* srcEnd, ref sbyte decodingMap, long remaining, out uint t2, out ui |
| | | 1822 | | { |
| | | 1823 | | uint t0; |
| | | 1824 | | uint t1; |
| | 1638 | 1825 | | t2 = EncodingPad; |
| | 1638 | 1826 | | t3 = EncodingPad; |
| | | 1827 | | switch (remaining) |
| | | 1828 | | { |
| | | 1829 | | case 2: |
| | 613 | 1830 | | t0 = srcEnd[-2]; |
| | 613 | 1831 | | t1 = srcEnd[-1]; |
| | 613 | 1832 | | break; |
| | | 1833 | | case 3: |
| | 443 | 1834 | | t0 = srcEnd[-3]; |
| | 443 | 1835 | | t1 = srcEnd[-2]; |
| | 443 | 1836 | | t2 = srcEnd[-1]; |
| | 443 | 1837 | | break; |
| | | 1838 | | case 4: |
| | 582 | 1839 | | t0 = srcEnd[-4]; |
| | 582 | 1840 | | t1 = srcEnd[-3]; |
| | 582 | 1841 | | t2 = srcEnd[-2]; |
| | 582 | 1842 | | t3 = srcEnd[-1]; |
| | 582 | 1843 | | break; |
| | | 1844 | | default: |
| | 0 | 1845 | | return -1; |
| | | 1846 | | } |
| | | 1847 | | |
| | 1638 | 1848 | | if (((t0 | t1 | t2 | t3) & 0xffffff00) != 0) |
| | | 1849 | | { |
| | 0 | 1850 | | return -1; |
| | | 1851 | | } |
| | | 1852 | | |
| | 1638 | 1853 | | int i0 = Unsafe.Add(ref decodingMap, (IntPtr)t0); |
| | 1638 | 1854 | | int i1 = Unsafe.Add(ref decodingMap, (IntPtr)t1); |
| | | 1855 | | |
| | 1638 | 1856 | | i0 <<= 18; |
| | 1638 | 1857 | | i1 <<= 12; |
| | | 1858 | | |
| | 1638 | 1859 | | i0 |= i1; |
| | 1638 | 1860 | | return i0; |
| | | 1861 | | } |
| | | 1862 | | |
| | | 1863 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1864 | | public int IndexOfAnyExceptWhiteSpace(ReadOnlySpan<ushort> span) |
| | | 1865 | | { |
| | 0 | 1866 | | for (int i = 0; i < span.Length; i++) |
| | | 1867 | | { |
| | 0 | 1868 | | if (!IsWhiteSpace(span[i])) |
| | | 1869 | | { |
| | 0 | 1870 | | return i; |
| | | 1871 | | } |
| | | 1872 | | } |
| | | 1873 | | |
| | 0 | 1874 | | return -1; |
| | | 1875 | | } |
| | | 1876 | | |
| | | 1877 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 1878 | | public OperationStatus DecodeWithWhiteSpaceBlockwiseWrapper<TBase64Decoder>(TBase64Decoder decoder, ReadOnly |
| | | 1879 | | Span<byte> bytes, ref int bytesConsumed, ref int bytesWritten, bool isFinalBlock = true) where TBase64De |
| | 0 | 1880 | | DecodeWithWhiteSpaceBlockwise(default(Base64DecoderChar), source, bytes, ref bytesConsumed, ref bytesWri |
| | | 1881 | | } |
| | | 1882 | | } |
| | | 1883 | | } |
| | | 1884 | | |