Browse Source

Added experimental json serializer

voice-allocs
RogueException 7 years ago
parent
commit
501c76b2bd
100 changed files with 21907 additions and 93 deletions
  1. +2
    -1
      Discord.Net.targets
  2. +7
    -1
      src/Discord.Net.Core/Discord.Net.Core.csproj
  3. +2
    -1
      src/Discord.Net.Core/Net/Rest/IRestClient.cs
  4. +5
    -5
      src/Discord.Net.Core/Net/Rest/RestResponse.cs
  5. +2
    -3
      src/Discord.Net.Core/Net/WebSockets/IWebSocketClient.cs
  6. +160
    -0
      src/Discord.Net.Core/_corefxlab/BufferExtensions.cs
  7. +109
    -0
      src/Discord.Net.Core/_corefxlab/System.Binary/System/Binary/BufferReader.cs
  8. +62
    -0
      src/Discord.Net.Core/_corefxlab/System.Binary/System/Binary/BufferWriter.cs
  9. +68
    -0
      src/Discord.Net.Core/_corefxlab/System.Binary/System/Binary/UnsafeUtilities.cs
  10. +238
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/Buffer.cs
  11. +19
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/BufferExtensions.cs
  12. +46
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/BufferHandle.cs
  13. +25
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/BufferPool.cs
  14. +14
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/IOutput.cs
  15. +12
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/IRetainable.cs
  16. +75
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/OwnedBuffer.cs
  17. +223
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/ReadOnlyBuffer.cs
  18. +18
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/Transformation.cs
  19. +101
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Internal/ManagedBufferPool.cs
  20. +88
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Internal/OwnedArray.cs
  21. +25
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Runtime/BufferDebuggerView.cs
  22. +131
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Runtime/BufferPrimitivesThrowHelper.cs
  23. +122
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Runtime/Contract.cs
  24. +23
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Runtime/HashingHelper.cs
  25. +11
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Runtime/PrimitiveAttribute.cs
  26. +25
    -0
      src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Runtime/ReadOnlyBufferDebuggerView.cs
  27. +41
    -0
      src/Discord.Net.Core/_corefxlab/System.Collections.Sequences/System/Collections/Sequences/ArrayList.cs
  28. +18
    -0
      src/Discord.Net.Core/_corefxlab/System.Collections.Sequences/System/Collections/Sequences/ISequence.cs
  29. +47
    -0
      src/Discord.Net.Core/_corefxlab/System.Collections.Sequences/System/Collections/Sequences/Position.cs
  30. +124
    -0
      src/Discord.Net.Core/_corefxlab/System.Collections.Sequences/System/Collections/Sequences/ResizableArray.cs
  31. +31
    -0
      src/Discord.Net.Core/_corefxlab/System.Collections.Sequences/System/Collections/Sequences/SequenceEnumerator.cs
  32. +402
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/CompositeFormat.cs
  33. +55
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/Formatters/ArrayFormatter.cs
  34. +32
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/Formatters/OutputFormatter.cs
  35. +105
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/Formatters/SequenceFormatter.cs
  36. +83
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/Formatters/StreamFormatter.cs
  37. +79
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/Formatters/StringFormatter.cs
  38. +346
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/IOutputExtensions.cs
  39. +15
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/ITextOutput.cs
  40. +319
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/ITextOutputExtensions.cs
  41. +159
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Parsing/SequenceParser.cs
  42. +34
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Precondition.cs
  43. +72
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/Ascii/Ascii_casing.cs
  44. +128
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/Ascii/Ascii_encoding.cs
  45. +529
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/Encoders.Utf16.cs
  46. +258
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/Encoders.Utf32.cs
  47. +838
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/Encoders.Utf8.cs
  48. +151
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/EncodingHelper.cs
  49. +343
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/ParsingTrie.cs
  50. +29
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/SymbolTable.Symbol.cs
  51. +98
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/SymbolTable.Utf16.cs
  52. +100
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/SymbolTable.Utf8.cs
  53. +242
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/SymbolTable.cs
  54. +91
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/FloatFormatter.cs
  55. +23
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/IBufferFormattable.cs
  56. +208
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/FormattingHelpers.cs
  57. +92
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/GuidFormatter.Utf16.cs
  58. +91
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/GuidFormatter.Utf8.cs
  59. +262
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/IntegerFormatter.Utf16.cs
  60. +250
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/IntegerFormatter.Utf8.cs
  61. +497
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/IntegerFormatter.cs
  62. +402
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/TimeFormatter.Utf16.cs
  63. +394
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/TimeFormatter.Utf8.cs
  64. +20
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/PrimitiveFormatter.Guid.cs
  65. +187
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/PrimitiveFormatter.Integer.cs
  66. +160
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/PrimitiveFormatter.Time.cs
  67. +32
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/PrimitiveFormatter_float.cs
  68. +62
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/ParsedFormat.cs
  69. +268
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/InvariantBool.cs
  70. +2395
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/InvariantSigned.cs
  71. +2594
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/InvariantSignedHex.cs
  72. +2332
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/InvariantUnsigned.cs
  73. +2594
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/InvariantUnsignedHex.cs
  74. +111
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/InvariantUtf16_decimal.cs
  75. +111
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/InvariantUtf8_decimal.cs
  76. +54
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/PrimitiveParser.cs
  77. +32
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/PrimitiveParser_Boolean.cs
  78. +32
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/PrimitiveParser_Decimal.cs
  79. +528
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/Signed.cs
  80. +385
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/Unsigned.cs
  81. +35
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf16LittleEndianCodePointEnumerable.cs
  82. +86
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf16LittleEndianCodePointEnumerator.cs
  83. +232
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf8Helper.cs
  84. +45
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf8String.CodePointEnumerable.cs
  85. +120
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf8String.CodePointEnumerator.cs
  86. +122
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf8String.CodePointReverseEnumerator.cs
  87. +41
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf8String.Enumerator.cs
  88. +641
    -0
      src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf8String.cs
  89. +13
    -7
      src/Discord.Net.Providers.WS4Net/WS4NetClient.cs
  90. +8
    -8
      src/Discord.Net.Rest/API/Common/Application.cs
  91. +8
    -8
      src/Discord.Net.Rest/API/Common/Attachment.cs
  92. +3
    -3
      src/Discord.Net.Rest/API/Common/Ban.cs
  93. +14
    -14
      src/Discord.Net.Rest/API/Common/Channel.cs
  94. +6
    -6
      src/Discord.Net.Rest/API/Common/Connection.cs
  95. +14
    -15
      src/Discord.Net.Rest/API/Common/Embed.cs
  96. +5
    -5
      src/Discord.Net.Rest/API/Common/EmbedAuthor.cs
  97. +4
    -4
      src/Discord.Net.Rest/API/Common/EmbedField.cs
  98. +4
    -4
      src/Discord.Net.Rest/API/Common/EmbedFooter.cs
  99. +5
    -5
      src/Discord.Net.Rest/API/Common/EmbedImage.cs
  100. +3
    -3
      src/Discord.Net.Rest/API/Common/EmbedProvider.cs

+ 2
- 1
Discord.Net.targets View File

@@ -2,6 +2,7 @@
<PropertyGroup>
<VersionPrefix>1.1.0-alpha</VersionPrefix>
<VersionSuffix></VersionSuffix>
<LangVersion>latest</LangVersion>
<Authors>RogueException</Authors>
<PackageTags>discord;discordapp</PackageTags>
<PackageProjectUrl>https://github.com/RogueException/Discord.Net</PackageProjectUrl>
@@ -18,7 +19,7 @@
<VersionSuffix Condition=" '$(VersionSuffix)' == '' ">build-$(BuildNumber)</VersionSuffix>
</PropertyGroup>
<PropertyGroup Condition=" '$(TargetFramework)' == 'netstandard1.3' Or '$(TargetFramework)' == 'net45' ">
<DefineConstants>$(DefineConstants);FILESYSTEM;DEFAULTUDPCLIENT;DEFAULTWEBSOCKET</DefineConstants>
<DefineConstants>$(DefineConstants);FILESYSTEM;DEFAULTUDPCLIENT;DEFAULTWEBSOCKET;MSBUFFER</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition=" '$(TargetFramework)' == 'netstandard1.3' ">
<DefineConstants>$(DefineConstants);FORMATSTR;UNIXTIME;MSTRYBUFFER;UDPDISPOSE</DefineConstants>


+ 7
- 1
src/Discord.Net.Core/Discord.Net.Core.csproj View File

@@ -5,10 +5,16 @@
<RootNamespace>Discord</RootNamespace>
<Description>The core components for the Discord.Net library.</Description>
<TargetFrameworks>net45;netstandard1.1;netstandard1.3</TargetFrameworks>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="10.0.2" />
<PackageReference Include="System.Buffers" Version="4.4.0-preview2-25405-01" />
<PackageReference Include="System.Collections.Immutable" Version="1.3.1" />
<PackageReference Include="System.Interactive.Async" Version="3.1.1" />
<PackageReference Include="System.Memory" Version="4.4.0-preview2-25405-01" />
<PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="4.4.0-preview2-25405-01" />
</ItemGroup>
<ItemGroup>
<Folder Include="Serialization\" />
</ItemGroup>
</Project>

+ 2
- 1
src/Discord.Net.Core/Net/Rest/IRestClient.cs View File

@@ -1,3 +1,4 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
@@ -10,7 +11,7 @@ namespace Discord.Net.Rest
void SetCancelToken(CancellationToken cancelToken);

Task<RestResponse> SendAsync(string method, string endpoint, CancellationToken cancelToken, bool headerOnly = false, string reason = null);
Task<RestResponse> SendAsync(string method, string endpoint, string json, CancellationToken cancelToken, bool headerOnly = false, string reason = null);
Task<RestResponse> SendAsync(string method, string endpoint, ReadOnlyBuffer<byte> jsonPayload, CancellationToken cancelToken, bool headerOnly = false, string reason = null);
Task<RestResponse> SendAsync(string method, string endpoint, IReadOnlyDictionary<string, object> multipartParams, CancellationToken cancelToken, bool headerOnly = false, string reason = null);
}
}

+ 5
- 5
src/Discord.Net.Core/Net/Rest/RestResponse.cs View File

@@ -1,5 +1,5 @@
using System.Collections.Generic;
using System.IO;
using System;
using System.Collections.Generic;
using System.Net;

namespace Discord.Net.Rest
@@ -8,13 +8,13 @@ namespace Discord.Net.Rest
{
public HttpStatusCode StatusCode { get; }
public Dictionary<string, string> Headers { get; }
public Stream Stream { get; }
public ReadOnlyBuffer<byte> Data { get; }

public RestResponse(HttpStatusCode statusCode, Dictionary<string, string> headers, Stream stream)
public RestResponse(HttpStatusCode statusCode, Dictionary<string, string> headers, ReadOnlyBuffer<byte> data)
{
StatusCode = statusCode;
Headers = headers;
Stream = stream;
Data = data;
}
}
}

+ 2
- 3
src/Discord.Net.Core/Net/WebSockets/IWebSocketClient.cs View File

@@ -6,8 +6,7 @@ namespace Discord.Net.WebSockets
{
public interface IWebSocketClient
{
event Func<byte[], int, int, Task> BinaryMessage;
event Func<string, Task> TextMessage;
event Func<ReadOnlyBuffer<byte>, bool, Task> Message;
event Func<Exception, Task> Closed;

void SetHeader(string key, string value);
@@ -16,6 +15,6 @@ namespace Discord.Net.WebSockets
Task ConnectAsync(string host);
Task DisconnectAsync();

Task SendAsync(byte[] data, int index, int count, bool isText);
Task SendAsync(ReadOnlyBuffer<byte> data, bool isText);
}
}

+ 160
- 0
src/Discord.Net.Core/_corefxlab/BufferExtensions.cs View File

@@ -0,0 +1,160 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Collections.Sequences;
using System.Diagnostics;
using System.Runtime.CompilerServices;

namespace System.Buffers
{
public static class ExperimentalBufferExtensions
{
public static ReadOnlySpan<byte> ToSpan<T>(this T bufferSequence) where T : ISequence<ReadOnlyBuffer<byte>>
{
Position position = Position.First;
ReadOnlyBuffer<byte> buffer;
ResizableArray<byte> array = new ResizableArray<byte>(1024);
while (bufferSequence.TryGet(ref position, out buffer))
{
array.AddAll(buffer.Span);
}
array.Resize(array.Count);
return array.Items.Slice(0, array.Count);
}

/// <summary>
/// Creates a new slice over the portion of the target array segment.
/// </summary>
/// <param name="arraySegment">The target array segment.</param>
/// </exception>
public static Span<T> Slice<T>(this ArraySegment<T> arraySegment)
{
return new Span<T>(arraySegment.Array, arraySegment.Offset, arraySegment.Count);
}

/// <summary>
/// Creates a new slice over the portion of the target array.
/// </summary>
/// <param name="array">The target array.</param>
/// <exception cref="System.ArgumentException">
/// Thrown if the 'array' parameter is null.
/// </exception>
public static Span<T> Slice<T>(this T[] array)
{
return new Span<T>(array);
}

/// <summary>
/// Creates a new slice over the portion of the target array beginning
/// at 'start' index.
/// </summary>
/// <param name="array">The target array.</param>
/// <param name="start">The index at which to begin the slice.</param>
/// <exception cref="System.ArgumentException">
/// Thrown if the 'array' parameter is null.
/// </exception>
/// <exception cref="System.ArgumentOutOfRangeException">
/// Thrown when the specified start index is not in range (&lt;0 or &gt;&eq;length).
/// </exception>
public static Span<T> Slice<T>(this T[] array, int start)
{
return new Span<T>(array, start);
}

/// <summary>
/// Creates a new slice over the portion of the target array beginning
/// at 'start' index and with 'length' items.
/// </summary>
/// <param name="array">The target array.</param>
/// <param name="start">The index at which to begin the slice.</param>
/// <param name="length">The number of items in the new slice.</param>
/// <exception cref="System.ArgumentException">
/// Thrown if the 'array' parameter is null.
/// </exception>
/// <exception cref="System.ArgumentOutOfRangeException">
/// Thrown when the specified start or end index is not in range (&lt;0 or &gt;&eq;length).
/// </exception>
public static Span<T> Slice<T>(this T[] array, int start, int length)
{
return new Span<T>(array, start, length);
}

/// <summary>
/// Creates a new readonly span over the portion of the target string.
/// </summary>
/// <param name="text">The target string.</param>
/// <exception cref="System.ArgumentNullException">Thrown when <paramref name="text"/> is null.</exception>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe ReadOnlySpan<char> Slice(this string text)
{
if (text == null)
throw new ArgumentNullException(nameof(text));

int textLength = text.Length;

if (textLength == 0) return ReadOnlySpan<char>.Empty;

fixed (char* charPointer = text)
{
return ReadOnlySpan<char>.DangerousCreate(text, ref *charPointer, textLength);
}
}

/// <summary>
/// Creates a new readonly span over the portion of the target string, beginning at 'start'.
/// </summary>
/// <param name="text">The target string.</param>
/// <param name="start">The index at which to begin this slice.</param>
/// <exception cref="System.ArgumentNullException">Thrown when <paramref name="text"/> is null.</exception>
/// <exception cref="System.ArgumentOutOfRangeException">
/// Thrown when the specified <paramref name="start"/> index is not in range (&lt;0 or &gt;Length).
/// </exception>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe ReadOnlySpan<char> Slice(this string text, int start)
{
if (text == null)
throw new ArgumentNullException(nameof(text));

int textLength = text.Length;

if ((uint) start > (uint) textLength)
throw new ArgumentOutOfRangeException(nameof(start));

if (textLength - start == 0) return ReadOnlySpan<char>.Empty;

fixed (char* charPointer = text)
{
return ReadOnlySpan<char>.DangerousCreate(text, ref *(charPointer + start), textLength - start);
}
}

/// <summary>
/// Creates a new readonly span over the portion of the target string, beginning at <paramref name="start"/>, of given <paramref name="length"/>.
/// </summary>
/// <param name="text">The target string.</param>
/// <param name="start">The index at which to begin this slice.</param>
/// <param name="length">The number of items in the span.</param>
/// <exception cref="System.ArgumentNullException">Thrown when <paramref name="text"/> is null.</exception>
/// <exception cref="System.ArgumentOutOfRangeException">
/// Thrown when the specified <paramref name="start"/> or end index is not in range (&lt;0 or &gt;=Length).
/// </exception>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe ReadOnlySpan<char> Slice(this string text, int start, int length)
{
if (text == null)
throw new ArgumentNullException(nameof(text));

int textLength = text.Length;

if ((uint)start > (uint)textLength || (uint)length > (uint)(textLength - start))
throw new ArgumentOutOfRangeException(nameof(start));

if (length == 0) return ReadOnlySpan<char>.Empty;

fixed (char* charPointer = text)
{
return ReadOnlySpan<char>.DangerousCreate(text, ref *(charPointer + start), length);
}
}
}
}

+ 109
- 0
src/Discord.Net.Core/_corefxlab/System.Binary/System/Binary/BufferReader.cs View File

@@ -0,0 +1,109 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime;
using System.Runtime.CompilerServices;

namespace System.Binary
{
/// <summary>
/// Reads bytes as primitives with specific endianness
/// </summary>
/// <remarks>
/// For native formats, SpanExtensions.Read<T> should be used.
/// Use these helpers when you need to read specific endinanness.
/// </remarks>
public static class BufferReader
{
/// <summary>
/// Reads a structure of type <typeparamref name="T"/> out of a span of bytes.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static T ReadBigEndian<[Primitive]T>(this ReadOnlySpan<byte> span) where T : struct
=> BitConverter.IsLittleEndian ? UnsafeUtilities.Reverse(span.Read<T>()) : span.Read<T>();

/// <summary>
/// Reads a structure of type <typeparamref name="T"/> out of a span of bytes.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static T ReadLittleEndian<[Primitive]T>(this ReadOnlySpan<byte> span) where T : struct
=> BitConverter.IsLittleEndian ? span.Read<T>() : UnsafeUtilities.Reverse(span.Read<T>());

/// <summary>
/// Reads a structure of type <typeparamref name="T"/> out of a span of bytes.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static T ReadBigEndian<[Primitive]T>(this Span<byte> span) where T : struct
=> BitConverter.IsLittleEndian ? UnsafeUtilities.Reverse(span.Read<T>()) : span.Read<T>();

/// <summary>
/// Reads a structure of type <typeparamref name="T"/> out of a span of bytes.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static T ReadLittleEndian<[Primitive]T>(this Span<byte> span) where T : struct
=> BitConverter.IsLittleEndian ? span.Read<T>() : UnsafeUtilities.Reverse(span.Read<T>());

/// <summary>
/// Reads a structure of type T out of a slice of bytes.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static T Read<[Primitive]T>(this Span<byte> slice)
where T : struct
{
RequiresInInclusiveRange(Unsafe.SizeOf<T>(), (uint)slice.Length);
return Unsafe.ReadUnaligned<T>(ref slice.DangerousGetPinnableReference());
}

/// <summary>
/// Reads a structure of type T out of a slice of bytes.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static T Read<[Primitive]T>(this ReadOnlySpan<byte> slice)
where T : struct
{
RequiresInInclusiveRange(Unsafe.SizeOf<T>(), (uint)slice.Length);
return Unsafe.ReadUnaligned<T>(ref slice.DangerousGetPinnableReference());
}

/// <summary>
/// Reads a structure of type T out of a slice of bytes.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool TryRead<[Primitive]T>(this ReadOnlySpan<byte> slice, out T value)
where T : struct
{
if (Unsafe.SizeOf<T>() > (uint)slice.Length)
{
value = default;
return false;
}
value = Unsafe.ReadUnaligned<T>(ref slice.DangerousGetPinnableReference());
return true;
}

/// <summary>
/// Reads a structure of type T out of a slice of bytes.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool TryRead<[Primitive]T>(this Span<byte> slice, out T value)
where T : struct
{
if (Unsafe.SizeOf<T>() > (uint)slice.Length)
{
value = default;
return false;
}
value = Unsafe.ReadUnaligned<T>(ref slice.DangerousGetPinnableReference());
return true;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void RequiresInInclusiveRange(int start, uint length)
{
if ((uint)start > length)
{
throw new ArgumentOutOfRangeException();
}
}
}
}

+ 62
- 0
src/Discord.Net.Core/_corefxlab/System.Binary/System/Binary/BufferWriter.cs View File

@@ -0,0 +1,62 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime;
using System.Runtime.CompilerServices;

namespace System.Binary
{
/// <summary>
/// Writes endian-specific primitives into spans.
/// </summary>
/// <remarks>
/// Use these helpers when you need to write specific endinaness.
/// </remarks>
public static class BufferWriter
{
/// <summary>
/// Writes a structure of type T to a span of bytes.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void WriteBigEndian<[Primitive]T>(this Span<byte> span, T value) where T : struct
=> span.Write(BitConverter.IsLittleEndian ? UnsafeUtilities.Reverse(value) : value);

/// <summary>
/// Writes a structure of type T to a span of bytes.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void WriteLittleEndian<[Primitive]T>(this Span<byte> span, T value) where T : struct
=> span.Write(BitConverter.IsLittleEndian ? value : UnsafeUtilities.Reverse(value));



/// <summary>
/// Writes a structure of type T into a slice of bytes.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Write<[Primitive]T>(this Span<byte> slice, T value)
where T : struct
{
if ((uint)Unsafe.SizeOf<T>() > (uint)slice.Length)
{
throw new ArgumentOutOfRangeException();
}
Unsafe.WriteUnaligned<T>(ref slice.DangerousGetPinnableReference(), value);
}

/// <summary>
/// Writes a structure of type T into a slice of bytes.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool TryWrite<[Primitive]T>(this Span<byte> slice, T value)
where T : struct
{
if (Unsafe.SizeOf<T>() > (uint)slice.Length)
{
return false;
}
Unsafe.WriteUnaligned<T>(ref slice.DangerousGetPinnableReference(), value);
return true;
}
}
}

+ 68
- 0
src/Discord.Net.Core/_corefxlab/System.Binary/System/Binary/UnsafeUtilities.cs View File

@@ -0,0 +1,68 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Runtime
{
/// <summary>
/// A collection of unsafe helper methods that we cannot implement in C#.
/// NOTE: these can be used for VeryBadThings(tm), so tread with care...
/// </summary>
internal static class UnsafeUtilities
{
/// <summary>
/// Reverses a primitive value - performs an endianness swap
/// </summary>
public static unsafe T Reverse<[Primitive]T>(T value) where T : struct
{
// note: relying on JIT goodness here!
if (typeof(T) == typeof(byte) || typeof(T) == typeof(sbyte)) {
return value;
}
else if (typeof(T) == typeof(ushort) || typeof(T) == typeof(short)) {
ushort val = 0;
Unsafe.Write(&val, value);
val = (ushort)((val >> 8) | (val << 8));
return Unsafe.Read<T>(&val);
}
else if (typeof(T) == typeof(uint) || typeof(T) == typeof(int)
|| typeof(T) == typeof(float)) {
uint val = 0;
Unsafe.Write(&val, value);
val = (val << 24)
| ((val & 0xFF00) << 8)
| ((val & 0xFF0000) >> 8)
| (val >> 24);
return Unsafe.Read<T>(&val);
}
else if (typeof(T) == typeof(ulong) || typeof(T) == typeof(long)
|| typeof(T) == typeof(double)) {
ulong val = 0;
Unsafe.Write(&val, value);
val = (val << 56)
| ((val & 0xFF00) << 40)
| ((val & 0xFF0000) << 24)
| ((val & 0xFF000000) << 8)
| ((val & 0xFF00000000) >> 8)
| ((val & 0xFF0000000000) >> 24)
| ((val & 0xFF000000000000) >> 40)
| (val >> 56);
return Unsafe.Read<T>(&val);
}
else {
// default implementation
int len = Unsafe.SizeOf<T>();
var val = stackalloc byte[len];
Unsafe.Write(val, value);
int to = len >> 1, dest = len - 1;
for (int i = 0; i < to; i++) {
var tmp = val[i];
val[i] = val[dest];
val[dest--] = tmp;
}
return Unsafe.Read<T>(val);
}
}
}
}

+ 238
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/Buffer.cs View File

@@ -0,0 +1,238 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Buffers;
using System.ComponentModel;
using System.Diagnostics;
using System.Runtime;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;

namespace System
{
[DebuggerTypeProxy(typeof(BufferDebuggerView<>))]
public struct Buffer<T>
{
// The highest order bit of _index is used to discern whether _arrayOrOwnedBuffer is an array or an owned buffer
// if (_index >> 31) == 1, object _arrayOrOwnedBuffer is an OwnedBuffer<T>
// else, object _arrayOrOwnedBuffer is a T[]
readonly object _arrayOrOwnedBuffer;
readonly int _index;
readonly int _length;

private const int bitMask = 0x7FFFFFFF;

internal Buffer(OwnedBuffer<T> owner, int index, int length)
{
_arrayOrOwnedBuffer = owner;
_index = index | (1 << 31); // Before using _index, check if _index < 0, then 'and' it with bitMask
_length = length;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Buffer(T[] array)
{
if (array == null)
BufferPrimitivesThrowHelper.ThrowArgumentNullException(ExceptionArgument.array);
if (default(T) == null && array.GetType() != typeof(T[]))
BufferPrimitivesThrowHelper.ThrowArrayTypeMismatchException(typeof(T));

_arrayOrOwnedBuffer = array;
_index = 0;
_length = array.Length;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Buffer(T[] array, int start)
{
if (array == null)
BufferPrimitivesThrowHelper.ThrowArgumentNullException(ExceptionArgument.array);
if (default(T) == null && array.GetType() != typeof(T[]))
BufferPrimitivesThrowHelper.ThrowArrayTypeMismatchException(typeof(T));

int arrayLength = array.Length;
if ((uint)start > (uint)arrayLength)
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);

_arrayOrOwnedBuffer = array;
_index = start;
_length = arrayLength - start;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Buffer(T[] array, int start, int length)
{
if (array == null)
BufferPrimitivesThrowHelper.ThrowArgumentNullException(ExceptionArgument.array);
if (default(T) == null && array.GetType() != typeof(T[]))
BufferPrimitivesThrowHelper.ThrowArrayTypeMismatchException(typeof(T));
if ((uint)start > (uint)array.Length || (uint)length > (uint)(array.Length - start))
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);

_arrayOrOwnedBuffer = array;
_index = start;
_length = length;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static implicit operator ReadOnlyBuffer<T>(Buffer<T> buffer)
{
// There is no need to 'and' _index by the bit mask here
// since the constructor will set the highest order bit again anyway
if (buffer._index < 0)
return new ReadOnlyBuffer<T>(Unsafe.As<OwnedBuffer<T>>(buffer._arrayOrOwnedBuffer), buffer._index, buffer._length);
return new ReadOnlyBuffer<T>(Unsafe.As<T[]>(buffer._arrayOrOwnedBuffer), buffer._index, buffer._length);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static implicit operator Buffer<T>(T[] array)
{
return new Buffer<T>(array, 0, array.Length);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static implicit operator Buffer<T>(ArraySegment<T> arraySegment)
{
return new Buffer<T>(arraySegment.Array, arraySegment.Offset, arraySegment.Count);
}

public static Buffer<T> Empty { get; } = OwnedBuffer<T>.EmptyArray;

public int Length => _length;

public bool IsEmpty => Length == 0;

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Buffer<T> Slice(int start)
{
if ((uint)start > (uint)_length)
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);

// There is no need to and _index by the bit mask here
// since the constructor will set the highest order bit again anyway
if (_index < 0)
return new Buffer<T>(Unsafe.As<OwnedBuffer<T>>(_arrayOrOwnedBuffer), _index + start, _length - start);
return new Buffer<T>(Unsafe.As<T[]>(_arrayOrOwnedBuffer), _index + start, _length - start);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Buffer<T> Slice(int start, int length)
{
if ((uint)start > (uint)_length || (uint)length > (uint)(_length - start))
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);

// There is no need to 'and' _index by the bit mask here
// since the constructor will set the highest order bit again anyway
if (_index < 0)
return new Buffer<T>(Unsafe.As<OwnedBuffer<T>>(_arrayOrOwnedBuffer), _index + start, length);
return new Buffer<T>(Unsafe.As<T[]>(_arrayOrOwnedBuffer), _index + start, length);
}

public Span<T> Span
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
{
if (_index < 0)
return Unsafe.As<OwnedBuffer<T>>(_arrayOrOwnedBuffer).AsSpan(_index & bitMask, _length);
return new Span<T>(Unsafe.As<T[]>(_arrayOrOwnedBuffer), _index, _length);
}
}

public BufferHandle Retain(bool pin = false)
{
BufferHandle bufferHandle;
if (pin)
{
if (_index < 0)
{
bufferHandle = Unsafe.As<OwnedBuffer<T>>(_arrayOrOwnedBuffer).Pin(_index & bitMask);
}
else
{
var handle = GCHandle.Alloc(Unsafe.As<T[]>(_arrayOrOwnedBuffer), GCHandleType.Pinned);
unsafe
{
var pointer = OwnedBuffer<T>.Add((void*)handle.AddrOfPinnedObject(), _index);
bufferHandle = new BufferHandle(null, pointer, handle);
}
}
}
else
{
if (_index < 0)
{
Unsafe.As<OwnedBuffer<T>>(_arrayOrOwnedBuffer).Retain();
bufferHandle = new BufferHandle(Unsafe.As<OwnedBuffer<T>>(_arrayOrOwnedBuffer));
}
else
{
bufferHandle = new BufferHandle(null);
}
}
return bufferHandle;
}

public bool TryGetArray(out ArraySegment<T> arraySegment)
{
if (_index < 0)
{
if (Unsafe.As<OwnedBuffer<T>>(_arrayOrOwnedBuffer).TryGetArray(out var segment))
{
arraySegment = new ArraySegment<T>(segment.Array, segment.Offset + (_index & bitMask), _length);
return true;
}
}
else
{
arraySegment = new ArraySegment<T>(Unsafe.As<T[]>(_arrayOrOwnedBuffer), _index, _length);
return true;
}

arraySegment = default;
return false;
}

public T[] ToArray() => Span.ToArray();

public void CopyTo(Span<T> span) => Span.CopyTo(span);

public void CopyTo(Buffer<T> buffer) => Span.CopyTo(buffer.Span);

public bool TryCopyTo(Span<T> span) => Span.TryCopyTo(span);

public bool TryCopyTo(Buffer<T> buffer) => Span.TryCopyTo(buffer.Span);

[EditorBrowsable(EditorBrowsableState.Never)]
public override bool Equals(object obj)
{
if (obj is ReadOnlyBuffer<T> readOnlyBuffer)
{
return readOnlyBuffer.Equals(this);
}
else if (obj is Buffer<T> buffer)
{
return Equals(buffer);
}
else
{
return false;
}
}

public bool Equals(Buffer<T> other)
{
return
_arrayOrOwnedBuffer == other._arrayOrOwnedBuffer &&
(_index & bitMask) == (other._index & bitMask) &&
_length == other._length;
}

[EditorBrowsable( EditorBrowsableState.Never)]
public override int GetHashCode()
{
return HashingHelper.CombineHashCodes(_arrayOrOwnedBuffer.GetHashCode(), (_index & bitMask).GetHashCode(), _length.GetHashCode());
}
}
}

+ 19
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/BufferExtensions.cs View File

@@ -0,0 +1,19 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

namespace System.Buffers
{
public static class BufferExtensions
{
public static bool SequenceEqual<T>(this Buffer<T> first, Buffer<T> second) where T : struct, IEquatable<T>
{
return first.Span.SequenceEqual(second.Span);
}

public static bool SequenceEqual<T>(this ReadOnlyBuffer<T> first, ReadOnlyBuffer<T> second) where T : struct, IEquatable<T>
{
return first.Span.SequenceEqual(second.Span);
}
}
}

+ 46
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/BufferHandle.cs View File

@@ -0,0 +1,46 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Runtime;
using System.Runtime.InteropServices;

namespace System.Buffers
{
public unsafe struct BufferHandle : IDisposable
{
IRetainable _owner;
void* _pointer;
GCHandle _handle;

public BufferHandle(IRetainable owner, void* pinnedPointer, GCHandle handle = default)
{
_pointer = pinnedPointer;
_handle = handle;
_owner = owner;
}

public BufferHandle(IRetainable owner) : this(owner, null) { }

public void* PinnedPointer {
get {
if (_pointer == null) BufferPrimitivesThrowHelper.ThrowInvalidOperationException();
return _pointer;
}
}

public void Dispose()
{
if (_handle.IsAllocated) {
_handle.Free();
}

if (_owner != null) {
_owner.Release();
_owner = null;
}

_pointer = null;
}
}
}

+ 25
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/BufferPool.cs View File

@@ -0,0 +1,25 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace System.Buffers
{
public abstract class BufferPool : IDisposable
{
public static BufferPool Default => Internal.ManagedBufferPool.Shared;

public abstract OwnedBuffer<byte> Rent(int minimumBufferSize);

public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}

~BufferPool()
{
Dispose(false);
}

protected abstract void Dispose(bool disposing);
}
}

+ 14
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/IOutput.cs View File

@@ -0,0 +1,14 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace System.Buffers
{
public interface IOutput
{
Span<byte> Buffer { get; }
void Advance(int bytes);

/// <summary>desiredBufferLength == 0 means "i don't care"</summary>
void Enlarge(int desiredBufferLength = 0);
}
}

+ 12
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/IRetainable.cs View File

@@ -0,0 +1,12 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace System.Buffers
{
public interface IRetainable
{
void Retain();
void Release();
bool IsRetained { get; }
}
}

+ 75
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/OwnedBuffer.cs View File

@@ -0,0 +1,75 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Runtime;
using System.Runtime.CompilerServices;

namespace System.Buffers
{
public abstract class OwnedBuffer<T> : IDisposable, IRetainable
{
protected OwnedBuffer() { }

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static implicit operator OwnedBuffer<T>(T[] array)
{
return new Internal.OwnedArray<T>(array);
}

public abstract int Length { get; }

public abstract Span<T> AsSpan(int index, int length);

public virtual Span<T> AsSpan()
{
if (IsDisposed) BufferPrimitivesThrowHelper.ThrowObjectDisposedException(nameof(OwnedBuffer<T>));
return AsSpan(0, Length);
}

public Buffer<T> Buffer
{
get {
if (IsDisposed) BufferPrimitivesThrowHelper.ThrowObjectDisposedException(nameof(OwnedBuffer<T>));
return new Buffer<T>(this, 0, Length);
}
}

public ReadOnlyBuffer<T> ReadOnlyBuffer
{
get {
if (IsDisposed) BufferPrimitivesThrowHelper.ThrowObjectDisposedException(nameof(OwnedBuffer<T>));
return new ReadOnlyBuffer<T>(this, 0, Length);
}
}

public abstract BufferHandle Pin(int index = 0);

protected internal abstract bool TryGetArray(out ArraySegment<T> arraySegment);

#region Lifetime Management
public abstract bool IsDisposed { get; }

public void Dispose()
{
if (IsRetained) throw new InvalidOperationException("outstanding references detected.");
Dispose(true);
}

protected abstract void Dispose(bool disposing);

public abstract bool IsRetained { get; }

public abstract void Retain();

public abstract void Release();
#endregion

protected internal static unsafe void* Add(void* pointer, int offset)
{
return (byte*)pointer + ((ulong)Unsafe.SizeOf<T>() * (ulong)offset);
}

internal static readonly T[] EmptyArray = new T[0];
}
}

+ 223
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/ReadOnlyBuffer.cs View File

@@ -0,0 +1,223 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Buffers;
using System.ComponentModel;
using System.Diagnostics;
using System.Runtime;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;

namespace System
{
[DebuggerTypeProxy(typeof(ReadOnlyBufferDebuggerView<>))]
public struct ReadOnlyBuffer<T>
{
// The highest order bit of _index is used to discern whether _arrayOrOwnedBuffer is an array or an owned buffer
// if (_index >> 31) == 1, object _arrayOrOwnedBuffer is an OwnedBuffer<T>
// else, object _arrayOrOwnedBuffer is a T[]
readonly object _arrayOrOwnedBuffer;
readonly int _index;
readonly int _length;

private const int bitMask = 0x7FFFFFFF;

internal ReadOnlyBuffer(OwnedBuffer<T> owner, int index, int length)
{
_arrayOrOwnedBuffer = owner;
_index = index | (1 << 31); // Before using _index, check if _index < 0, then 'and' it with bitMask
_length = length;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ReadOnlyBuffer(T[] array)
{
if (array == null)
BufferPrimitivesThrowHelper.ThrowArgumentNullException(ExceptionArgument.array);

_arrayOrOwnedBuffer = array;
_index = 0;
_length = array.Length;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ReadOnlyBuffer(T[] array, int start)
{
if (array == null)
BufferPrimitivesThrowHelper.ThrowArgumentNullException(ExceptionArgument.array);

int arrayLength = array.Length;
if ((uint)start > (uint)arrayLength)
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);

_arrayOrOwnedBuffer = array;
_index = start;
_length = arrayLength - start;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ReadOnlyBuffer(T[] array, int start, int length)
{
if (array == null)
BufferPrimitivesThrowHelper.ThrowArgumentNullException(ExceptionArgument.array);
if ((uint)start > (uint)array.Length || (uint)length > (uint)(array.Length - start))
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);

_arrayOrOwnedBuffer = array;
_index = start;
_length = length;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static implicit operator ReadOnlyBuffer<T>(T[] array)
{
return new ReadOnlyBuffer<T>(array, 0, array.Length);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static implicit operator ReadOnlyBuffer<T>(ArraySegment<T> arraySegment)
{
return new ReadOnlyBuffer<T>(arraySegment.Array, arraySegment.Offset, arraySegment.Count);
}

public static ReadOnlyBuffer<T> Empty { get; } = OwnedBuffer<T>.EmptyArray;

public int Length => _length;

public bool IsEmpty => Length == 0;

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ReadOnlyBuffer<T> Slice(int start)
{
if ((uint)start > (uint)_length)
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);

// There is no need to 'and' _index by the bit mask here
// since the constructor will set the highest order bit again anyway
if (_index < 0)
return new ReadOnlyBuffer<T>(Unsafe.As<OwnedBuffer<T>>(_arrayOrOwnedBuffer), _index + start, _length - start);
return new ReadOnlyBuffer<T>(Unsafe.As<T[]>(_arrayOrOwnedBuffer), _index + start, _length - start);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ReadOnlyBuffer<T> Slice(int start, int length)
{
if ((uint)start > (uint)_length || (uint)length > (uint)(_length - start))
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);

// There is no need to 'and' _index by the bit mask here
// since the constructor will set the highest order bit again anyway
if (_index < 0)
return new ReadOnlyBuffer<T>(Unsafe.As<OwnedBuffer<T>>(_arrayOrOwnedBuffer), _index + start, length);
return new ReadOnlyBuffer<T>(Unsafe.As<T[]>(_arrayOrOwnedBuffer), _index + start, length);
}

public ReadOnlySpan<T> Span
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
{
if (_index < 0)
return Unsafe.As<OwnedBuffer<T>>(_arrayOrOwnedBuffer).AsSpan(_index & bitMask, _length);
return new ReadOnlySpan<T>(Unsafe.As<T[]>(_arrayOrOwnedBuffer), _index, _length);
}
}

public BufferHandle Retain(bool pin = false)
{
BufferHandle bufferHandle;
if (pin)
{
if (_index < 0)
{
bufferHandle = Unsafe.As<OwnedBuffer<T>>(_arrayOrOwnedBuffer).Pin(_index & bitMask);
}
else
{
var handle = GCHandle.Alloc(Unsafe.As<T[]>(_arrayOrOwnedBuffer), GCHandleType.Pinned);
unsafe
{
var pointer = OwnedBuffer<T>.Add((void*)handle.AddrOfPinnedObject(), _index);
bufferHandle = new BufferHandle(null, pointer, handle);
}
}
}
else
{
if (_index < 0)
{
Unsafe.As<OwnedBuffer<T>>(_arrayOrOwnedBuffer).Retain();
bufferHandle = new BufferHandle(Unsafe.As<OwnedBuffer<T>>(_arrayOrOwnedBuffer));
}
else
{
bufferHandle = new BufferHandle(null);
}
}
return bufferHandle;
}

public T[] ToArray() => Span.ToArray();

[EditorBrowsable(EditorBrowsableState.Never)]
public bool DangerousTryGetArray(out ArraySegment<T> arraySegment)
{
if (_index < 0)
{
if (Unsafe.As<OwnedBuffer<T>>(_arrayOrOwnedBuffer).TryGetArray(out var segment))
{
arraySegment = new ArraySegment<T>(segment.Array, segment.Offset + (_index & bitMask), _length);
return true;
}
}
else
{
arraySegment = new ArraySegment<T>(Unsafe.As<T[]>(_arrayOrOwnedBuffer), _index, _length);
return true;
}

arraySegment = default;
return false;
}

public void CopyTo(Span<T> span) => Span.CopyTo(span);

public void CopyTo(Buffer<T> buffer) => Span.CopyTo(buffer.Span);

public bool TryCopyTo(Span<T> span) => Span.TryCopyTo(span);

public bool TryCopyTo(Buffer<T> buffer) => Span.TryCopyTo(buffer.Span);

[EditorBrowsable(EditorBrowsableState.Never)]
public override bool Equals(object obj)
{
if (obj is ReadOnlyBuffer<T> readOnlyBuffer)
{
return Equals(readOnlyBuffer);
}
else if (obj is Buffer<T> buffer)
{
return Equals(buffer);
}
else
{
return false;
}
}
public bool Equals(ReadOnlyBuffer<T> other)
{
return
_arrayOrOwnedBuffer == other._arrayOrOwnedBuffer &&
(_index & bitMask) == (other._index & bitMask) &&
_length == other._length;
}

[EditorBrowsable(EditorBrowsableState.Never)]
public override int GetHashCode()
{
return HashingHelper.CombineHashCodes(_arrayOrOwnedBuffer.GetHashCode(), (_index & bitMask).GetHashCode(), _length.GetHashCode());
}
}
}

+ 18
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Buffers/Transformation.cs View File

@@ -0,0 +1,18 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace System.Buffers
{
public abstract class Transformation
{
public abstract TransformationStatus Transform(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten);
}

public enum TransformationStatus
{
Done,
DestinationTooSmall,
NeedMoreSourceData,
InvalidData // TODO: how do we communicate details of the error
}
}

+ 101
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Internal/ManagedBufferPool.cs View File

@@ -0,0 +1,101 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime;
using System.Runtime.InteropServices;
using System.Threading;

namespace System.Buffers.Internal
{
internal sealed class ManagedBufferPool : BufferPool
{
readonly static ManagedBufferPool s_shared = new ManagedBufferPool();

public static ManagedBufferPool Shared
{
get
{
return s_shared;
}
}

public override OwnedBuffer<byte> Rent(int minimumBufferSize)
{
var buffer = new ArrayPoolBuffer(minimumBufferSize);
return buffer;
}

protected override void Dispose(bool disposing)
{
}

private sealed class ArrayPoolBuffer : OwnedBuffer<byte>
{
byte[] _array;
bool _disposed;
int _referenceCount;
public ArrayPoolBuffer(int size)
{
_array = ArrayPool<byte>.Shared.Rent(size);
}

public override int Length => _array.Length;

public override bool IsDisposed => _disposed;

public override bool IsRetained => _referenceCount > 0;

public override Span<byte> AsSpan(int index, int length)
{
if (IsDisposed) BufferPrimitivesThrowHelper.ThrowObjectDisposedException(nameof(ArrayPoolBuffer));
return new Span<byte>(_array, index, length);
}

protected override void Dispose(bool disposing)
{
var array = Interlocked.Exchange(ref _array, null);
if (array != null) {
_disposed = true;
ArrayPool<byte>.Shared.Return(array);
}
}

protected internal override bool TryGetArray(out ArraySegment<byte> arraySegment)
{
if (IsDisposed) BufferPrimitivesThrowHelper.ThrowObjectDisposedException(nameof(ManagedBufferPool));
arraySegment = new ArraySegment<byte>(_array);
return true;
}

public override BufferHandle Pin(int index = 0)
{
unsafe
{
Retain(); // this checks IsDisposed
var handle = GCHandle.Alloc(_array, GCHandleType.Pinned);
var pointer = Add((void*)handle.AddrOfPinnedObject(), index);
return new BufferHandle(this, pointer, handle);
}
}

public override void Retain()
{
if (IsDisposed) BufferPrimitivesThrowHelper.ThrowObjectDisposedException(nameof(ArrayPoolBuffer));
Interlocked.Increment(ref _referenceCount);
}

public override void Release()
{
var newRefCount = Interlocked.Decrement(ref _referenceCount);
if (newRefCount == 0) {
Dispose();
return;
}
if(newRefCount < 0) {
throw new InvalidOperationException();
}
}
}
}
}

+ 88
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Internal/OwnedArray.cs View File

@@ -0,0 +1,88 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime;
using System.Runtime.InteropServices;
using System.Threading;

namespace System.Buffers.Internal
{
internal class OwnedArray<T> : OwnedBuffer<T>
{
T[] _array;
int _referenceCount;

public OwnedArray(int length)
{
_array = new T[length];
}

public OwnedArray(T[] array)
{
if (array == null) BufferPrimitivesThrowHelper.ThrowArgumentNullException(nameof(array));
_array = array;
}

public static implicit operator OwnedArray<T>(T[] array) => new OwnedArray<T>(array);

public override int Length => _array.Length;

public override Span<T> AsSpan(int index, int length)
{
if (IsDisposed) BufferPrimitivesThrowHelper.ThrowObjectDisposedException(nameof(OwnedArray<T>));
return new Span<T>(_array, index, length);
}

public override Span<T> AsSpan()
{
if (IsDisposed) BufferPrimitivesThrowHelper.ThrowObjectDisposedException(nameof(OwnedArray<T>));
return new Span<T>(_array, 0, _array.Length);
}

public override BufferHandle Pin(int index = 0)
{
unsafe
{
Retain();
var handle = GCHandle.Alloc(_array, GCHandleType.Pinned);
var pointer = Add((void*)handle.AddrOfPinnedObject(), index);
return new BufferHandle(this, pointer, handle);
}
}

protected internal override bool TryGetArray(out ArraySegment<T> arraySegment)
{
if (IsDisposed) BufferPrimitivesThrowHelper.ThrowObjectDisposedException(nameof(OwnedArray<T>));
arraySegment = new ArraySegment<T>(_array);
return true;
}

protected override void Dispose(bool disposing)
{
_array = null;
}

public override void Retain()
{
if (IsDisposed) BufferPrimitivesThrowHelper.ThrowObjectDisposedException(nameof(OwnedArray<T>));
Interlocked.Increment(ref _referenceCount);
}

public override void Release()
{
if (!IsRetained) BufferPrimitivesThrowHelper.ThrowInvalidOperationException();
if (Interlocked.Decrement(ref _referenceCount) == 0)
{
OnNoReferences();
}
}

protected virtual void OnNoReferences()
{
}

public override bool IsRetained => _referenceCount > 0;

public override bool IsDisposed => _array == null;
}
}

+ 25
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Runtime/BufferDebuggerView.cs View File

@@ -0,0 +1,25 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Diagnostics;

namespace System.Runtime
{
internal class BufferDebuggerView<T>
{
private ReadOnlyBuffer<T> _buffer;

public BufferDebuggerView(Buffer<T> buffer)
{
_buffer = buffer;
}

[DebuggerBrowsable(DebuggerBrowsableState.RootHidden)]
public T[] Items
{
get {
return _buffer.ToArray();
}
}
}
}

+ 131
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Runtime/BufferPrimitivesThrowHelper.cs View File

@@ -0,0 +1,131 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Diagnostics;
using System.Runtime.CompilerServices;

namespace System.Runtime
{
internal static class BufferPrimitivesThrowHelper
{
public static void ThrowArgumentNullException(string argument)
{
throw new ArgumentNullException(argument);
}

public static void ThrowArgumentNullException(ExceptionArgument argument)
{
throw GetArgumentNullException(argument);
}

public static void ThrowArgumentException()
{
throw GetArgumentException();
}

public static void ThrowArgumentException(ExceptionArgument argument)
{
throw GetArgumentException(argument);
}

public static void ThrowArgumentOutOfRangeException()
{
throw GetArgumentOutOfRangeException();
}

public static void ThrowArgumentOutOfRangeException(ExceptionArgument argument)
{
throw GetArgumentOutOfRangeException(argument);
}

public static void ThrowInvalidOperationException()
{
throw GetInvalidOperationException();
}

public static void ThrowInvalidOperationException_ForBoxingSpans()
{
throw GetInvalidOperationException_ForBoxingSpans();
}

public static void ThrowObjectDisposedException(string objectName)
{
throw GetObjectDisposedException(objectName);
}

public static void ThrowArrayTypeMismatchException(Type type)
{
throw CreateArrayTypeMismatchException(type);
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static ArgumentNullException GetArgumentNullException(ExceptionArgument argument)
{
return new ArgumentNullException(GetArgumentName(argument));
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static ArgumentException GetArgumentException()
{
return new ArgumentException();
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static ArgumentException GetArgumentException(ExceptionArgument argument)
{
return new ArgumentException(GetArgumentName(argument));
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static ArgumentOutOfRangeException GetArgumentOutOfRangeException()
{
return new ArgumentOutOfRangeException();
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static ArgumentOutOfRangeException GetArgumentOutOfRangeException(ExceptionArgument argument)
{
return new ArgumentOutOfRangeException(GetArgumentName(argument));
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static InvalidOperationException GetInvalidOperationException()
{
return new InvalidOperationException();
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static InvalidOperationException GetInvalidOperationException_ForBoxingSpans()
{
return new InvalidOperationException("Spans must not be boxed");
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static ObjectDisposedException GetObjectDisposedException(string objectName)
{
return new ObjectDisposedException(objectName);
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static ArrayTypeMismatchException CreateArrayTypeMismatchException(Type type)
{
return new ArrayTypeMismatchException(type.ToString());
}

private static string GetArgumentName(ExceptionArgument argument)
{
Debug.Assert(Enum.IsDefined(typeof(ExceptionArgument), argument),
"The enum value is not defined, please check the ExceptionArgument Enum.");

return argument.ToString();
}
}

internal enum ExceptionArgument
{
pointer,
array,
start
}
}

+ 122
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Runtime/Contract.cs View File

@@ -0,0 +1,122 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Runtime
{
internal static class Contract
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Requires(bool condition)
{
if (!condition)
{
BufferPrimitivesThrowHelper.ThrowArgumentException();
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void RequiresNotNull<T>(ExceptionArgument argument, T obj) where T : class
{
if (obj == null)
{
BufferPrimitivesThrowHelper.ThrowArgumentNullException(argument);
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe void RequiresNotNull(ExceptionArgument argument, void* ptr)
{
if (ptr == null)
{
BufferPrimitivesThrowHelper.ThrowArgumentNullException(argument);
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe void RequiresSameReference(void* ptr0, void* ptr1)
{
if (ptr0 != ptr1)
{
BufferPrimitivesThrowHelper.ThrowArgumentException(ExceptionArgument.pointer);
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void RequiresNonNegative(int n)
{
if (n < 0)
{
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException();
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void RequiresInRange(int start, uint length)
{
if ((uint)start >= length)
{
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void RequiresInRange(uint start, uint length)
{
if (start >= length)
{
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException();
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void RequiresInInclusiveRange(int start, uint length)
{
if ((uint)start > length)
{
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException();
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static unsafe void RequiresOneNotNull<T>(T[] array, void* pointer)
{
if (array == null && pointer == null)
{
BufferPrimitivesThrowHelper.ThrowArgumentException();
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void RequiresInInclusiveRange( uint start, uint length)
{
if (start > length)
{
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException();
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void RequiresInInclusiveRange(int start, int length, uint existingLength)
{
if ((uint)start > existingLength
|| length < 0
|| (uint)(start + length) > existingLength)
{
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void RequiresInInclusiveRange(uint start, uint length, uint existingLength)
{
if (start > existingLength
|| (start + length) > existingLength)
{
BufferPrimitivesThrowHelper.ThrowArgumentOutOfRangeException();
}
}
}
}


+ 23
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Runtime/HashingHelper.cs View File

@@ -0,0 +1,23 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace System.Runtime
{
static class HashingHelper
{
public static int CombineHashCodes(int left, int right)
{
return ((left << 5) + left) ^ right;
}

public static int CombineHashCodes(int h1, int h2, int h3)
{
return CombineHashCodes(CombineHashCodes(h1, h2), h3);
}

public static int CombineHashCodes(int h1, int h2, int h3, int h4)
{
return CombineHashCodes(CombineHashCodes(h1, h2), CombineHashCodes(h3, h4));
}
}
}

+ 11
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Runtime/PrimitiveAttribute.cs View File

@@ -0,0 +1,11 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace System.Runtime
{
[AttributeUsage(AttributeTargets.GenericParameter)]
public sealed class PrimitiveAttribute : Attribute
{
}
}


+ 25
- 0
src/Discord.Net.Core/_corefxlab/System.Buffers.Primitives/System/Runtime/ReadOnlyBufferDebuggerView.cs View File

@@ -0,0 +1,25 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Diagnostics;

namespace System.Runtime
{
internal class ReadOnlyBufferDebuggerView<T>
{
private ReadOnlyBuffer<T> _buffer;

public ReadOnlyBufferDebuggerView(ReadOnlyBuffer<T> buffer)
{
_buffer = buffer;
}

[DebuggerBrowsable(DebuggerBrowsableState.RootHidden)]
public T[] Items
{
get {
return _buffer.ToArray();
}
}
}
}

+ 41
- 0
src/Discord.Net.Core/_corefxlab/System.Collections.Sequences/System/Collections/Sequences/ArrayList.cs View File

@@ -0,0 +1,41 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.


namespace System.Collections.Sequences
{
// This type is illustrating how to implement the new enumerable on index based datastructure
public sealed class ArrayList<T> : ISequence<T>
{
ResizableArray<T> _items;

public ArrayList()
{
_items = new ResizableArray<T>(0);
}
public ArrayList(int capacity)
{
_items = new ResizableArray<T>(capacity);
}

public int Length => _items.Count;

public T this[int index] => _items[index];

public void Add(T item)
{
_items.Add(item);
}

public SequenceEnumerator<T> GetEnumerator()
{
return new SequenceEnumerator<T>(this);
}

public bool TryGet(ref Position position, out T item, bool advance = true)
{
return _items.TryGet(ref position, out item, advance);
}
}
}

+ 18
- 0
src/Discord.Net.Core/_corefxlab/System.Collections.Sequences/System/Collections/Sequences/ISequence.cs View File

@@ -0,0 +1,18 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace System.Collections.Sequences
{
// new interface
public interface ISequence<T>
{
/// <summary>
///
/// </summary>
/// <param name="position"></param>
/// <param name="advance"></param>
/// <returns></returns>
/// <remarks></remarks>
bool TryGet(ref Position position, out T item, bool advance = true);
}
}

+ 47
- 0
src/Discord.Net.Core/_corefxlab/System.Collections.Sequences/System/Collections/Sequences/Position.cs View File

@@ -0,0 +1,47 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Collections.Sequences
{
public struct Position : IEquatable<Position>
{
public object ObjectPosition;
public int IntegerPosition;
public int Tag;

public static readonly Position First = new Position();
public static readonly Position AfterLast = new Position() { IntegerPosition = int.MaxValue - 1 };

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool operator==(Position left, Position right)
{
return left.Equals(right);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool operator!=(Position left, Position right)
{
return left.Equals(right);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool Equals(Position other)
{
return IntegerPosition == other.IntegerPosition && ObjectPosition == other.ObjectPosition;
}

public override int GetHashCode()
{
return ObjectPosition == null ? IntegerPosition.GetHashCode() : ObjectPosition.GetHashCode();
}

public override bool Equals(object obj)
{
if(obj is Position)
return base.Equals((Position)obj);
return false;
}
}
}

+ 124
- 0
src/Discord.Net.Core/_corefxlab/System.Collections.Sequences/System/Collections/Sequences/ResizableArray.cs View File

@@ -0,0 +1,124 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.


namespace System.Collections.Sequences
{
// a List<T> like type designed to be embeded in other types
public struct ResizableArray<T>
{
private T[] _array;
private int _count;

public ResizableArray(int capacity)
{
_array = new T[capacity];
_count = 0;
}

public ResizableArray(T[] array, int count = 0)
{
_array = array;
_count = count;
}

public T[] Items
{
get { return _array; }
set { _array = value; }
}
public int Count
{
get { return _count; }
set { _count = value; }
}

public int Capacity => _array.Length;

public T this[int index]
{
get {
if (index > _count - 1) throw new IndexOutOfRangeException();
return _array[index];
}
set {
if (index > _count - 1) throw new IndexOutOfRangeException();
_array[index] = value;
}
}

public void Add(T item)
{
if (_array.Length == _count) {
Resize();
}
_array[_count++] = item;
}

public void AddAll(T[] items)
{
if (items.Length > _array.Length - _count) {
Resize(items.Length + _count);
}
items.CopyTo(_array, _count);
_count += items.Length;
}

public void AddAll(ReadOnlySpan<T> items)
{
if (items.Length > _array.Length - _count) {
Resize(items.Length + _count);
}
items.CopyTo(new Span<T>(_array, _count));
_count += items.Length;
}

public void Clear()
{
_count = 0;
}

public T[] Resize(int newSize = -1)
{
var oldArray = _array;
if (newSize == -1) {
if(_array == null || _array.Length == 0) {
newSize = 4;
}
else {
newSize = _array.Length << 1;
}
}

var newArray = new T[newSize];
new Span<T>(_array, 0, _count).CopyTo(newArray);
_array = newArray;
return oldArray;
}

public T[] Resize(T[] newArray)
{
if (newArray.Length < _count) throw new ArgumentOutOfRangeException(nameof(newArray));
var oldArray = _array;
Array.Copy(_array, 0, newArray, 0, _count);
_array = newArray;
return oldArray;
}

public bool TryGet(ref Position position, out T item, bool advance = true)
{
if (position.IntegerPosition < _count) {
item = _array[position.IntegerPosition];
if (advance) { position.IntegerPosition++; }
return true;
}

item = default;
position = Position.AfterLast;
return false;
}

public ArraySegment<T> Full => new ArraySegment<T>(_array, 0, _count);
public ArraySegment<T> Free => new ArraySegment<T>(_array, _count, _array.Length - _count);
}
}

+ 31
- 0
src/Discord.Net.Core/_corefxlab/System.Collections.Sequences/System/Collections/Sequences/SequenceEnumerator.cs View File

@@ -0,0 +1,31 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Collections.Sequences
{
public struct SequenceEnumerator<T>
{
Position _position;
ISequence<T> _sequence;
T _current;
bool first; // this is needed so that MoveNext does not advance the first time it's called

public SequenceEnumerator(ISequence<T> sequence) {
_sequence = sequence;
_position = Position.First;
_current = default;
first = true;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool MoveNext() {
var result = _sequence.TryGet(ref _position, out _current, advance: !first);
first = false;
return result;
}

public T Current => _current;
}
}

+ 402
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/CompositeFormat.cs View File

@@ -0,0 +1,402 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Buffers;
using System.Diagnostics;

namespace System.Text.Formatting
{

// This whole API is very speculative, i.e. I am not sure I am happy with the design
// This API is trying to do composite formatting without boxing (or any other allocations).
// And because not all types in the platfrom implement IBufferFormattable (in particular built-in primitives don't),
// it needs to play some tricks with generic type parameters. But as you can see at the end of AppendUntyped, I am not sure how to tick the type system
// not never box.
public static class CompositeFormattingExtensions
{
public static void Format<TFormatter, T0>(this TFormatter formatter, string compositeFormat, T0 arg0) where TFormatter : ITextOutput
{
var reader = new CompositeFormatReader(compositeFormat);
while (true)
{
var segment = reader.Next();
if (segment == null) return;

if (segment.Value.Count == 0) // insertion point
{
if (segment.Value.Index == 0) formatter.AppendUntyped(arg0, segment.Value.Format);
else throw new Exception("invalid insertion point");
}
else // literal
{
formatter.Append(compositeFormat, segment.Value.Index, segment.Value.Count);
}
}
}

public static void Format<TFormatter, T0, T1>(this TFormatter formatter, string compositeFormat, T0 arg0, T1 arg1) where TFormatter : ITextOutput
{
var reader = new CompositeFormatReader(compositeFormat);
while (true)
{
var segment = reader.Next();
if (segment == null) return;

if (segment.Value.Count == 0) // insertion point
{
if (segment.Value.Index == 0) formatter.AppendUntyped(arg0, segment.Value.Format);
else if (segment.Value.Index == 1) formatter.AppendUntyped(arg1, segment.Value.Format);
else throw new Exception("invalid insertion point");
}
else // literal
{
formatter.Append(compositeFormat, segment.Value.Index, segment.Value.Count);
}
}
}

public static void Format<TFormatter, T0, T1, T2>(this TFormatter formatter, string compositeFormat, T0 arg0, T1 arg1, T2 arg2) where TFormatter : ITextOutput
{
var reader = new CompositeFormatReader(compositeFormat);
while (true)
{
var segment = reader.Next();
if (segment == null) return;

if (segment.Value.Count == 0) // insertion point
{
if (segment.Value.Index == 0) formatter.AppendUntyped(arg0, segment.Value.Format);
else if (segment.Value.Index == 1) formatter.AppendUntyped(arg1, segment.Value.Format);
else if (segment.Value.Index == 2) formatter.AppendUntyped(arg2, segment.Value.Format);
else throw new Exception("invalid insertion point");
}
else // literal
{
formatter.Append(compositeFormat, segment.Value.Index, segment.Value.Count);
}
}
}

public static void Format<TFormatter, T0, T1, T2, T3>(this TFormatter formatter, string compositeFormat, T0 arg0, T1 arg1, T2 arg2, T3 arg3) where TFormatter : ITextOutput
{
var reader = new CompositeFormatReader(compositeFormat);
while (true)
{
var segment = reader.Next();
if (segment == null) return;

if (segment.Value.Count == 0) // insertion point
{
if (segment.Value.Index == 0) formatter.AppendUntyped(arg0, segment.Value.Format);
else if (segment.Value.Index == 1) formatter.AppendUntyped(arg1, segment.Value.Format);
else if (segment.Value.Index == 2) formatter.AppendUntyped(arg2, segment.Value.Format);
else if (segment.Value.Index == 3) formatter.AppendUntyped(arg3, segment.Value.Format);
else throw new Exception("invalid insertion point");
}
else // literal
{
formatter.Append(compositeFormat, segment.Value.Index, segment.Value.Count);
}
}
}

public static void Format<TFormatter, T0, T1, T2, T3, T4>(this TFormatter formatter, string compositeFormat, T0 arg0, T1 arg1, T2 arg2, T3 arg3, T4 arg4) where TFormatter : ITextOutput
{
var reader = new CompositeFormatReader(compositeFormat);
while (true)
{
var segment = reader.Next();
if (segment == null) return;

if (segment.Value.Count == 0) // insertion point
{
if (segment.Value.Index == 0) formatter.AppendUntyped(arg0, segment.Value.Format);
else if (segment.Value.Index == 1) formatter.AppendUntyped(arg1, segment.Value.Format);
else if (segment.Value.Index == 2) formatter.AppendUntyped(arg2, segment.Value.Format);
else if (segment.Value.Index == 3) formatter.AppendUntyped(arg3, segment.Value.Format);
else if (segment.Value.Index == 4) formatter.AppendUntyped(arg4, segment.Value.Format);
else throw new Exception("invalid insertion point");
}
else // literal
{
formatter.Append(compositeFormat, segment.Value.Index, segment.Value.Count);
}
}
}

// TODO: this should be removed and an ability to append substrings should be added
static void Append<TFormatter>(this TFormatter formatter, string whole, int index, int count) where TFormatter : ITextOutput
{
var buffer = formatter.Buffer;
var maxBytes = count << 4; // this is the worst case, i.e. 4 bytes per char
while(buffer.Length < maxBytes)
{
formatter.Enlarge(maxBytes);
buffer = formatter.Buffer;
}

// this should be optimized using fixed pointer to substring, but I will wait with this till we design proper substring

var characters = whole.Slice(index, count);
if (!formatter.TryAppend(characters, formatter.SymbolTable))
{
Debug.Assert(false, "this should never happen"); // because I pre-resized the buffer to 4 bytes per char at the top of this method.
}
}

static void AppendUntyped<TFormatter, T>(this TFormatter formatter, T value, ParsedFormat format) where TFormatter : ITextOutput
{
#region Built in types
var i32 = value as int?;
if (i32 != null)
{
formatter.Append(i32.Value, format);
return;
}
var i64 = value as long?;
if (i64 != null)
{
formatter.Append(i64.Value, format);
return;
}
var i16 = value as short?;
if (i16 != null)
{
formatter.Append(i16.Value, format);
return;
}
var b = value as byte?;
if (b != null)
{
formatter.Append(b.Value, format);
return;
}
var c = value as char?;
if (c != null)
{
formatter.Append(c.Value);
return;
}
var u32 = value as uint?;
if (u32 != null)
{
formatter.Append(u32.Value, format);
return;
}
var u64 = value as ulong?;
if (u64 != null)
{
formatter.Append(u64.Value, format);
return;
}
var u16 = value as ushort?;
if (u16 != null)
{
formatter.Append(u16.Value, format);
return;
}
var sb = value as sbyte?;
if (sb != null)
{
formatter.Append(sb.Value, format);
return;
}
var str = value as string;
if (str != null)
{
formatter.Append(str);
return;
}
var dt = value as DateTime?;
if (dt != null)
{
formatter.Append(dt.Value, format);
return;
}
var dto = value as DateTimeOffset?;
if (dto != null) {
formatter.Append(dto.Value, format);
return;
}
var ts = value as TimeSpan?;
if (ts != null)
{
formatter.Append(ts.Value, format);
return;
}
var guid = value as Guid?;
if (guid != null) {
formatter.Append(guid.Value, format);
return;
}
#endregion

if (value is IBufferFormattable)
{
formatter.Append((IBufferFormattable)value, format); // this is boxing. not sure how to avoid it.
return;
}

throw new NotSupportedException("value is not formattable.");
}

// this is just a state machine walking the composite format and instructing CompositeFormattingExtensions.Format overloads on what to do.
// this whole type is not just a hacky prototype.
// I will clean it up later if I decide that I like this whole composite format model.
struct CompositeFormatReader
{
string _compositeFormatString;
int _currentIndex;
int _spanStart;
State _state;

public CompositeFormatReader(string format)
{
_compositeFormatString = format;
_currentIndex = 0;
_spanStart = 0;
_state = State.New;
}

public CompositeSegment? Next()
{
while (_currentIndex < _compositeFormatString.Length)
{
char c = _compositeFormatString[_currentIndex];
if (c == '{')
{
if (_state == State.Literal)
{
_state = State.New;
return CompositeSegment.Literal(_spanStart, _currentIndex);
}
if ((_currentIndex + 1 < _compositeFormatString.Length) && (_compositeFormatString[_currentIndex + 1] == c))
{
_state = State.Literal;
_currentIndex++;
_spanStart = _currentIndex;
}
else
{
_currentIndex++;
return ParseInsertionPoint();
}
}
else if (c == '}')
{
if ((_currentIndex + 1 < _compositeFormatString.Length) && (_compositeFormatString[_currentIndex + 1] == c))
{
if (_state == State.Literal)
{
_state = State.New;
return CompositeSegment.Literal(_spanStart, _currentIndex);
}
_state = State.Literal;
_currentIndex++;
_spanStart = _currentIndex;
}
else
{
throw new Exception("missing start bracket");
}
}
else
{
if (_state != State.Literal)
{
_state = State.Literal;
_spanStart = _currentIndex;
}
}
_currentIndex++;
}
if (_state == State.Literal)
{
_state = State.New;
return CompositeSegment.Literal(_spanStart, _currentIndex);
}
return null;
}

// this should be replaced with InvariantFormatter.Parse
static bool TryParse(string compositeFormat, int start, int count, out uint value, out int consumed)
{
consumed = 0;
value = 0;
for (int i = start; i < start + count; i++)
{
var digit = (byte)(compositeFormat[i] - '0');
if (digit >= 0 && digit <= 9)
{
value *= 10;
value += digit;
consumed++;
}
else
{
if (i == start) return false;
else return true;
}
}
return true;
}

CompositeSegment ParseInsertionPoint()
{
uint arg;
int consumed;
char? formatSpecifier = null;

if (!TryParse(_compositeFormatString, _currentIndex, 5, out arg, out consumed))
{
throw new Exception("invalid insertion point");
}
_currentIndex += consumed;
if (_currentIndex >= _compositeFormatString.Length)
{
throw new Exception("missing end bracket");
}

if(_compositeFormatString[_currentIndex] == ':')
{
_currentIndex++;
formatSpecifier = _compositeFormatString[_currentIndex];
_currentIndex++;
}

if (_compositeFormatString[_currentIndex] != '}')
{
throw new Exception("missing end bracket");
}

_currentIndex++;
var parsedFormat = (formatSpecifier.HasValue && formatSpecifier.Value != 0) ? new ParsedFormat(formatSpecifier.Value) : default;
return CompositeSegment.InsertionPoint(arg, parsedFormat);
}

public enum State : byte
{
New,
Literal,
InsertionPoint
}

public struct CompositeSegment
{
public ParsedFormat Format { get; private set; }
public int Index { get; private set; }
public int Count { get; private set; }

public static CompositeSegment InsertionPoint(uint argIndex, ParsedFormat format)
{
return new CompositeSegment() { Index = (int)argIndex, Format = format };
}

public static CompositeSegment Literal(int startIndex, int endIndex)
{
return new CompositeSegment() { Index = startIndex, Count = endIndex - startIndex };
}
}
}
}
}

+ 55
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/Formatters/ArrayFormatter.cs View File

@@ -0,0 +1,55 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Buffers;
using System.Collections.Sequences;

namespace System.Text.Formatting
{
public class ArrayFormatter : ITextOutput
{
ResizableArray<byte> _buffer;
SymbolTable _symbolTable;
ArrayPool<byte> _pool;

public ArrayFormatter(int capacity, SymbolTable symbolTable, ArrayPool<byte> pool = null)
{
_pool = pool ?? ArrayPool<byte>.Shared;
_symbolTable = symbolTable;
_buffer = new ResizableArray<byte>(_pool.Rent(capacity));
}

public int CommitedByteCount => _buffer.Count;

public void Clear() {
_buffer.Count = 0;
}

public ArraySegment<byte> Free => _buffer.Free;
public ArraySegment<byte> Formatted => _buffer.Full;

public SymbolTable SymbolTable => _symbolTable;

public Span<byte> Buffer => Free.AsSpan();

void IOutput.Enlarge(int desiredBufferLength)
{
if (desiredBufferLength < 1) desiredBufferLength = 1;
var doubleCount = _buffer.Free.Count * 2;
int newSize = desiredBufferLength>doubleCount?desiredBufferLength:doubleCount;
var newArray = _pool.Rent(newSize + _buffer.Count);
var oldArray = _buffer.Resize(newArray);
_pool.Return(oldArray);
}

void IOutput.Advance(int bytes)
{
_buffer.Count += bytes;
if(_buffer.Count > _buffer.Count)
{
throw new InvalidOperationException("More bytes commited than returned from FreeBuffer");
}
}
}
}

+ 32
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/Formatters/OutputFormatter.cs View File

@@ -0,0 +1,32 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Buffers;

namespace System.Text.Formatting
{
public struct OutputFormatter<TOutput> : ITextOutput where TOutput : IOutput
{
TOutput _output;
SymbolTable _symbolTable;

public OutputFormatter(TOutput output, SymbolTable symbolTable)
{
_output = output;
_symbolTable = symbolTable;
}

public OutputFormatter(TOutput output) : this(output, SymbolTable.InvariantUtf8)
{
}

public Span<byte> Buffer => _output.Buffer;

public SymbolTable SymbolTable => _symbolTable;

public void Advance(int bytes) => _output.Advance(bytes);

public void Enlarge(int desiredBufferLength = 0) => _output.Enlarge(desiredBufferLength);
}
}

+ 105
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/Formatters/SequenceFormatter.cs View File

@@ -0,0 +1,105 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Buffers;
using System.Collections.Sequences;

namespace System.Text.Formatting
{
public static class SequenceFormatterExtensions
{
public static SequenceFormatter<TSequence> CreateFormatter<TSequence>(this TSequence sequence, SymbolTable symbolTable = null) where TSequence : ISequence<Buffer<byte>>
{
return new SequenceFormatter<TSequence>(sequence, symbolTable);
}
}

public class SequenceFormatter<TSequence> : ITextOutput where TSequence : ISequence<Buffer<byte>>
{
ISequence<Buffer<byte>> _buffers;
SymbolTable _symbolTable;

Position _currentPosition = Position.First;
int _currentWrittenBytes;
Position _previousPosition = Position.AfterLast;
int _previousWrittenBytes;
int _totalWritten;

public SequenceFormatter(TSequence buffers, SymbolTable symbolTable)
{
_symbolTable = symbolTable;
_buffers = buffers;
_previousWrittenBytes = -1;
}

Span<byte> IOutput.Buffer
{
get {
return Current.Span.Slice(_currentWrittenBytes);
}
}

private Buffer<byte> Current {
get {
Buffer<byte> result;
if (!_buffers.TryGet(ref _currentPosition, out result, advance: false)) { throw new InvalidOperationException(); }
return result;
}
}
private Buffer<byte> Previous
{
get {
Buffer<byte> result;
if (!_buffers.TryGet(ref _previousPosition, out result, advance: false)) { throw new InvalidOperationException(); }
return result;
}
}
private bool NeedShift => _previousWrittenBytes != -1;

SymbolTable ITextOutput.SymbolTable => _symbolTable;

public int TotalWritten => _totalWritten;

void IOutput.Enlarge(int desiredBufferLength)
{
if (NeedShift) throw new NotImplementedException("need to allocate temp array");

_previousPosition = _currentPosition;
_previousWrittenBytes = _currentWrittenBytes;

Buffer<byte> span;
if (!_buffers.TryGet(ref _currentPosition, out span)) {
throw new InvalidOperationException();
}
_currentWrittenBytes = 0;
}

void IOutput.Advance(int bytes)
{
var current = Current;
if (NeedShift) {
var previous = Previous;
var spaceInPrevious = previous.Length - _previousWrittenBytes;
if(spaceInPrevious < bytes) {
current.Slice(0, spaceInPrevious).CopyTo(previous.Span.Slice(_previousWrittenBytes));
current.Slice(spaceInPrevious, bytes - spaceInPrevious).CopyTo(current.Span);
_previousWrittenBytes = -1;
_currentWrittenBytes = bytes - spaceInPrevious;
}
else {
current.Slice(0, bytes).CopyTo(previous.Span.Slice(_previousWrittenBytes));
_currentPosition = _previousPosition;
_currentWrittenBytes = _previousWrittenBytes + bytes;
}

}
else {
if (current.Length - _currentWrittenBytes < bytes) throw new NotImplementedException();
_currentWrittenBytes += bytes;
}

_totalWritten += bytes;
}
}
}

+ 83
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/Formatters/StreamFormatter.cs View File

@@ -0,0 +1,83 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.IO;
using System.Buffers;

namespace System.Text.Formatting
{
public struct StreamFormatter : ITextOutput, IDisposable
{
Stream _stream;
SymbolTable _symbolTable;
byte[] _buffer;
ArrayPool<byte> _pool;

public StreamFormatter(Stream stream, ArrayPool<byte> pool) : this(stream, SymbolTable.InvariantUtf16, pool)
{
}

public StreamFormatter(Stream stream, SymbolTable symbolTable, ArrayPool<byte> pool, int bufferSize = 256)
{
_pool = pool;
_buffer = null;
if (bufferSize > 0)
{
_buffer = _pool.Rent(bufferSize);
}
_symbolTable = symbolTable;
_stream = stream;
}

Span<byte> IOutput.Buffer
{
get
{
if (_buffer == null)
{
_buffer = _pool.Rent(256);
}
return new Span<byte>(_buffer);
}
}

void IOutput.Enlarge(int desiredBufferLength)
{
var newSize = _buffer.Length * 2;
if(desiredBufferLength != 0){
newSize = desiredBufferLength;
}
var temp = _buffer;
_buffer = _pool.Rent(newSize);
_pool.Return(temp);
}

// ISSUE
// I would like to lazy write to the stream, but unfortunatelly this seems to be exclusive with this type being a struct.
// If the write was lazy, passing this struct by value could result in data loss.
// A stack frame could write more data to the buffer, and then when the frame pops, the infroamtion about how much was written could be lost.
// On the other hand, I cannot make this type a class and keep using it as it can be used today (i.e. pass streams around and create instances of this type on demand).
// Too bad we don't support move semantics and stack only structs.
void IOutput.Advance(int bytes)
{
_stream.Write(_buffer, 0, bytes);
}

SymbolTable ITextOutput.SymbolTable
{
get {
return _symbolTable;
}
}

/// <summary>
/// Returns buffers to the pool
/// </summary>
public void Dispose()
{
_pool.Return(_buffer);
_buffer = null;
_stream = null;
}
}
}

+ 79
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/Formatters/StringFormatter.cs View File

@@ -0,0 +1,79 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Buffers;
using System.Collections.Sequences;

namespace System.Text.Formatting
{
public class StringFormatter : ITextOutput, IDisposable
{
ResizableArray<byte> _buffer;
ArrayPool<byte> _pool;
public SymbolTable SymbolTable { get; set; } = SymbolTable.InvariantUtf16;

public StringFormatter(int characterCapacity = 32, ArrayPool<byte> pool = null)
{
if (pool == null) _pool = ArrayPool<byte>.Shared;
else _pool = pool;
_buffer = new ResizableArray<byte>(_pool.Rent(characterCapacity * 2));
}

public void Dispose()
{
_pool.Return(_buffer.Items);
_buffer.Count = 0;
}

public void Append(char character) {
_buffer.Add((byte)character);
_buffer.Add((byte)(character >> 8));
}

//TODO: this should use Span<byte>
public void Append(string text)
{
foreach (char character in text)
{
Append(character);
}
}

//TODO: this should use Span<byte>
public void Append(ReadOnlySpan<char> substring)
{
for (int i = 0; i < substring.Length; i++)
{
Append(substring[i]);
}
}

public void Clear()
{
_buffer.Clear();
}

public override string ToString()
{
var text = Encoding.Unicode.GetString(_buffer.Items, 0, _buffer.Count);
return text;
}

Span<byte> IOutput.Buffer => _buffer.Free.AsSpan();

void IOutput.Enlarge(int desiredBufferLength)
{
if (desiredBufferLength < 1) desiredBufferLength = 1;
var doubleCount = _buffer.Free.Count * 2;
int newSize = desiredBufferLength > doubleCount ? desiredBufferLength : doubleCount;
var newArray = _pool.Rent(newSize + _buffer.Count);
var oldArray = _buffer.Resize(newArray);
_pool.Return(oldArray);
}

void IOutput.Advance(int bytes)
{
_buffer.Count += bytes;
}
}
}

+ 346
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/IOutputExtensions.cs View File

@@ -0,0 +1,346 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Buffers;
using System.Text.Utf8;

namespace System.Text.Formatting
{
public static class IOutputExtensions
{
public static void Append<TFormatter, T>(this TFormatter formatter, T value, SymbolTable symbolTable, ParsedFormat format = default) where T : IBufferFormattable where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter, T>(this TFormatter formatter, T value, SymbolTable symbolTable, ParsedFormat format = default) where T : IBufferFormattable where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, string value, SymbolTable symbolTable) where TFormatter : IOutput
{
formatter.Append(value.AsSpan(), symbolTable);
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, string value, SymbolTable symbolTable) where TFormatter : IOutput
{
return formatter.TryAppend(value.AsSpan(), symbolTable);
}

public static void Append<TFormatter>(this TFormatter formatter, ReadOnlySpan<char> value, SymbolTable symbolTable) where TFormatter : IOutput
{
if (value.Length <= 256)
{
while (!formatter.TryAppend(value, symbolTable)) {
formatter.Enlarge();
}
}
else // slice the span into smaller pieces, otherwise the enlarge might fail.
{
var leftToWrite = value;
while (leftToWrite.Length > 0)
{
var chunkLength = leftToWrite.Length < 256 ? leftToWrite.Length : 256;
if (char.IsHighSurrogate(leftToWrite[chunkLength - 1]))
{
chunkLength--;
if (chunkLength == 0) throw new Exception("value ends in a high surrogate");
}

var chunk = leftToWrite.Slice(0, chunkLength);
formatter.Append(chunk, symbolTable);
leftToWrite = leftToWrite.Slice(chunkLength);
}
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, ReadOnlySpan<char> value, SymbolTable symbolTable) where TFormatter : IOutput
{
var result = symbolTable.TryEncode(value, formatter.Buffer, out int consumed, out int written);
if (result)
formatter.Advance(written);

return result;
}

public static void Append<TFormatter>(this TFormatter formatter, char value, SymbolTable symbolTable) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, char value, SymbolTable symbolTable) where TFormatter : IOutput
{
unsafe
{
ReadOnlySpan<char> input = new ReadOnlySpan<char>(&value, 1);
return formatter.TryAppend(input, symbolTable);
}
}

public static void Append<TFormatter>(this TFormatter formatter, Utf8String value, SymbolTable encoder) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, encoder)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, Utf8String value, SymbolTable symbolTable) where TFormatter : IOutput
{
int bytesWritten;
int consumed;
if (!symbolTable.TryEncode(value, formatter.Buffer, out consumed, out bytesWritten)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, uint value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, uint value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, ulong value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, ulong value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, int value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, int value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, long value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, long value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, byte value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, byte value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, sbyte value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, sbyte value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, ushort value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, ushort value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, short value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, short value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, Guid value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, Guid value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, DateTime value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, DateTime value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, DateTimeOffset value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, DateTimeOffset value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, TimeSpan value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, TimeSpan value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, float value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, float value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, double value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, symbolTable, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, double value, SymbolTable symbolTable, ParsedFormat format = default) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, symbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}
}
}

+ 15
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/ITextOutput.cs View File

@@ -0,0 +1,15 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Buffers;

namespace System.Text.Formatting
{
// this interface would be implemented by types that want to support formatting, i.e. TextWriter/StringBuilder-like types.
// the interface is used by an extension method in IFormatterExtensions.
// One thing I am not sure here is if it's ok for these APIs to be synchronous, but I guess I will wait till I find a concrete issue with this.
public interface ITextOutput : IOutput
{
SymbolTable SymbolTable { get; }
}
}

+ 319
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Formatting/ITextOutputExtensions.cs View File

@@ -0,0 +1,319 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Text.Utf8;

namespace System.Text.Formatting
{
public static class ITextOutputExtensions
{
public static void Append<TFormatter, T>(this TFormatter formatter, T value, ParsedFormat format = default) where T : IBufferFormattable where TFormatter : ITextOutput
{
while(!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter, T>(this TFormatter formatter, T value, ParsedFormat format = default) where T : IBufferFormattable where TFormatter : ITextOutput
{
int bytesWritten;
if(!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, byte value, ParsedFormat format = default) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, byte value, ParsedFormat format = default) where TFormatter : ITextOutput
{
int bytesWritten;
if(!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, sbyte value, ParsedFormat format = default) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, sbyte value, ParsedFormat format = default) where TFormatter : ITextOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, ushort value, ParsedFormat format = default) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, ushort value, ParsedFormat format = default) where TFormatter : ITextOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, short value, ParsedFormat format = default) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, short value, ParsedFormat format = default) where TFormatter : ITextOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, uint value, ParsedFormat format = default) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, uint value, ParsedFormat format = default) where TFormatter : ITextOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, int value, ParsedFormat format = default) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, int value, ParsedFormat format = default) where TFormatter : ITextOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, ulong value, ParsedFormat format = default) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, ulong value, ParsedFormat format = default) where TFormatter : ITextOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, long value, ParsedFormat format = default) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, long value, ParsedFormat format = default) where TFormatter : ITextOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, char value) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, char value) where TFormatter : ITextOutput
{
return formatter.TryAppend(value, formatter.SymbolTable);
}

public static void Append<TFormatter>(this TFormatter formatter, ReadOnlySpan<char> value) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, ReadOnlySpan<char> value) where TFormatter : ITextOutput
{
return formatter.TryAppend(value, formatter.SymbolTable);
}

public static void Append<TFormatter>(this TFormatter formatter, string value) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, string value) where TFormatter : ITextOutput
{
return formatter.TryAppend(value, formatter.SymbolTable);
}

public static void Append<TFormatter>(this TFormatter formatter, Utf8String value) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, Utf8String value) where TFormatter : ITextOutput
{
int bytesWritten;
int consumed;
if (!formatter.SymbolTable.TryEncode(value, formatter.Buffer, out consumed, out bytesWritten)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, Guid value, ParsedFormat format = default) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, Guid value, ParsedFormat format = default) where TFormatter : ITextOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, DateTime value, ParsedFormat format = default) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, DateTime value, ParsedFormat format = default) where TFormatter : ITextOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, DateTimeOffset value, ParsedFormat format = default) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, DateTimeOffset value, ParsedFormat format = default) where TFormatter : ITextOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, TimeSpan value, ParsedFormat format = default) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, TimeSpan value, ParsedFormat format = default) where TFormatter : ITextOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, float value, ParsedFormat format = default) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, float value, ParsedFormat format = default) where TFormatter : ITextOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}

public static void Append<TFormatter>(this TFormatter formatter, double value, ParsedFormat format = default) where TFormatter : ITextOutput
{
while (!formatter.TryAppend(value, format)) {
formatter.Enlarge();
}
}

public static bool TryAppend<TFormatter>(this TFormatter formatter, double value, ParsedFormat format = default) where TFormatter : ITextOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, out bytesWritten, format, formatter.SymbolTable)) {
return false;
}
formatter.Advance(bytesWritten);
return true;
}
}
}

+ 159
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Formatting/System/Text/Parsing/SequenceParser.cs View File

@@ -0,0 +1,159 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Buffers;
using System.Collections.Sequences;

namespace System.Text.Parsing
{
public static class TextSequenceExtensions
{
const int StackBufferSize = 128;

public static bool TryParseUInt64<T>(this T bufferSequence, out ulong value, out int consumed) where T : ISequence<ReadOnlyBuffer<byte>>
{
value = default;
consumed = default;
Position position = Position.First;

// Fetch the first segment
ReadOnlyBuffer<byte> first;
if (!bufferSequence.TryGet(ref position, out first)) {
return false;
}

// Attempt to parse the first segment. If it works (and it should in most cases), then return success.
bool parsed = PrimitiveParser.InvariantUtf8.TryParseUInt64(first.Span, out value, out consumed);
if (parsed && consumed < first.Length) {
return true;
}

// Apparently the we need data from the second segment to succesfully parse, and so fetch the second segment.
ReadOnlyBuffer<byte> second;
if (!bufferSequence.TryGet(ref position, out second)) {
// if there is no second segment and the first parsed succesfully, return the result of the parsing.
if (parsed) return true;
return false;
}

// Combine the first, the second, and potentially more segments into a stack allocated buffer
ReadOnlySpan<byte> combinedSpan;
unsafe
{
if (first.Length < StackBufferSize) {
var data = stackalloc byte[StackBufferSize];
var destination = new Span<byte>(data, StackBufferSize);

first.CopyTo(destination);
var free = destination.Slice(first.Length);

if (second.Length > free.Length) second = second.Slice(0, free.Length);
second.CopyTo(free);
free = free.Slice(second.Length);

ReadOnlyBuffer<byte> next;
while (free.Length > 0) {
if (bufferSequence.TryGet(ref position, out next)) {
if (next.Length > free.Length) next = next.Slice(0, free.Length);
next.CopyTo(free);
free = free.Slice(next.Length);
}
else {
break;
}
}

combinedSpan = destination.Slice(0, StackBufferSize - free.Length);

// if the stack allocated buffer parsed succesfully (and for uint it should always do), then return success.
if (PrimitiveParser.InvariantUtf8.TryParseUInt64(combinedSpan, out value, out consumed)) {
if(consumed < combinedSpan.Length || combinedSpan.Length < StackBufferSize) {
return true;
}
}
}
}

// for invariant culture, we should never reach this point, as invariant uint text is never longer than 127 bytes.
// I left this code here, as we will need it for custom cultures and possibly when we shrink the stack allocated buffer.
combinedSpan = bufferSequence.ToSpan();
if (!PrimitiveParser.InvariantUtf8.TryParseUInt64(first.Span, out value, out consumed)) {
return false;
}
return true;
}

public static bool TryParseUInt32<T>(this T bufferSequence, out uint value, out int consumed) where T : ISequence<ReadOnlyBuffer<byte>>
{
value = default;
consumed = default;
Position position = Position.First;

// Fetch the first segment
ReadOnlyBuffer<byte> first;
if (!bufferSequence.TryGet(ref position, out first)) {
return false;
}

// Attempt to parse the first segment. If it works (and it should in most cases), then return success.
bool parsed = PrimitiveParser.InvariantUtf8.TryParseUInt32(first.Span, out value, out consumed);
if (parsed && consumed < first.Length) {
return true;
}

// Apparently the we need data from the second segment to succesfully parse, and so fetch the second segment.
ReadOnlyBuffer<byte> second;
if (!bufferSequence.TryGet(ref position, out second)) {
// if there is no second segment and the first parsed succesfully, return the result of the parsing.
if (parsed) return true;
return false;
}

// Combine the first, the second, and potentially more segments into a stack allocated buffer
ReadOnlySpan<byte> combinedSpan;
unsafe
{
if (first.Length < StackBufferSize) {
var data = stackalloc byte[StackBufferSize];
var destination = new Span<byte>(data, StackBufferSize);

first.CopyTo(destination);
var free = destination.Slice(first.Length);

if (second.Length > free.Length) second = second.Slice(0, free.Length);
second.CopyTo(free);
free = free.Slice(second.Length);

ReadOnlyBuffer<byte> next;
while (free.Length > 0) {
if (bufferSequence.TryGet(ref position, out next)) {
if (next.Length > free.Length) next = next.Slice(0, free.Length);
next.CopyTo(free);
free = free.Slice(next.Length);
}
else {
break;
}
}

combinedSpan = destination.Slice(0, StackBufferSize - free.Length);

// if the stack allocated buffer parsed succesfully (and for uint it should always do), then return success.
if (PrimitiveParser.InvariantUtf8.TryParseUInt32(combinedSpan, out value, out consumed)) {
if(consumed < combinedSpan.Length || combinedSpan.Length < StackBufferSize) {
return true;
}
}
}
}

// for invariant culture, we should never reach this point, as invariant uint text is never longer than 127 bytes.
// I left this code here, as we will need it for custom cultures and possibly when we shrink the stack allocated buffer.
combinedSpan = bufferSequence.ToSpan();
if (!PrimitiveParser.InvariantUtf8.TryParseUInt32(combinedSpan, out value, out consumed)) {
return false;
}
return true;
}
}
}

+ 34
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Precondition.cs View File

@@ -0,0 +1,34 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Diagnostics
{
internal static class Precondition
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Require(bool condition)
{
if (!condition)
{
Fail();
}
}

private static void Fail()
{
if (Debugger.IsAttached)
{
Debugger.Break();
}
throw new Failure();
}

public sealed class Failure : Exception
{
static string s_message = "precondition failed";
internal Failure() : base(s_message) { }
}
}
}

+ 72
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/Ascii/Ascii_casing.cs View File

@@ -0,0 +1,72 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Buffers;

namespace System.Text.Encoders
{
public static partial class Ascii
{
static readonly byte[] s_toLower = new byte[128];
static readonly byte[] s_toUpper = new byte[128];

static Ascii()
{
for (int i = 0; i < s_toLower.Length; i++)
{
s_toLower[i] = (byte)char.ToLowerInvariant(((char)i));
s_toUpper[i] = (byte)char.ToUpperInvariant(((char)i));
}
}

public static TransformationStatus ToLowerInPlace(Span<byte> ascii, out int bytesChanged)
{
for (bytesChanged = 0; bytesChanged < ascii.Length; bytesChanged++)
{
byte next = ascii[bytesChanged];
if (next > 127)
{
return TransformationStatus.InvalidData;
}
ascii[bytesChanged] = s_toLower[next];
}
return TransformationStatus.Done;
}

public static TransformationStatus ToLower(ReadOnlySpan<byte> input, Span<byte> output, out int processedBytes)
{
int min = input.Length < output.Length ? input.Length : output.Length;
for (processedBytes = 0; processedBytes < min; processedBytes++)
{
byte next = input[processedBytes];
if (next > 127) return TransformationStatus.InvalidData;
output[processedBytes] = s_toLower[next];
}
return TransformationStatus.Done;
}

public static TransformationStatus ToUpperInPlace(Span<byte> ascii, out int bytesChanged)
{
for (bytesChanged = 0; bytesChanged < ascii.Length; bytesChanged++)
{
byte next = ascii[bytesChanged];
if (next > 127) return TransformationStatus.InvalidData;
ascii[bytesChanged] = s_toUpper[next];
}
return TransformationStatus.Done;
}

public static TransformationStatus ToUpper(ReadOnlySpan<byte> input, Span<byte> output, out int processedBytes)
{
int min = input.Length < output.Length ? input.Length : output.Length;
for (processedBytes = 0; processedBytes < min; processedBytes++)
{
byte next = input[processedBytes];
if (next > 127) return TransformationStatus.InvalidData;
output[processedBytes] = s_toUpper[next];
}
return TransformationStatus.Done;
}
}
}

+ 128
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/Ascii/Ascii_encoding.cs View File

@@ -0,0 +1,128 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Runtime.CompilerServices;

namespace System.Text.Encoders
{
public static partial class Ascii
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static string ToUtf16String(ReadOnlySpan<byte> bytes)
{
var len = bytes.Length;
if (len == 0) {
return string.Empty;
}

var result = new string('\0', len);

unsafe
{
fixed (char* destination = result)
fixed (byte* source = &bytes.DangerousGetPinnableReference()) {
if (!TryGetAsciiString(source, destination, len)) {
ThrowArgumentException();
}
}
}

return result;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static string ToUtf16String(Span<byte> bytes)
{
var len = bytes.Length;
if (len == 0) {
return string.Empty;
}

var result = new string('\0', len);

unsafe
{
fixed (char* destination = result)
fixed (byte* source = &bytes.DangerousGetPinnableReference()) {
if (!TryGetAsciiString(source, destination, len)) {
ThrowArgumentException();
}
}
}

return result;
}

static void ThrowArgumentException()
{
throw new ArgumentException();
}

static unsafe bool TryGetAsciiString(byte* input, char* output, int count)
{
var i = 0;

int isValid = 0;
while (i < count - 11) {
isValid = isValid | *input | *(input + 1) | *(input + 2) |
*(input + 3) | *(input + 4) | *(input + 5) | *(input + 6) |
*(input + 7) | *(input + 8) | *(input + 9) | *(input + 10) |
*(input + 11);

i += 12;
*(output) = (char)*(input);
*(output + 1) = (char)*(input + 1);
*(output + 2) = (char)*(input + 2);
*(output + 3) = (char)*(input + 3);
*(output + 4) = (char)*(input + 4);
*(output + 5) = (char)*(input + 5);
*(output + 6) = (char)*(input + 6);
*(output + 7) = (char)*(input + 7);
*(output + 8) = (char)*(input + 8);
*(output + 9) = (char)*(input + 9);
*(output + 10) = (char)*(input + 10);
*(output + 11) = (char)*(input + 11);
output += 12;
input += 12;
}
if (i < count - 5) {
isValid = isValid | *input | *(input + 1) | *(input + 2) |
*(input + 3) | *(input + 4) | *(input + 5);

i += 6;
*(output) = (char)*(input);
*(output + 1) = (char)*(input + 1);
*(output + 2) = (char)*(input + 2);
*(output + 3) = (char)*(input + 3);
*(output + 4) = (char)*(input + 4);
*(output + 5) = (char)*(input + 5);
output += 6;
input += 6;
}
if (i < count - 3) {
isValid = isValid | *input | *(input + 1) | *(input + 2) |
*(input + 3);

i += 4;
*(output) = (char)*(input);
*(output + 1) = (char)*(input + 1);
*(output + 2) = (char)*(input + 2);
*(output + 3) = (char)*(input + 3);
output += 4;
input += 4;
}

while (i < count) {
isValid = isValid | *input;

i++;
*output = (char)*input;
output++;
input++;
}

return isValid <= 127;
}
}
}

+ 529
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/Encoders.Utf16.cs View File

@@ -0,0 +1,529 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Buffers;
using System.Diagnostics;
using System.Runtime.CompilerServices;

namespace System.Text.Encoders
{
public static class Utf16
{
#region UTF-8 Conversions

/// <summary>
/// Calculates the byte count needed to encode the UTF-16 bytes from the specified UTF-8 sequence.
///
/// This method will consume as many of the input bytes as possible.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-8 bytes.</param>
/// <param name="bytesNeeded">On exit, contains the number of bytes required for encoding from the <paramref name="source"/>.</param>
/// <returns>A <see cref="TransformationStatus"/> value representing the expected state of the conversion.</returns>
public static TransformationStatus FromUtf8Length(ReadOnlySpan<byte> source, out int bytesNeeded)
=> Utf8.ToUtf16Length(source, out bytesNeeded);

/// <summary>
/// Converts a span containing a sequence of UTF-8 bytes into UTF-16 bytes.
///
/// This method will consume as many of the input bytes as possible.
///
/// On successful exit, the entire input was consumed and encoded successfully. In this case, <paramref name="bytesConsumed"/> will be
/// equal to the length of the <paramref name="source"/> and <paramref name="bytesWritten"/> will equal the total number of bytes written to
/// the <paramref name="destination"/>.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-8 bytes.</param>
/// <param name="destination">A span to write the UTF-16 bytes into.</param>
/// <param name="bytesConsumed">On exit, contains the number of bytes that were consumed from the <paramref name="source"/>.</param>
/// <param name="bytesWritten">On exit, contains the number of bytes written to <paramref name="destination"/></param>
/// <returns>A <see cref="TransformationStatus"/> value representing the state of the conversion.</returns>
public static TransformationStatus FromUtf8(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
=> Utf8.ToUtf16(source, destination, out bytesConsumed, out bytesWritten);

/// <summary>
/// Calculates the byte count needed to encode the UTF-8 bytes from the specified UTF-16 sequence.
///
/// This method will consume as many of the input bytes as possible.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-16 bytes.</param>
/// <param name="bytesNeeded">On exit, contains the number of bytes required for encoding from the <paramref name="source"/>.</param>
/// <returns>A <see cref="TransformationStatus"/> value representing the expected state of the conversion.</returns>
public static TransformationStatus ToUtf8Length(ReadOnlySpan<byte> source, out int bytesNeeded)
{
bytesNeeded = 0;

// try? because Convert.ConvertToUtf32 can throw
// if the high/low surrogates aren't valid; no point
// running all the tests twice per code-point
try
{
ref char utf16 = ref Unsafe.As<byte, char>(ref source.DangerousGetPinnableReference());
int utf16Length = source.Length >> 1; // byte => char count

for (int i = 0; i < utf16Length; i++)
{
var ch = Unsafe.Add(ref utf16, i);

if ((ushort)ch <= 0x7f) // Fast path for ASCII
bytesNeeded++;
else if (!char.IsSurrogate(ch))
bytesNeeded += EncodingHelper.GetUtf8EncodedBytes((uint)ch);
else
{
if (++i >= utf16Length)
return TransformationStatus.NeedMoreSourceData;

uint codePoint = (uint)char.ConvertToUtf32(ch, Unsafe.Add(ref utf16, i));
bytesNeeded += EncodingHelper.GetUtf8EncodedBytes(codePoint);
}
}

if ((utf16Length << 1) != source.Length)
return TransformationStatus.NeedMoreSourceData;

return TransformationStatus.Done;
}
catch (ArgumentOutOfRangeException)
{
return TransformationStatus.InvalidData;
}
}

/// <summary>
/// Converts a span containing a sequence of UTF-16 bytes into UTF-8 bytes.
///
/// This method will consume as many of the input bytes as possible.
///
/// On successful exit, the entire input was consumed and encoded successfully. In this case, <paramref name="bytesConsumed"/> will be
/// equal to the length of the <paramref name="source"/> and <paramref name="bytesWritten"/> will equal the total number of bytes written to
/// the <paramref name="destination"/>.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-16 bytes.</param>
/// <param name="destination">A span to write the UTF-8 bytes into.</param>
/// <param name="bytesConsumed">On exit, contains the number of bytes that were consumed from the <paramref name="source"/>.</param>
/// <param name="bytesWritten">On exit, contains the number of bytes written to <paramref name="destination"/></param>
/// <returns>A <see cref="TransformationStatus"/> value representing the state of the conversion.</returns>
public unsafe static TransformationStatus ToUtf8(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
{
//
//
// KEEP THIS IMPLEMENTATION IN SYNC WITH https://github.com/dotnet/corert/blob/master/src/System.Private.CoreLib/src/System/Text/UTF8Encoding.cs
//
//
fixed (byte* chars = &source.DangerousGetPinnableReference())
fixed (byte* bytes = &destination.DangerousGetPinnableReference())
{
char* pSrc = (char*)chars;
byte* pTarget = bytes;

char* pEnd = (char*)(chars + source.Length);
byte* pAllocatedBufferEnd = pTarget + destination.Length;

// assume that JIT will enregister pSrc, pTarget and ch

// Entering the fast encoding loop incurs some overhead that does not get amortized for small
// number of characters, and the slow encoding loop typically ends up running for the last few
// characters anyway since the fast encoding loop needs 5 characters on input at least.
// Thus don't use the fast decoding loop at all if we don't have enough characters. The threashold
// was choosen based on performance testing.
// Note that if we don't have enough bytes, pStop will prevent us from entering the fast loop.
while (pEnd - pSrc > 13)
{
// we need at least 1 byte per character, but Convert might allow us to convert
// only part of the input, so try as much as we can. Reduce charCount if necessary
int available = Math.Min(EncodingHelper.PtrDiff(pEnd, pSrc), EncodingHelper.PtrDiff(pAllocatedBufferEnd, pTarget));

// FASTLOOP:
// - optimistic range checks
// - fallbacks to the slow loop for all special cases, exception throwing, etc.

// To compute the upper bound, assume that all characters are ASCII characters at this point,
// the boundary will be decreased for every non-ASCII character we encounter
// Also, we need 5 chars reserve for the unrolled ansi decoding loop and for decoding of surrogates
// If there aren't enough bytes for the output, then pStop will be <= pSrc and will bypass the loop.
char* pStop = pSrc + available - 5;
if (pSrc >= pStop)
break;

do
{
int ch = *pSrc;
pSrc++;

if (ch > 0x7F)
{
goto LongCode;
}
*pTarget = (byte)ch;
pTarget++;

// get pSrc aligned
if ((unchecked((int)pSrc) & 0x2) != 0)
{
ch = *pSrc;
pSrc++;
if (ch > 0x7F)
{
goto LongCode;
}
*pTarget = (byte)ch;
pTarget++;
}

// Run 4 characters at a time!
while (pSrc < pStop)
{
ch = *(int*)pSrc;
int chc = *(int*)(pSrc + 2);
if (((ch | chc) & unchecked((int)0xFF80FF80)) != 0)
{
goto LongCodeWithMask;
}

// Unfortunately, this is endianess sensitive
#if BIGENDIAN
*pTarget = (byte)(ch >> 16);
*(pTarget + 1) = (byte)ch;
pSrc += 4;
*(pTarget + 2) = (byte)(chc >> 16);
*(pTarget + 3) = (byte)chc;
pTarget += 4;
#else // BIGENDIAN
*pTarget = (byte)ch;
*(pTarget + 1) = (byte)(ch >> 16);
pSrc += 4;
*(pTarget + 2) = (byte)chc;
*(pTarget + 3) = (byte)(chc >> 16);
pTarget += 4;
#endif // BIGENDIAN
}
continue;

LongCodeWithMask:
#if BIGENDIAN
// be careful about the sign extension
ch = (int)(((uint)ch) >> 16);
#else // BIGENDIAN
ch = (char)ch;
#endif // BIGENDIAN
pSrc++;

if (ch > 0x7F)
{
goto LongCode;
}
*pTarget = (byte)ch;
pTarget++;
continue;

LongCode:
// use separate helper variables for slow and fast loop so that the jit optimizations
// won't get confused about the variable lifetimes
int chd;
if (ch <= 0x7FF)
{
// 2 byte encoding
chd = unchecked((sbyte)0xC0) | (ch >> 6);
}
else
{
// if (!IsLowSurrogate(ch) && !IsHighSurrogate(ch))
if (!EncodingHelper.InRange(ch, EncodingHelper.HighSurrogateStart, EncodingHelper.LowSurrogateEnd))
{
// 3 byte encoding
chd = unchecked((sbyte)0xE0) | (ch >> 12);
}
else
{
// 4 byte encoding - high surrogate + low surrogate
// if (!IsHighSurrogate(ch))
if (ch > EncodingHelper.HighSurrogateEnd)
{
// low without high -> bad
goto InvalidData;
}

chd = *pSrc;

// if (!IsLowSurrogate(chd)) {
if (!EncodingHelper.InRange(chd, EncodingHelper.LowSurrogateStart, EncodingHelper.LowSurrogateEnd))
{
// high not followed by low -> bad
goto InvalidData;
}

pSrc++;

ch = chd + (ch << 10) +
(0x10000
- EncodingHelper.LowSurrogateStart
- (EncodingHelper.HighSurrogateStart << 10));

*pTarget = (byte)(unchecked((sbyte)0xF0) | (ch >> 18));
// pStop - this byte is compensated by the second surrogate character
// 2 input chars require 4 output bytes. 2 have been anticipated already
// and 2 more will be accounted for by the 2 pStop-- calls below.
pTarget++;

chd = unchecked((sbyte)0x80) | (ch >> 12) & 0x3F;
}
*pTarget = (byte)chd;
pStop--; // 3 byte sequence for 1 char, so need pStop-- and the one below too.
pTarget++;

chd = unchecked((sbyte)0x80) | (ch >> 6) & 0x3F;
}
*pTarget = (byte)chd;
pStop--; // 2 byte sequence for 1 char so need pStop--.

*(pTarget + 1) = (byte)(unchecked((sbyte)0x80) | ch & 0x3F);
// pStop - this byte is already included

pTarget += 2;
}
while (pSrc < pStop);

Debug.Assert(pTarget <= pAllocatedBufferEnd, "[UTF8Encoding.GetBytes]pTarget <= pAllocatedBufferEnd");
}

while (pSrc < pEnd)
{
// SLOWLOOP: does all range checks, handles all special cases, but it is slow

// read next char. The JIT optimization seems to be getting confused when
// compiling "ch = *pSrc++;", so rather use "ch = *pSrc; pSrc++;" instead
int ch = *pSrc;
pSrc++;

if (ch <= 0x7F)
{
if (pAllocatedBufferEnd - pTarget <= 0)
goto DestinationFull;

*pTarget = (byte)ch;
pTarget++;
continue;
}

int chd;
if (ch <= 0x7FF)
{
if (pAllocatedBufferEnd - pTarget <= 1)
goto DestinationFull;

// 2 byte encoding
chd = unchecked((sbyte)0xC0) | (ch >> 6);
}
else
{
// if (!IsLowSurrogate(ch) && !IsHighSurrogate(ch))
if (!EncodingHelper.InRange(ch, EncodingHelper.HighSurrogateStart, EncodingHelper.LowSurrogateEnd))
{
if (pAllocatedBufferEnd - pTarget <= 2)
goto DestinationFull;

// 3 byte encoding
chd = unchecked((sbyte)0xE0) | (ch >> 12);
}
else
{
if (pAllocatedBufferEnd - pTarget <= 3)
goto DestinationFull;

// 4 byte encoding - high surrogate + low surrogate
// if (!IsHighSurrogate(ch))
if (ch > EncodingHelper.HighSurrogateEnd)
{
// low without high -> bad
goto InvalidData;
}

if (pSrc >= pEnd)
goto NeedMoreData;

chd = *pSrc;

// if (!IsLowSurrogate(chd)) {
if (!EncodingHelper.InRange(chd, EncodingHelper.LowSurrogateStart, EncodingHelper.LowSurrogateEnd))
{
// high not followed by low -> bad
goto InvalidData;
}

pSrc++;

ch = chd + (ch << 10) +
(0x10000
- EncodingHelper.LowSurrogateStart
- (EncodingHelper.HighSurrogateStart << 10));

*pTarget = (byte)(unchecked((sbyte)0xF0) | (ch >> 18));
pTarget++;

chd = unchecked((sbyte)0x80) | (ch >> 12) & 0x3F;
}
*pTarget = (byte)chd;
pTarget++;

chd = unchecked((sbyte)0x80) | (ch >> 6) & 0x3F;
}

*pTarget = (byte)chd;
*(pTarget + 1) = (byte)(unchecked((sbyte)0x80) | ch & 0x3F);

pTarget += 2;
}

bytesConsumed = (int)((byte*)pSrc - chars);
bytesWritten = (int)(pTarget - bytes);
return TransformationStatus.Done;

InvalidData:
bytesConsumed = (int)((byte*)(pSrc - 1) - chars);
bytesWritten = (int)(pTarget - bytes);
return TransformationStatus.InvalidData;

DestinationFull:
bytesConsumed = (int)((byte*)(pSrc - 1) - chars);
bytesWritten = (int)(pTarget - bytes);
return TransformationStatus.DestinationTooSmall;

NeedMoreData:
bytesConsumed = (int)((byte*)(pSrc - 1) - chars);
bytesWritten = (int)(pTarget - bytes);
return TransformationStatus.NeedMoreSourceData;
}
}

#endregion UTF-8 Conversions

#region UTF-32 Conversions

/// <summary>
/// Calculates the byte count needed to encode the UTF-16 bytes from the specified UTF-32 sequence.
///
/// This method will consume as many of the input bytes as possible.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-32 bytes.</param>
/// <param name="bytesNeeded">On exit, contains the number of bytes required for encoding from the <paramref name="source"/>.</param>
/// <returns>A <see cref="TransformationStatus"/> value representing the expected state of the conversion.</returns>
public static TransformationStatus FromUtf32Length(ReadOnlySpan<byte> source, out int bytesNeeded)
=> Utf32.ToUtf16Length(source, out bytesNeeded);

/// <summary>
/// Converts a span containing a sequence of UTF-32 bytes into UTF-16 bytes.
///
/// This method will consume as many of the input bytes as possible.
///
/// On successful exit, the entire input was consumed and encoded successfully. In this case, <paramref name="bytesConsumed"/> will be
/// equal to the length of the <paramref name="source"/> and <paramref name="bytesWritten"/> will equal the total number of bytes written to
/// the <paramref name="destination"/>.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-32 bytes.</param>
/// <param name="destination">A span to write the UTF-16 bytes into.</param>
/// <param name="bytesConsumed">On exit, contains the number of bytes that were consumed from the <paramref name="source"/>.</param>
/// <param name="bytesWritten">On exit, contains the number of bytes written to <paramref name="destination"/></param>
/// <returns>A <see cref="TransformationStatus"/> value representing the state of the conversion.</returns>
public static TransformationStatus FromUtf32(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
=> Utf32.ToUtf16(source, destination, out bytesConsumed, out bytesWritten);

/// <summary>
/// Calculates the byte count needed to encode the UTF-32 bytes from the specified UTF-16 sequence.
///
/// This method will consume as many of the input bytes as possible.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-16 bytes.</param>
/// <param name="bytesNeeded">On exit, contains the number of bytes required for encoding from the <paramref name="source"/>.</param>
/// <returns>A <see cref="TransformationStatus"/> value representing the expected state of the conversion.</returns>
public static TransformationStatus ToUtf32Length(ReadOnlySpan<byte> source, out int bytesNeeded)
{
bytesNeeded = 0;

ref byte src = ref source.DangerousGetPinnableReference();
int srcLength = source.Length;
int srcIndex = 0;

while (srcLength - srcIndex >= sizeof(char))
{
uint codePoint = Unsafe.As<byte, char>(ref Unsafe.Add(ref src, srcIndex));
if (EncodingHelper.IsSurrogate(codePoint))
{
if (!EncodingHelper.IsHighSurrogate(codePoint))
return TransformationStatus.InvalidData;

if (srcLength - srcIndex < sizeof(char) * 2)
return TransformationStatus.NeedMoreSourceData;

uint lowSurrogate = Unsafe.As<byte, char>(ref Unsafe.Add(ref src, srcIndex + 2));
if (!EncodingHelper.IsLowSurrogate(lowSurrogate))
return TransformationStatus.InvalidData;

srcIndex += 2;
}

srcIndex += 2;
bytesNeeded += 4;
}

return srcIndex < srcLength ? TransformationStatus.NeedMoreSourceData : TransformationStatus.Done;
}

/// <summary>
/// Converts a span containing a sequence of UTF-16 bytes into UTF-32 bytes.
///
/// This method will consume as many of the input bytes as possible.
///
/// On successful exit, the entire input was consumed and encoded successfully. In this case, <paramref name="bytesConsumed"/> will be
/// equal to the length of the <paramref name="source"/> and <paramref name="bytesWritten"/> will equal the total number of bytes written to
/// the <paramref name="destination"/>.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-16 bytes.</param>
/// <param name="destination">A span to write the UTF-32 bytes into.</param>
/// <param name="bytesConsumed">On exit, contains the number of bytes that were consumed from the <paramref name="source"/>.</param>
/// <param name="bytesWritten">On exit, contains the number of bytes written to <paramref name="destination"/></param>
/// <returns>A <see cref="TransformationStatus"/> value representing the state of the conversion.</returns>
public static TransformationStatus ToUtf32(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
{
bytesConsumed = 0;
bytesWritten = 0;

ref byte src = ref source.DangerousGetPinnableReference();
int srcLength = source.Length;

ref byte dst = ref destination.DangerousGetPinnableReference();
int dstLength = destination.Length;

while (srcLength - bytesConsumed >= sizeof(char))
{
if (dstLength - bytesWritten < sizeof(uint))
return TransformationStatus.DestinationTooSmall;

uint codePoint = Unsafe.As<byte, char>(ref Unsafe.Add(ref src, bytesConsumed));
if (EncodingHelper.IsSurrogate(codePoint))
{
if (!EncodingHelper.IsHighSurrogate(codePoint))
return TransformationStatus.InvalidData;

if (srcLength - bytesConsumed < sizeof(char) * 2)
return TransformationStatus.NeedMoreSourceData;

uint lowSurrogate = Unsafe.As<byte, char>(ref Unsafe.Add(ref src, bytesConsumed + 2));
if (!EncodingHelper.IsLowSurrogate(lowSurrogate))
return TransformationStatus.InvalidData;

codePoint -= EncodingHelper.HighSurrogateStart;
lowSurrogate -= EncodingHelper.LowSurrogateStart;
codePoint = ((codePoint << 10) | lowSurrogate) + 0x010000u;
bytesConsumed += 2;
}

Unsafe.As<byte, uint>(ref Unsafe.Add(ref dst, bytesWritten)) = codePoint;
bytesConsumed += 2;
bytesWritten += 4;
}

return bytesConsumed < srcLength ? TransformationStatus.NeedMoreSourceData : TransformationStatus.Done;
}

#endregion UTF-32 Conversions
}
}

+ 258
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/Encoders.Utf32.cs View File

@@ -0,0 +1,258 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Buffers;
using System.Runtime.CompilerServices;

namespace System.Text.Encoders
{
public static class Utf32
{
#region UTF-8 Conversions

/// <summary>
/// Calculates the byte count needed to encode the UTF-32 bytes from the specified UTF-8 sequence.
///
/// This method will consume as many of the input bytes as possible.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-8 bytes.</param>
/// <param name="bytesNeeded">On exit, contains the number of bytes required for encoding from the <paramref name="source"/>.</param>
/// <returns>A <see cref="TransformationStatus"/> value representing the expected state of the conversion.</returns>
public static TransformationStatus FromUtf8Length(ReadOnlySpan<byte> source, out int bytesNeeded)
=> Utf8.ToUtf32Length(source, out bytesNeeded);

/// <summary>
/// Converts a span containing a sequence of UTF-8 bytes into UTF-32 bytes.
///
/// This method will consume as many of the input bytes as possible.
///
/// On successful exit, the entire input was consumed and encoded successfully. In this case, <paramref name="bytesConsumed"/> will be
/// equal to the length of the <paramref name="source"/> and <paramref name="bytesWritten"/> will equal the total number of bytes written to
/// the <paramref name="destination"/>.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-8 bytes.</param>
/// <param name="destination">A span to write the UTF-32 bytes into.</param>
/// <param name="bytesConsumed">On exit, contains the number of bytes that were consumed from the <paramref name="source"/>.</param>
/// <param name="bytesWritten">On exit, contains the number of bytes written to <paramref name="destination"/></param>
/// <returns>A <see cref="TransformationStatus"/> value representing the state of the conversion.</returns>
public static TransformationStatus FromUtf8(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
=> Utf8.ToUtf32(source, destination, out bytesConsumed, out bytesWritten);

/// <summary>
/// Calculates the byte count needed to encode the UTF-8 bytes from the specified UTF-32 sequence.
///
/// This method will consume as many of the input bytes as possible.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-32 bytes.</param>
/// <param name="bytesNeeded">On exit, contains the number of bytes required for encoding from the <paramref name="source"/>.</param>
/// <returns>A <see cref="TransformationStatus"/> value representing the expected state of the conversion.</returns>
public static TransformationStatus ToUtf8Length(ReadOnlySpan<byte> source, out int bytesNeeded)
{
bytesNeeded = 0;

ref uint utf32 = ref Unsafe.As<byte, uint>(ref source.DangerousGetPinnableReference());
int utf32Length = source.Length >> 2; // byte => uint count

for (int i = 0; i < utf32Length; i++)
{
uint codePoint = Unsafe.Add(ref utf32, i);
if (!EncodingHelper.IsSupportedCodePoint(codePoint))
return TransformationStatus.InvalidData;

bytesNeeded += EncodingHelper.GetUtf8EncodedBytes(codePoint);
}

if (utf32Length << 2 != source.Length)
return TransformationStatus.NeedMoreSourceData;

return TransformationStatus.Done;
}

/// <summary>
/// Converts a span containing a sequence of UTF-32 bytes into UTF-8 bytes.
///
/// This method will consume as many of the input bytes as possible.
///
/// On successful exit, the entire input was consumed and encoded successfully. In this case, <paramref name="bytesConsumed"/> will be
/// equal to the length of the <paramref name="source"/> and <paramref name="bytesWritten"/> will equal the total number of bytes written to
/// the <paramref name="destination"/>.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-32 bytes.</param>
/// <param name="destination">A span to write the UTF-8 bytes into.</param>
/// <param name="bytesConsumed">On exit, contains the number of bytes that were consumed from the <paramref name="source"/>.</param>
/// <param name="bytesWritten">On exit, contains the number of bytes written to <paramref name="destination"/></param>
/// <returns>A <see cref="TransformationStatus"/> value representing the state of the conversion.</returns>
public static TransformationStatus ToUtf8(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
{
bytesConsumed = 0;
bytesWritten = 0;

ref byte src = ref source.DangerousGetPinnableReference();
int srcLength = source.Length;

ref byte dst = ref destination.DangerousGetPinnableReference();
int dstLength = destination.Length;

while (srcLength - bytesConsumed >= sizeof(uint))
{
uint codePoint = Unsafe.As<byte, uint>(ref Unsafe.Add(ref src, bytesConsumed));
if (!EncodingHelper.IsSupportedCodePoint(codePoint))
return TransformationStatus.InvalidData;

int bytesNeeded = EncodingHelper.GetUtf8EncodedBytes(codePoint);
if (dstLength - bytesWritten < bytesNeeded)
return TransformationStatus.DestinationTooSmall;

switch (bytesNeeded)
{
case 1:
Unsafe.Add(ref dst, bytesWritten) = (byte)(EncodingHelper.b0111_1111U & codePoint);
break;

case 2:
Unsafe.Add(ref dst, bytesWritten) = (byte)(((codePoint >> 6) & EncodingHelper.b0001_1111U) | EncodingHelper.b1100_0000U);
Unsafe.Add(ref dst, bytesWritten + 1) = (byte)((codePoint & EncodingHelper.b0011_1111U) | EncodingHelper.b1000_0000U);
break;

case 3:
Unsafe.Add(ref dst, bytesWritten) = (byte)(((codePoint >> 12) & EncodingHelper.b0000_1111U) | EncodingHelper.b1110_0000U);
Unsafe.Add(ref dst, bytesWritten + 1) = (byte)(((codePoint >> 6) & EncodingHelper.b0011_1111U) | EncodingHelper.b1000_0000U);
Unsafe.Add(ref dst, bytesWritten + 2) = (byte)((codePoint & EncodingHelper.b0011_1111U) | EncodingHelper.b1000_0000U);
break;

case 4:
Unsafe.Add(ref dst, bytesWritten) = (byte)(((codePoint >> 18) & EncodingHelper.b0000_0111U) | EncodingHelper.b1111_0000U);
Unsafe.Add(ref dst, bytesWritten + 1) = (byte)(((codePoint >> 12) & EncodingHelper.b0011_1111U) | EncodingHelper.b1000_0000U);
Unsafe.Add(ref dst, bytesWritten + 2) = (byte)(((codePoint >> 6) & EncodingHelper.b0011_1111U) | EncodingHelper.b1000_0000U);
Unsafe.Add(ref dst, bytesWritten + 3) = (byte)((codePoint & EncodingHelper.b0011_1111U) | EncodingHelper.b1000_0000U);
break;

default:
return TransformationStatus.InvalidData;
}

bytesConsumed += 4;
bytesWritten += bytesNeeded;
}

return bytesConsumed < srcLength ? TransformationStatus.NeedMoreSourceData : TransformationStatus.Done;
}

#endregion UTF-8 Conversions

#region UTF-16 Conversions

/// <summary>
/// Calculates the byte count needed to encode the UTF-32 bytes from the specified UTF-16 sequence.
///
/// This method will consume as many of the input bytes as possible.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-16 bytes.</param>
/// <param name="bytesNeeded">On exit, contains the number of bytes required for encoding from the <paramref name="source"/>.</param>
/// <returns>A <see cref="TransformationStatus"/> value representing the expected state of the conversion.</returns>
public static TransformationStatus FromUtf16Length(ReadOnlySpan<byte> source, out int bytesNeeded)
=> Utf16.ToUtf32Length(source, out bytesNeeded);

/// <summary>
/// Converts a span containing a sequence of UTF-16 bytes into UTF-32 bytes.
///
/// This method will consume as many of the input bytes as possible.
///
/// On successful exit, the entire input was consumed and encoded successfully. In this case, <paramref name="bytesConsumed"/> will be
/// equal to the length of the <paramref name="source"/> and <paramref name="bytesWritten"/> will equal the total number of bytes written to
/// the <paramref name="destination"/>.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-16 bytes.</param>
/// <param name="destination">A span to write the UTF-32 bytes into.</param>
/// <param name="bytesConsumed">On exit, contains the number of bytes that were consumed from the <paramref name="source"/>.</param>
/// <param name="bytesWritten">On exit, contains the number of bytes written to <paramref name="destination"/></param>
/// <returns>A <see cref="TransformationStatus"/> value representing the state of the conversion.</returns>
public static TransformationStatus FromUtf16(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
=> Utf16.ToUtf32(source, destination, out bytesConsumed, out bytesWritten);

/// <summary>
/// Calculates the byte count needed to encode the UTF-16 bytes from the specified UTF-32 sequence.
///
/// This method will consume as many of the input bytes as possible.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-32 bytes.</param>
/// <param name="bytesNeeded">On exit, contains the number of bytes required for encoding from the <paramref name="source"/>.</param>
/// <returns>A <see cref="TransformationStatus"/> value representing the expected state of the conversion.</returns>
public static TransformationStatus ToUtf16Length(ReadOnlySpan<byte> source, out int bytesNeeded)
{
int index = 0;
int length = source.Length;
ref byte src = ref source.DangerousGetPinnableReference();

bytesNeeded = 0;

while (length - index >= 4)
{
ref uint codePoint = ref Unsafe.As<byte, uint>(ref Unsafe.Add(ref src, index));

if (!EncodingHelper.IsSupportedCodePoint(codePoint))
return TransformationStatus.InvalidData;

bytesNeeded += EncodingHelper.IsBmp(codePoint) ? 2 : 4;
index += 4;
}

return index < length ? TransformationStatus.NeedMoreSourceData : TransformationStatus.Done;
}

/// <summary>
/// Converts a span containing a sequence of UTF-32 bytes into UTF-16 bytes.
///
/// This method will consume as many of the input bytes as possible.
///
/// On successful exit, the entire input was consumed and encoded successfully. In this case, <paramref name="bytesConsumed"/> will be
/// equal to the length of the <paramref name="source"/> and <paramref name="bytesWritten"/> will equal the total number of bytes written to
/// the <paramref name="destination"/>.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-32 bytes.</param>
/// <param name="destination">A span to write the UTF-16 bytes into.</param>
/// <param name="bytesConsumed">On exit, contains the number of bytes that were consumed from the <paramref name="source"/>.</param>
/// <param name="bytesWritten">On exit, contains the number of bytes written to <paramref name="destination"/></param>
/// <returns>A <see cref="TransformationStatus"/> value representing the state of the conversion.</returns>
public static TransformationStatus ToUtf16(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
{
ref byte src = ref source.DangerousGetPinnableReference();
ref byte dst = ref destination.DangerousGetPinnableReference();
int srcLength = source.Length;
int dstLength = destination.Length;

bytesConsumed = 0;
bytesWritten = 0;

while (srcLength - bytesConsumed >= sizeof(uint))
{
ref uint codePoint = ref Unsafe.As<byte, uint>(ref Unsafe.Add(ref src, bytesConsumed));

if (!EncodingHelper.IsSupportedCodePoint(codePoint))
return TransformationStatus.InvalidData;

int written = EncodingHelper.IsBmp(codePoint) ? 2 : 4;
if (dstLength - bytesWritten < written)
return TransformationStatus.DestinationTooSmall;

unchecked
{
if (written == 2)
Unsafe.As<byte, char>(ref Unsafe.Add(ref dst, bytesWritten)) = (char)codePoint;
else
{
Unsafe.As<byte, char>(ref Unsafe.Add(ref dst, bytesWritten)) = (char)(((codePoint - 0x010000u) >> 10) + EncodingHelper.HighSurrogateStart);
Unsafe.As<byte, char>(ref Unsafe.Add(ref dst, bytesWritten + 2)) = (char)((codePoint & 0x3FF) + EncodingHelper.LowSurrogateStart);
}
}

bytesWritten += written;
bytesConsumed += 4;
}

return bytesConsumed < srcLength ? TransformationStatus.NeedMoreSourceData : TransformationStatus.Done;
}

#endregion UTF-16 Conversions
}
}

+ 838
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/Encoders.Utf8.cs View File

@@ -0,0 +1,838 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Buffers;
using System.Diagnostics;
using System.Runtime.CompilerServices;

namespace System.Text.Encoders
{
public static class Utf8
{
#region UTF-16 Conversions

/// <summary>
/// Calculates the byte count needed to encode the UTF-8 bytes from the specified UTF-16 sequence.
///
/// This method will consume as many of the input bytes as possible.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-16 bytes.</param>
/// <param name="bytesNeeded">On exit, contains the number of bytes required for encoding from the <paramref name="source"/>.</param>
/// <returns>A <see cref="TransformationStatus"/> value representing the expected state of the conversion.</returns>
public static TransformationStatus FromUtf16Length(ReadOnlySpan<byte> source, out int bytesNeeded)
=> Utf16.ToUtf8Length(source, out bytesNeeded);

/// <summary>
/// Converts a span containing a sequence of UTF-16 bytes into UTF-8 bytes.
///
/// This method will consume as many of the input bytes as possible.
///
/// On successful exit, the entire input was consumed and encoded successfully. In this case, <paramref name="bytesConsumed"/> will be
/// equal to the length of the <paramref name="source"/> and <paramref name="bytesWritten"/> will equal the total number of bytes written to
/// the <paramref name="destination"/>.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-16 bytes.</param>
/// <param name="destination">A span to write the UTF-8 bytes into.</param>
/// <param name="bytesConsumed">On exit, contains the number of bytes that were consumed from the <paramref name="source"/>.</param>
/// <param name="bytesWritten">On exit, contains the number of bytes written to <paramref name="destination"/></param>
/// <returns>A <see cref="TransformationStatus"/> value representing the state of the conversion.</returns>
public static TransformationStatus FromUtf16(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
=> Utf16.ToUtf8(source, destination, out bytesConsumed, out bytesWritten);

/// <summary>
/// Calculates the byte count needed to encode the UTF-16 bytes from the specified UTF-8 sequence.
///
/// This method will consume as many of the input bytes as possible.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-8 bytes.</param>
/// <param name="bytesNeeded">On exit, contains the number of bytes required for encoding from the <paramref name="source"/>.</param>
/// <returns>A <see cref="TransformationStatus"/> value representing the expected state of the conversion.</returns>
public unsafe static TransformationStatus ToUtf16Length(ReadOnlySpan<byte> source, out int bytesNeeded)
{
fixed (byte* pUtf8 = &source.DangerousGetPinnableReference())
{
byte* pSrc = pUtf8;
byte* pSrcEnd = pSrc + source.Length;

bytesNeeded = 0;

int ch = 0;
while (pSrc < pSrcEnd)
{
int availableBytes = EncodingHelper.PtrDiff(pSrcEnd, pSrc);

// don't fall into the fast decoding loop if we don't have enough bytes
if (availableBytes <= 13)
{
// try to get over the remainder of the ascii characters fast though
byte* pLocalEnd = pSrc + availableBytes;
while (pSrc < pLocalEnd)
{
ch = *pSrc;
pSrc++;

if (ch > 0x7F)
goto LongCodeSlow;

bytesNeeded++;
}

// we are done
break;
}

// To compute the upper bound, assume that all characters are ASCII characters at this point,
// the boundary will be decreased for every non-ASCII character we encounter
// Also, we need 7 chars reserve for the unrolled ansi decoding loop and for decoding of multibyte sequences
byte* pStop = pSrc + availableBytes - 7;

// Fast loop
while (pSrc < pStop)
{
ch = *pSrc;
pSrc++;

if (ch > 0x7F)
goto LongCode;

bytesNeeded++;

// 2-byte align
if ((unchecked((int)pSrc) & 0x1) != 0)
{
ch = *pSrc;
pSrc++;

if (ch > 0x7F)
goto LongCode;

bytesNeeded++;
}

// 4-byte align
if ((unchecked((int)pSrc) & 0x2) != 0)
{
ch = *(ushort*)pSrc;
if ((ch & 0x8080) != 0)
goto LongCodeWithMask16;
pSrc += 2;
bytesNeeded += 2;
}

// Run 8 characters at a time!
while (pSrc < pStop)
{
ch = *(int*)pSrc;
int chb = *(int*)(pSrc + 4);
if (((ch | chb) & unchecked((int)0x80808080)) != 0)
goto LongCodeWithMask32;
pSrc += 8;
bytesNeeded += 8;
}

break;

#if BIGENDIAN
LongCodeWithMask32:
// be careful about the sign extension
ch = (int)(((uint)ch) >> 16);
LongCodeWithMask16:
ch = (int)(((uint)ch) >> 8);
#else // BIGENDIAN
LongCodeWithMask32:
LongCodeWithMask16:
ch &= 0xFF;
#endif // BIGENDIAN
pSrc++;
if (ch <= 0x7F)
{
bytesNeeded++;
continue;
}

LongCode:
int chc = *pSrc;
pSrc++;

// Bit 6 should be 0, and trailing byte should be 10vvvvvv
if ((ch & 0x40) == 0 || (chc & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

chc &= 0x3F;

if ((ch & 0x20) != 0)
{
// Handle 3 or 4 byte encoding.

// Fold the first 2 bytes together
chc |= (ch & 0x0F) << 6;

if ((ch & 0x10) != 0)
{
// 4 byte - surrogate pair
ch = *pSrc;

// Bit 4 should be zero + the surrogate should be in the range 0x000000 - 0x10FFFF
// and the trailing byte should be 10vvvvvv
if (!EncodingHelper.InRange(chc >> 4, 0x01, 0x10) || (ch & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

// Merge 3rd byte then read the last byte
chc = (chc << 6) | (ch & 0x3F);
ch = *(pSrc + 1);

// The last trailing byte still holds the form 10vvvvvv
if ((ch & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

pSrc += 2;
ch = (chc << 6) | (ch & 0x3F);

bytesNeeded++;

ch = (ch & 0x3FF) + unchecked((short)(EncodingHelper.LowSurrogateStart));
}
else
{
// 3 byte encoding
ch = *pSrc;

// Check for non-shortest form of 3 byte sequence
// No surrogates
// Trailing byte must be in the form 10vvvvvv
if ((chc & (0x1F << 5)) == 0 ||
(chc & (0xF800 >> 6)) == (0xD800 >> 6) ||
(ch & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

pSrc++;
ch = (chc << 6) | (ch & 0x3F);
}

// extra byte, we're already planning 2 chars for 2 of these bytes,
// but the big loop is testing the target against pStop, so we need
// to subtract 2 more or we risk overrunning the input. Subtract
// one here and one below.
pStop--;
}
else
{
// 2 byte encoding
ch &= 0x1F;

// Check for non-shortest form
if (ch <= 1)
goto InvalidData;

ch = (ch << 6) | chc;
}

bytesNeeded++;

// extra byte, we're only expecting 1 char for each of these 2 bytes,
// but the loop is testing the target (not source) against pStop.
// subtract an extra count from pStop so that we don't overrun the input.
pStop--;
}

continue;

LongCodeSlow:
if (pSrc >= pSrcEnd)
{
// This is a special case where hit the end of the buffer but are in the middle
// of decoding a long code. The error exit thinks we have read 2 extra bytes already,
// so we add +1 to pSrc to get the count correct for the bytes consumed value.
pSrc++;
goto NeedMoreData;
}

int chd = *pSrc;
pSrc++;

// Bit 6 should be 0, and trailing byte should be 10vvvvvv
if ((ch & 0x40) == 0 || (chd & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

chd &= 0x3F;

if ((ch & 0x20) != 0)
{
// Handle 3 or 4 byte encoding.

// Fold the first 2 bytes together
chd |= (ch & 0x0F) << 6;

if ((ch & 0x10) != 0)
{
// 4 byte - surrogate pair
// We need 2 more bytes
if (pSrc >= pSrcEnd - 1)
goto NeedMoreData;

ch = *pSrc;

// Bit 4 should be zero + the surrogate should be in the range 0x000000 - 0x10FFFF
// and the trailing byte should be 10vvvvvv
if (!EncodingHelper.InRange(chd >> 4, 0x01, 0x10) || (ch & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

// Merge 3rd byte then read the last byte
chd = (chd << 6) | (ch & 0x3F);
ch = *(pSrc + 1);

// The last trailing byte still holds the form 10vvvvvv
// We only know for sure we have room for one more char, but we need an extra now.
if ((ch & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

pSrc += 2;
ch = (chd << 6) | (ch & 0x3F);

bytesNeeded++;

ch = (ch & 0x3FF) + unchecked((short)(EncodingHelper.LowSurrogateStart));
}
else
{
// 3 byte encoding
if (pSrc >= pSrcEnd)
goto NeedMoreData;

ch = *pSrc;

// Check for non-shortest form of 3 byte sequence
// No surrogates
// Trailing byte must be in the form 10vvvvvv
if ((chd & (0x1F << 5)) == 0 ||
(chd & (0xF800 >> 6)) == (0xD800 >> 6) ||
(ch & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

pSrc++;
ch = (chd << 6) | (ch & 0x3F);
}
}
else
{
// 2 byte encoding
ch &= 0x1F;

// Check for non-shortest form
if (ch <= 1)
goto InvalidData;

ch = (ch << 6) | chd;
}

bytesNeeded++;
}

bytesNeeded <<= 1; // Count we have is chars, double for bytes.
return EncodingHelper.PtrDiff(pSrcEnd, pSrc) == 0 ? TransformationStatus.Done : TransformationStatus.DestinationTooSmall;

NeedMoreData:
bytesNeeded <<= 1; // Count we have is chars, double for bytes.
return TransformationStatus.NeedMoreSourceData;

InvalidData:
bytesNeeded <<= 1; // Count we have is chars, double for bytes.
return TransformationStatus.InvalidData;
}
}

/// <summary>
/// Converts a span containing a sequence of UTF-8 bytes into UTF-16 bytes.
///
/// This method will consume as many of the input bytes as possible.
///
/// On successful exit, the entire input was consumed and encoded successfully. In this case, <paramref name="bytesConsumed"/> will be
/// equal to the length of the <paramref name="source"/> and <paramref name="bytesWritten"/> will equal the total number of bytes written to
/// the <paramref name="destination"/>.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-8 bytes.</param>
/// <param name="destination">A span to write the UTF-16 bytes into.</param>
/// <param name="bytesConsumed">On exit, contains the number of bytes that were consumed from the <paramref name="source"/>.</param>
/// <param name="bytesWritten">On exit, contains the number of bytes written to <paramref name="destination"/></param>
/// <returns>A <see cref="TransformationStatus"/> value representing the state of the conversion.</returns>
public unsafe static TransformationStatus ToUtf16(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
{
fixed (byte* pUtf8 = &source.DangerousGetPinnableReference())
fixed (byte* pUtf16 = &destination.DangerousGetPinnableReference())
{
byte* pSrc = pUtf8;
byte* pSrcEnd = pSrc + source.Length;
char* pDst = (char*)pUtf16;
char* pDstEnd = pDst + (destination.Length >> 1); // Conversion from bytes to chars - div by sizeof(char)

int ch = 0;
while (pSrc < pSrcEnd && pDst < pDstEnd)
{
// we may need as many as 1 character per byte, so reduce the byte count if necessary.
// If availableChars is too small, pStop will be before pTarget and we won't do fast loop.
int availableChars = EncodingHelper.PtrDiff(pDstEnd, pDst);
int availableBytes = EncodingHelper.PtrDiff(pSrcEnd, pSrc);

if (availableChars < availableBytes)
availableBytes = availableChars;

// don't fall into the fast decoding loop if we don't have enough bytes
if (availableBytes <= 13)
{
// try to get over the remainder of the ascii characters fast though
byte* pLocalEnd = pSrc + availableBytes;
while (pSrc < pLocalEnd)
{
ch = *pSrc;
pSrc++;

if (ch > 0x7F)
goto LongCodeSlow;

*pDst = (char)ch;
pDst++;
}

// we are done
break;
}

// To compute the upper bound, assume that all characters are ASCII characters at this point,
// the boundary will be decreased for every non-ASCII character we encounter
// Also, we need 7 chars reserve for the unrolled ansi decoding loop and for decoding of multibyte sequences
char* pStop = pDst + availableBytes - 7;

// Fast loop
while (pDst < pStop)
{
ch = *pSrc;
pSrc++;

if (ch > 0x7F)
goto LongCode;

*pDst = (char)ch;
pDst++;

// 2-byte align
if ((unchecked((int)pSrc) & 0x1) != 0)
{
ch = *pSrc;
pSrc++;

if (ch > 0x7F)
goto LongCode;

*pDst = (char)ch;
pDst++;
}

// 4-byte align
if ((unchecked((int)pSrc) & 0x2) != 0)
{
ch = *(ushort*)pSrc;
if ((ch & 0x8080) != 0)
goto LongCodeWithMask16;

// Unfortunately, endianness sensitive
#if BIGENDIAN
*pDst = (char)((ch >> 8) & 0x7F);
pSrc += 2;
*(pDst + 1) = (char)(ch & 0x7F);
pDst += 2;
#else // BIGENDIAN
*pDst = (char)(ch & 0x7F);
pSrc += 2;
*(pDst + 1) = (char)((ch >> 8) & 0x7F);
pDst += 2;
#endif // BIGENDIAN
}

// Run 8 characters at a time!
while (pDst < pStop)
{
ch = *(int*)pSrc;
int chb = *(int*)(pSrc + 4);
if (((ch | chb) & unchecked((int)0x80808080)) != 0)
goto LongCodeWithMask32;

// Unfortunately, endianness sensitive
#if BIGENDIAN
*pDst = (char)((ch >> 24) & 0x7F);
*(pDst+1) = (char)((ch >> 16) & 0x7F);
*(pDst+2) = (char)((ch >> 8) & 0x7F);
*(pDst+3) = (char)(ch & 0x7F);
pSrc += 8;
*(pDst+4) = (char)((chb >> 24) & 0x7F);
*(pDst+5) = (char)((chb >> 16) & 0x7F);
*(pDst+6) = (char)((chb >> 8) & 0x7F);
*(pDst+7) = (char)(chb & 0x7F);
pDst += 8;
#else // BIGENDIAN
*pDst = (char)(ch & 0x7F);
*(pDst + 1) = (char)((ch >> 8) & 0x7F);
*(pDst + 2) = (char)((ch >> 16) & 0x7F);
*(pDst + 3) = (char)((ch >> 24) & 0x7F);
pSrc += 8;
*(pDst + 4) = (char)(chb & 0x7F);
*(pDst + 5) = (char)((chb >> 8) & 0x7F);
*(pDst + 6) = (char)((chb >> 16) & 0x7F);
*(pDst + 7) = (char)((chb >> 24) & 0x7F);
pDst += 8;
#endif // BIGENDIAN
}

break;

#if BIGENDIAN
LongCodeWithMask32:
// be careful about the sign extension
ch = (int)(((uint)ch) >> 16);
LongCodeWithMask16:
ch = (int)(((uint)ch) >> 8);
#else // BIGENDIAN
LongCodeWithMask32:
LongCodeWithMask16:
ch &= 0xFF;
#endif // BIGENDIAN
pSrc++;
if (ch <= 0x7F)
{
*pDst = (char)ch;
pDst++;
continue;
}

LongCode:
int chc = *pSrc;
pSrc++;

// Bit 6 should be 0, and trailing byte should be 10vvvvvv
if ((ch & 0x40) == 0 || (chc & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

chc &= 0x3F;

if ((ch & 0x20) != 0)
{
// Handle 3 or 4 byte encoding.

// Fold the first 2 bytes together
chc |= (ch & 0x0F) << 6;

if ((ch & 0x10) != 0)
{
// 4 byte - surrogate pair
ch = *pSrc;

// Bit 4 should be zero + the surrogate should be in the range 0x000000 - 0x10FFFF
// and the trailing byte should be 10vvvvvv
if (!EncodingHelper.InRange(chc >> 4, 0x01, 0x10) || (ch & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

// Merge 3rd byte then read the last byte
chc = (chc << 6) | (ch & 0x3F);
ch = *(pSrc + 1);

// The last trailing byte still holds the form 10vvvvvv
if ((ch & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

pSrc += 2;
ch = (chc << 6) | (ch & 0x3F);

*pDst = (char)(((ch >> 10) & 0x7FF) + unchecked((short)(EncodingHelper.HighSurrogateStart - (0x10000 >> 10))));
pDst++;

ch = (ch & 0x3FF) + unchecked((short)(EncodingHelper.LowSurrogateStart));
}
else
{
// 3 byte encoding
ch = *pSrc;

// Check for non-shortest form of 3 byte sequence
// No surrogates
// Trailing byte must be in the form 10vvvvvv
if ((chc & (0x1F << 5)) == 0 ||
(chc & (0xF800 >> 6)) == (0xD800 >> 6) ||
(ch & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

pSrc++;
ch = (chc << 6) | (ch & 0x3F);
}

// extra byte, we're already planning 2 chars for 2 of these bytes,
// but the big loop is testing the target against pStop, so we need
// to subtract 2 more or we risk overrunning the input. Subtract
// one here and one below.
pStop--;
}
else
{
// 2 byte encoding
ch &= 0x1F;

// Check for non-shortest form
if (ch <= 1)
goto InvalidData;

ch = (ch << 6) | chc;
}

*pDst = (char)ch;
pDst++;

// extra byte, we're only expecting 1 char for each of these 2 bytes,
// but the loop is testing the target (not source) against pStop.
// subtract an extra count from pStop so that we don't overrun the input.
pStop--;
}

continue;

LongCodeSlow:
if (pSrc >= pSrcEnd)
{
// This is a special case where hit the end of the buffer but are in the middle
// of decoding a long code. The error exit thinks we have read 2 extra bytes already,
// so we add +1 to pSrc to get the count correct for the bytes consumed value.
pSrc++;
goto NeedMoreData;
}

int chd = *pSrc;
pSrc++;

// Bit 6 should be 0, and trailing byte should be 10vvvvvv
if ((ch & 0x40) == 0 || (chd & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

chd &= 0x3F;

if ((ch & 0x20) != 0)
{
// Handle 3 or 4 byte encoding.

// Fold the first 2 bytes together
chd |= (ch & 0x0F) << 6;

if ((ch & 0x10) != 0)
{
// 4 byte - surrogate pair
// We need 2 more bytes
if (pSrc >= pSrcEnd - 1)
goto NeedMoreData;

ch = *pSrc;

// Bit 4 should be zero + the surrogate should be in the range 0x000000 - 0x10FFFF
// and the trailing byte should be 10vvvvvv
if (!EncodingHelper.InRange(chd >> 4, 0x01, 0x10) || (ch & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

// Merge 3rd byte then read the last byte
chd = (chd << 6) | (ch & 0x3F);
ch = *(pSrc + 1);

// The last trailing byte still holds the form 10vvvvvv
// We only know for sure we have room for one more char, but we need an extra now.
if ((ch & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

if (EncodingHelper.PtrDiff(pDstEnd, pDst) < 2)
goto DestinationFull;

pSrc += 2;
ch = (chd << 6) | (ch & 0x3F);

*pDst = (char)(((ch >> 10) & 0x7FF) + unchecked((short)(EncodingHelper.HighSurrogateStart - (0x10000 >> 10))));
pDst++;

ch = (ch & 0x3FF) + unchecked((short)(EncodingHelper.LowSurrogateStart));
}
else
{
// 3 byte encoding
if (pSrc >= pSrcEnd)
goto NeedMoreData;

ch = *pSrc;

// Check for non-shortest form of 3 byte sequence
// No surrogates
// Trailing byte must be in the form 10vvvvvv
if ((chd & (0x1F << 5)) == 0 ||
(chd & (0xF800 >> 6)) == (0xD800 >> 6) ||
(ch & unchecked((sbyte)0xC0)) != 0x80)
goto InvalidData;

pSrc++;
ch = (chd << 6) | (ch & 0x3F);
}
}
else
{
// 2 byte encoding
ch &= 0x1F;

// Check for non-shortest form
if (ch <= 1)
goto InvalidData;

ch = (ch << 6) | chd;
}

*pDst = (char)ch;
pDst++;
}

DestinationFull:
bytesConsumed = EncodingHelper.PtrDiff(pSrc, pUtf8);
bytesWritten = EncodingHelper.PtrDiff((byte*)pDst, pUtf16);
return EncodingHelper.PtrDiff(pSrcEnd, pSrc) == 0 ? TransformationStatus.Done : TransformationStatus.DestinationTooSmall;

NeedMoreData:
bytesConsumed = EncodingHelper.PtrDiff(pSrc - 2, pUtf8);
bytesWritten = EncodingHelper.PtrDiff((byte*)pDst, pUtf16);
return TransformationStatus.NeedMoreSourceData;

InvalidData:
bytesConsumed = EncodingHelper.PtrDiff(pSrc - 2, pUtf8);
bytesWritten = EncodingHelper.PtrDiff((byte*)pDst, pUtf16);
return TransformationStatus.InvalidData;
}
}

#endregion UTF-16 Conversions

#region UTF-32 Conversions

/// <summary>
/// Calculates the byte count needed to encode the UTF-8 bytes from the specified UTF-32 sequence.
///
/// This method will consume as many of the input bytes as possible.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-32 bytes.</param>
/// <param name="bytesNeeded">On exit, contains the number of bytes required for encoding from the <paramref name="source"/>.</param>
/// <returns>A <see cref="TransformationStatus"/> value representing the expected state of the conversion.</returns>
public static TransformationStatus FromUtf32Length(ReadOnlySpan<byte> source, out int bytesNeeded)
=> Utf32.ToUtf8Length(source, out bytesNeeded);

/// <summary>
/// Converts a span containing a sequence of UTF-32 bytes into UTF-8 bytes.
///
/// This method will consume as many of the input bytes as possible.
///
/// On successful exit, the entire input was consumed and encoded successfully. In this case, <paramref name="bytesConsumed"/> will be
/// equal to the length of the <paramref name="source"/> and <paramref name="bytesWritten"/> will equal the total number of bytes written to
/// the <paramref name="destination"/>.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-32 bytes.</param>
/// <param name="destination">A span to write the UTF-8 bytes into.</param>
/// <param name="bytesConsumed">On exit, contains the number of bytes that were consumed from the <paramref name="source"/>.</param>
/// <param name="bytesWritten">On exit, contains the number of bytes written to <paramref name="destination"/></param>
/// <returns>A <see cref="TransformationStatus"/> value representing the state of the conversion.</returns>
public static TransformationStatus FromUtf32(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
=> Utf32.ToUtf8(source, destination, out bytesConsumed, out bytesWritten);

/// <summary>
/// Calculates the byte count needed to encode the UTF-32 bytes from the specified UTF-8 sequence.
///
/// This method will consume as many of the input bytes as possible.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-8 bytes.</param>
/// <param name="bytesNeeded">On exit, contains the number of bytes required for encoding from the <paramref name="source"/>.</param>
/// <returns>A <see cref="TransformationStatus"/> value representing the expected state of the conversion.</returns>
public static TransformationStatus ToUtf32Length(ReadOnlySpan<byte> source, out int bytesNeeded)
{
bytesNeeded = 0;

int index = 0;
int length = source.Length;
ref byte src = ref source.DangerousGetPinnableReference();

while (index < length)
{
int count = EncodingHelper.GetUtf8DecodedBytes(Unsafe.Add(ref src, index));
if (count == 0)
goto InvalidData;
if (length - index < count)
goto NeedMoreData;

bytesNeeded += count;
}

return index < length ? TransformationStatus.DestinationTooSmall : TransformationStatus.Done;

InvalidData:
return TransformationStatus.InvalidData;

NeedMoreData:
return TransformationStatus.NeedMoreSourceData;
}

/// <summary>
/// Converts a span containing a sequence of UTF-8 bytes into UTF-32 bytes.
///
/// This method will consume as many of the input bytes as possible.
///
/// On successful exit, the entire input was consumed and encoded successfully. In this case, <paramref name="bytesConsumed"/> will be
/// equal to the length of the <paramref name="source"/> and <paramref name="bytesWritten"/> will equal the total number of bytes written to
/// the <paramref name="destination"/>.
/// </summary>
/// <param name="source">A span containing a sequence of UTF-8 bytes.</param>
/// <param name="destination">A span to write the UTF-32 bytes into.</param>
/// <param name="bytesConsumed">On exit, contains the number of bytes that were consumed from the <paramref name="source"/>.</param>
/// <param name="bytesWritten">On exit, contains the number of bytes written to <paramref name="destination"/></param>
/// <returns>A <see cref="TransformationStatus"/> value representing the state of the conversion.</returns>
public static TransformationStatus ToUtf32(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
{
bytesConsumed = 0;
bytesWritten = 0;

int srcLength = source.Length;
int dstLength = destination.Length;
ref byte src = ref source.DangerousGetPinnableReference();
ref byte dst = ref destination.DangerousGetPinnableReference();

while (bytesConsumed < srcLength && bytesWritten < dstLength)
{
uint codePoint = Unsafe.Add(ref src, bytesConsumed);

int byteCount = EncodingHelper.GetUtf8DecodedBytes((byte)codePoint);
if (byteCount == 0)
goto InvalidData;
if (srcLength - bytesConsumed < byteCount)
goto NeedMoreData;

if (byteCount > 1)
codePoint &= (byte)(0x7F >> byteCount);

for (var i = 1; i < byteCount; i++)
{
ref byte next = ref Unsafe.Add(ref src, bytesConsumed + i);
if ((next & EncodingHelper.b1100_0000U) != EncodingHelper.b1000_0000U)
goto InvalidData;

codePoint = (codePoint << 6) | (uint)(EncodingHelper.b0011_1111U & next);
}

Unsafe.As<byte, uint>(ref Unsafe.Add(ref dst, bytesWritten)) = codePoint;
bytesWritten += 4;
bytesConsumed += byteCount;
}

return bytesConsumed < srcLength ? TransformationStatus.DestinationTooSmall : TransformationStatus.Done;

InvalidData:
return TransformationStatus.InvalidData;

NeedMoreData:
return TransformationStatus.NeedMoreSourceData;
}

#endregion UTF-32 Conversions
}
}

+ 151
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/EncodingHelper.cs View File

@@ -0,0 +1,151 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Text
{
internal static class EncodingHelper
{
#region Constants

private const uint FirstNotSupportedCodePoint = 0x110000; // 17 * 2^16
private const uint BasicMultilingualPlaneEndMarker = 0x10000;

// TODO: Make this immutable and let them be strong typed
// http://unicode.org/cldr/utility/list-unicodeset.jsp?a=\p{whitespace}&g=&i=
private static readonly uint[] SortedWhitespaceCodePoints = new uint[25]
{
0x0009, 0x000A, 0x000B, 0x000C, 0x000D,
0x0020,
0x0085,
0x00A0,
0x1680,
0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006,
0x2007,
0x2008, 0x2009, 0x200A,
0x2028, 0x2029,
0x202F,
0x205F,
0x3000
};

public const char HighSurrogateStart = '\ud800';
public const char HighSurrogateEnd = '\udbff';
public const char LowSurrogateStart = '\udc00';
public const char LowSurrogateEnd = '\udfff';

// To get this to compile with dotnet cli, we need to temporarily un-binary the magic values
public const byte b0000_0111U = 0x07; //7
public const byte b0000_1111U = 0x0F; //15
public const byte b0001_1111U = 0x1F; //31
public const byte b0011_1111U = 0x3F; //63
public const byte b0111_1111U = 0x7F; //127
public const byte b1000_0000U = 0x80; //128
public const byte b1100_0000U = 0xC0; //192
public const byte b1110_0000U = 0xE0; //224
public const byte b1111_0000U = 0xF0; //240
public const byte b1111_1000U = 0xF8; //248

#endregion Constants

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsWhitespace(uint codePoint)
{
return Array.BinarySearch<uint>(SortedWhitespaceCodePoints, codePoint) >= 0;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsSupportedCodePoint(uint codePoint)
{
if (codePoint >= FirstNotSupportedCodePoint)
return false;
if (codePoint >= HighSurrogateStart && codePoint <= LowSurrogateEnd)
return false;
if (codePoint >= 0xFDD0 && codePoint <= 0xFDEF)
return false;
if (codePoint == 0xFFFE || codePoint == 0xFFFF)
return false;

return true;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsBmp(uint codePoint)
{
return codePoint < BasicMultilingualPlaneEndMarker;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public unsafe static int PtrDiff(char* a, char* b)
{
return (int)(((uint)((byte*)a - (byte*)b)) >> 1);
}

// byte* flavor just for parity
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public unsafe static int PtrDiff(byte* a, byte* b)
{
return (int)(a - b);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool InRange(int ch, int start, int end)
{
return (uint)(ch - start) <= (uint)(end - start);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int GetUtf8DecodedBytes(byte b)
{
if ((b & b1000_0000U) == 0)
return 1;

if ((b & b1110_0000U) == b1100_0000U)
return 2;

if ((b & b1111_0000U) == b1110_0000U)
return 3;

if ((b & b1111_1000U) == b1111_0000U)
return 4;

return 0;
}

internal static int GetUtf8EncodedBytes(uint codePoint)
{
if (codePoint <= 0x7F)
return 1;

if (codePoint <= 0x7FF)
return 2;

if (codePoint <= 0xFFFF)
return 3;

if (codePoint <= 0x10FFFF)
return 4;

return 0;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsSurrogate(uint codePoint)
{
return codePoint >= HighSurrogateStart && codePoint <= LowSurrogateEnd;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsLowSurrogate(uint codePoint)
{
return codePoint >= LowSurrogateStart && codePoint <= LowSurrogateEnd;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsHighSurrogate(uint codePoint)
{
return codePoint >= HighSurrogateStart && codePoint <= HighSurrogateEnd;
}
}
}

+ 343
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/ParsingTrie.cs View File

@@ -0,0 +1,343 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Collections;
using System.Collections.Generic;

namespace System.Text
{
internal static class ParsingTrie
{
#region Parsing trie struct

// The parsing trie is structured as an array, which means that there are two types of
// "nodes" for representational purposes
//
// The first node type (the parent node) uses the valueOrNumChildren to represent the number of children
// underneath it. The index is unused for this type of node, except when it's used for
// sequential node mapping (see below). If valueOrNumChildren is zero for this type of node, the index
// is used and represents an index into _digitsAndSymbols.
//
// The second node types immediately follow the first (the childe nodes). They are composed of a value
// (valueOrNumChildren), which is walked via binary search, and an index, which points to another
// node contained in the array.
//
// We use the int index here to encode max-min info for sequential leaves
// It's very common for digits to be encoded sequentially, so we save time by mapping here
// The index is formatted as such: 0xAABBCCDD, where AA = the min value,
// BB = the index of the min value relative to the current node (1-indexed),
// CC = the max value, and DD = the max value's index in the same coord-system as BB.
public struct Node
{
public byte ValueOrNumChildren;
public int IndexOrSymbol;
}

#endregion Parsing trie struct

/// <summary>
/// A Suffix represents the ending sequence of bytes that correspond to a symbol.
/// Suffixes play an important role in the parsing trie generation algorithm.
///
/// Let's say there are four symbols:
/// Symbol 0: Sequence 1, 1, 2, 3
/// Symbol 1: Sequence 0, 1, 2, 3
/// Symbol 2: Sequence 0, 1, 4, 4
/// Symbol 3: Sequence 1, 1, 2, 1
///
/// First, a Suffix is created for each symbol's sequence, and the Suffixes are sorted by their byte sequences:
/// ListOfSuffix {
/// Suffix { SymbolIndex: 1, Bytes: { 0, 1, 2, 3 } }
/// Suffix { SymbolIndex: 2, Bytes: { 0, 1, 4, 4 } }
/// Suffix { SymbolIndex: 3, Bytes: { 1, 1, 2, 1 } }
/// Suffix { SymbolIndex: 0, Bytes: { 1, 1, 2, 3 } }
/// }
///
/// Next, the Suffixes are clumped into SuffixClumps, based on the beginning byte:
/// ListOfSuffixClump {
/// SuffixClump {
/// BeginningByte: 0
/// Suffixes {
/// Suffix { SymbolIndex: 1, Bytes: { 1, 2, 3 } }
/// Suffix { SymbolIndex: 2, Bytes: { 1, 4, 4 } }
/// }
/// }
/// SuffixClump {
/// BeginningByte: 1
/// Suffixes {
/// Suffix { SymbolIndex: 3, Bytes: { 1, 2, 1 } }
/// Suffix { SymbolIndex: 0, Bytes: { 1, 2, 3 } }
/// }
/// }
/// }
///
/// Then, a parent ParsingTrieNode is created, with its NumChildren equal to the number of SuffixClumps.
/// Each SuffixClump represents both a "child" node in the parsing trie, and the "parent" node that child
/// node points to.
///
/// Each SuffixClump that has more than one Suffix will require further clumping; that is to say, it does
/// not represent a leaf node in the parsing trie. Such SuffixClumps will be recursively clumped.
/// </summary>
private struct Suffix : IComparable<Suffix>
{
public int SymbolIndex;
public byte[] Bytes;

public Suffix(int symbolIndex, byte[] bytes)
{
SymbolIndex = symbolIndex;
Bytes = bytes;
}

public Suffix(int symbolIndex, ReadOnlySpan<byte> bytes)
{
SymbolIndex = symbolIndex;

// HACKHACK: Keeping Bytes as a Span property on Suffix will cause crashing in .NET Core 2.0.
// Storing as pure array for now until we can re-visit.
// This is necessary to unblock usage of fast Span for Kestrel and others.
Bytes = bytes.ToArray();
}

public int CompareTo(Suffix other)
{
var shorter = Math.Min(other.Bytes.Length, Bytes.Length);
for(int index = 0; index < shorter; index++)
{
if (Bytes[index] == other.Bytes[index]) continue;
return Bytes[index].CompareTo(other.Bytes[index]);
}
return Bytes.Length.CompareTo(other.Bytes.Length);
}
}

private struct SuffixClump
{
public byte BeginningByte;
public List<Suffix> Suffixes;

public SuffixClump(byte beginningByte)
{
BeginningByte = beginningByte;
// This list of suffixes will not exceed the number of symbols. Initialize
// the list to be of size 20, which is slightly larger than the number of symbols.
Suffixes = new List<Suffix>(20);
}
}

private struct Sequence : IComparable<Sequence>
{
public int BeginningIndex;
public int EndIndex;
public byte BeginningValue;
public byte EndValue;

// This constructor creates a sequence of length 0.
public Sequence(int index, byte value)
{
BeginningIndex = index;
EndIndex = index;
BeginningValue = value;
EndValue = value;
}

public int CompareTo(Sequence other)
{
int thisLength = EndIndex - BeginningIndex;
int otherLength = other.EndIndex - other.BeginningIndex;
return thisLength.CompareTo(otherLength);
}

public int Length
{
get
{
return EndIndex - BeginningIndex;
}
}

// Sequence map is formatted as such:
// 0xAABBCCDD
// AA: The min value
// BB: The index of the min value relative to the current node (1-indexed)
// CC: The max value
// DD: The max value's index in the same coord-system as BB
public int CreateSequenceMap()
{
int sequenceMap = 0;
// AA
sequenceMap += BeginningValue << 24;
// BB: Add 1 to BeginningIndex because the parent node is located 1 place before the 0-indexed child node
sequenceMap += (BeginningIndex + 1) << 16;
// CC
sequenceMap += EndValue << 8;
// DD: Add 1 to EndIndex for same reason as BB
sequenceMap += EndIndex + 1;

return sequenceMap;
}
}

// The return value here is the index in parsingTrieList at which the parent node was placed.
private static int CreateParsingTrieNodeAndChildren(ref List<Node> parsingTrieList, List<Suffix> sortedSuffixes)
{
// If there is only one suffix, create a leaf node
if (sortedSuffixes.Count == 1)
{
Node leafNode = new Node();
leafNode.ValueOrNumChildren = 0;
leafNode.IndexOrSymbol = sortedSuffixes[0].SymbolIndex;
int leafNodeIndex = parsingTrieList.Count;
parsingTrieList.Add(leafNode);
return leafNodeIndex;
}

// Group suffixes into clumps based on first byte
List<SuffixClump> clumps = new List<SuffixClump>(sortedSuffixes.Count);
byte beginningByte = sortedSuffixes[0].Bytes[0];
SuffixClump currentClump = new SuffixClump(beginningByte);
clumps.Add(currentClump);

// Initialize sequence detection
Sequence currentSequence = new Sequence(0, beginningByte);
Sequence longestSequence = currentSequence;

foreach (Suffix suffix in sortedSuffixes)
{
var bytesSpan = new Span<byte>(suffix.Bytes);
if (suffix.Bytes[0] == beginningByte)
{
currentClump.Suffixes.Add(new Suffix(suffix.SymbolIndex, bytesSpan.Slice(1)));
}
else
{
beginningByte = suffix.Bytes[0];

// Determine if the new clump is part of a sequence
if (beginningByte == currentSequence.EndValue + 1)
{
// This clump is part of the current sequence
currentSequence.EndIndex++;
currentSequence.EndValue++;

if (!currentSequence.Equals(longestSequence) && currentSequence.CompareTo(longestSequence) > 0)
{
// Replace the longest sequence with this sequence
longestSequence = currentSequence;
}
}
else
{
// This clump is part of a new sequence
currentSequence = new Sequence(clumps.Count, beginningByte);
}

// This is a new clump, with at least one suffix inside it. Add to the list of clumps.
currentClump = new SuffixClump(beginningByte);
currentClump.Suffixes.Add(new Suffix(suffix.SymbolIndex, bytesSpan.Slice(1)));
clumps.Add(currentClump);
}
}

// Now that we know how many children there are, create parent node and place in list
Node parentNode = new Node();
parentNode.ValueOrNumChildren = (byte)clumps.Count;
// Only bother specifying a sequence if the longest sequence is sufficiently long
if (longestSequence.Length > 5)
{
parentNode.IndexOrSymbol = longestSequence.CreateSequenceMap();
}
else
{
parentNode.IndexOrSymbol = 0;
}
int parentNodeIndex = parsingTrieList.Count;
parsingTrieList.Add(parentNode);

// Reserve space in list for child nodes. In this algorithm, all parent nodes are created first, leaving gaps for the child nodes
// to be filled in once it is known where they point to.
int childNodeStartIndex = parsingTrieList.Count;
for (int i = 0; i < clumps.Count; i++)
{
parsingTrieList.Add(default);
}

// Process child nodes
List<Node> childNodes = new List<Node>();
foreach (SuffixClump clump in clumps)
{
Node childNode = new Node();
childNode.ValueOrNumChildren = clump.BeginningByte;
childNode.IndexOrSymbol = CreateParsingTrieNodeAndChildren(ref parsingTrieList, clump.Suffixes);
childNodes.Add(childNode);
}

// Place child nodes in spots allocated for them
int childNodeIndex = childNodeStartIndex;
foreach (Node childNode in childNodes)
{
parsingTrieList[childNodeIndex] = childNode;
childNodeIndex++;
}

return parentNodeIndex;
}

public static Node[] Create(byte[][] symbols)
{
List<Suffix> symbolList = new List<Suffix>(symbols.Length);
for (int i = 0; i < symbols.Length; i++)
{
if (symbols[i] != null)
{
symbolList.Add(new Suffix(i, symbols[i]));
}
}

// Sort the symbol list. This is important for allowing binary search of the child nodes, as well as
// counting the number of children a node has.
symbolList.Sort();

// validate symbol consistemcy:
// a) each symbol must be unique
// b) a symbol cannot be a prefix of another symbol
// c) symbols cannot be empty
for(int i = 1; i < symbolList.Count; i++)
{
var first = symbolList[i - 1];
var second = symbolList[i];

if(first.Bytes.Length == 0 || second.Bytes.Length == 0)
{
throw new ArgumentException("Symbol cannot be zero bytes long");
}
var firstSpan = first.Bytes.AsSpan();
if (firstSpan.SequenceEqual(second.Bytes))
{
throw new ArgumentException("Symbols cannot be identical");
}
if (first.Bytes.Length > second.Bytes.Length)
{
if (firstSpan.StartsWith(second.Bytes))
{
throw new ArgumentException("Symbols are ambiguous");
}
}
else if(first.Bytes.Length < second.Bytes.Length)
{
if (second.Bytes.AsSpan().StartsWith(first.Bytes))
{
throw new ArgumentException("Symbols are ambiguous");
}
}
}

List<Node> parsingTrieList = new List<Node>(100);
CreateParsingTrieNodeAndChildren(ref parsingTrieList, symbolList);

return parsingTrieList.ToArray();
}
}
}

+ 29
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/SymbolTable.Symbol.cs View File

@@ -0,0 +1,29 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace System.Text
{
public partial class SymbolTable
{
// Do not change the specific enum values without careful consideration of the impacts to the parsers.
public enum Symbol : ushort {
D0 = (ushort)0,
D1 = (ushort)1,
D2 = (ushort)2,
D3 = (ushort)3,
D4 = (ushort)4,
D5 = (ushort)5,
D6 = (ushort)6,
D7 = (ushort)7,
D8 = (ushort)8,
D9 = (ushort)9,
DecimalSeparator = (ushort)10,
Exponent = (ushort)16,
GroupSeparator = (ushort)11,
InfinitySign = (ushort)12,
MinusSign = (ushort)13,
NaN = (ushort)15,
PlusSign = (ushort)14,
}
}
}

+ 98
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/SymbolTable.Utf16.cs View File

@@ -0,0 +1,98 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Buffers;
using System.Runtime.CompilerServices;

namespace System.Text
{
public partial class SymbolTable
{
private sealed class Utf16InvariantSymbolTable : SymbolTable
{
private static readonly byte[][] Utf16DigitsAndSymbols = new byte[][]
{
new byte[] { 48, 0, }, // digit 0
new byte[] { 49, 0, },
new byte[] { 50, 0, },
new byte[] { 51, 0, },
new byte[] { 52, 0, },
new byte[] { 53, 0, },
new byte[] { 54, 0, },
new byte[] { 55, 0, },
new byte[] { 56, 0, },
new byte[] { 57, 0, }, // digit 9
new byte[] { 46, 0, }, // decimal separator
new byte[] { 44, 0, }, // group separator
new byte[] { 73, 0, 110, 0, 102, 0, 105, 0, 110, 0, 105, 0, 116, 0, 121, 0, }, // Infinity
new byte[] { 45, 0, }, // minus sign
new byte[] { 43, 0, }, // plus sign
new byte[] { 78, 0, 97, 0, 78, 0, }, // NaN
new byte[] { 69, 0, }, // E
new byte[] { 101, 0, }, // e
};

public Utf16InvariantSymbolTable() : base(Utf16DigitsAndSymbols) {}

public override bool TryEncode(byte utf8, Span<byte> destination, out int bytesWritten)
{
if (destination.Length < 2)
goto ExitFailed;

if (utf8 > 0x7F)
goto ExitFailed;

Unsafe.As<byte, char>(ref destination.DangerousGetPinnableReference()) = (char)utf8;
bytesWritten = 2;
return true;

ExitFailed:
bytesWritten = 0;
return false;
}

public override bool TryEncode(ReadOnlySpan<byte> utf8, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
{
var status = Encoders.Utf8.ToUtf16(utf8, destination, out bytesConsumed, out bytesWritten);
if (status != TransformationStatus.Done)
{
bytesConsumed = bytesWritten = 0;
return false;
}

return true;
}

public override bool TryParse(ReadOnlySpan<byte> source, out byte utf8, out int bytesConsumed)
{
if (source.Length < 2)
goto ExitFailed;

ref char value = ref Unsafe.As<byte, char>(ref source.DangerousGetPinnableReference());
if (value > 0x7F)
goto ExitFailed;

bytesConsumed = 2;
utf8 = (byte)value;
return true;

ExitFailed:
utf8 = 0;
bytesConsumed = 0;
return false;
}

public override bool TryParse(ReadOnlySpan<byte> source, Span<byte> utf8, out int bytesConsumed, out int bytesWritten)
{
var status = Encoders.Utf16.ToUtf8(source, utf8, out bytesConsumed, out bytesWritten);
if (status != TransformationStatus.Done)
{
bytesConsumed = bytesWritten = 0;
return false;
}

return true;
}
}
}
}

+ 100
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/SymbolTable.Utf8.cs View File

@@ -0,0 +1,100 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Text
{
public partial class SymbolTable
{
private sealed class Utf8InvariantSymbolTable : SymbolTable
{
private static readonly byte[][] Utf8DigitsAndSymbols = new byte[][]
{
new byte[] { 48, },
new byte[] { 49, },
new byte[] { 50, },
new byte[] { 51, },
new byte[] { 52, },
new byte[] { 53, },
new byte[] { 54, },
new byte[] { 55, },
new byte[] { 56, },
new byte[] { 57, }, // digit 9
new byte[] { 46, }, // decimal separator
new byte[] { 44, }, // group separator
new byte[] { 73, 110, 102, 105, 110, 105, 116, 121, },
new byte[] { 45, }, // minus sign
new byte[] { 43, }, // plus sign
new byte[] { 78, 97, 78, }, // NaN
new byte[] { 69, }, // E
new byte[] { 101, }, // e
};

public Utf8InvariantSymbolTable() : base(Utf8DigitsAndSymbols) {}

public override bool TryEncode(byte utf8, Span<byte> destination, out int bytesWritten)
{
if (destination.Length < 1)
goto ExitFailed;

if (utf8 > 0x7F)
goto ExitFailed;

destination[0] = utf8;
bytesWritten = 1;
return true;

ExitFailed:
bytesWritten = 0;
return false;
}

public override bool TryEncode(ReadOnlySpan<byte> utf8, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
{
// TODO: We might want to validate that the stream we are moving from utf8 to destination (also UTF-8) is valid.
// For now, we are just doing a copy.
if (utf8.TryCopyTo(destination))
{
bytesConsumed = bytesWritten = utf8.Length;
return true;
}

bytesConsumed = bytesWritten = 0;
return false;
}

public override bool TryParse(ReadOnlySpan<byte> source, out byte utf8, out int bytesConsumed)
{
if (source.Length < 1)
goto ExitFailed;

utf8 = source[0];
if (utf8 > 0x7F)
goto ExitFailed;

bytesConsumed = 1;
return true;

ExitFailed:
utf8 = 0;
bytesConsumed = 0;
return false;
}

public override bool TryParse(ReadOnlySpan<byte> source, Span<byte> utf8, out int bytesConsumed, out int bytesWritten)
{
// TODO: We might want to validate that the stream we are moving from utf8 to destination (also UTF-8) is valid.
// For now, we are just doing a copy.
if (source.TryCopyTo(utf8))
{
bytesConsumed = bytesWritten = source.Length;
return true;
}

bytesConsumed = bytesWritten = 0;
return false;
}
}
}
}

+ 242
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Encoding/SymbolTable.cs View File

@@ -0,0 +1,242 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace System.Text
{
public abstract partial class SymbolTable
{
#region Private data

private readonly byte[][] _symbols; // this could be flattened into a single array
private readonly ParsingTrie.Node[] _parsingTrie; // prefix tree used for parsing

#endregion Private data

#region Constructors

protected SymbolTable(byte[][] symbols)
{
_symbols = symbols;
_parsingTrie = ParsingTrie.Create(symbols);
}

#endregion Constructors

#region Static instances

public readonly static SymbolTable InvariantUtf8 = new Utf8InvariantSymbolTable();

public readonly static SymbolTable InvariantUtf16 = new Utf16InvariantSymbolTable();

#endregion Static instances

public bool TryEncode(Symbol symbol, Span<byte> destination, out int bytesWritten)
{
byte[] bytes = _symbols[(int)symbol];
bytesWritten = bytes.Length;
if (bytesWritten > destination.Length)
{
bytesWritten = 0;
return false;
}

if (bytesWritten == 2)
{
destination[0] = bytes[0];
destination[1] = bytes[1];
return true;
}

if (bytesWritten == 1)
{
destination[0] = bytes[0];
return true;
}

new Span<byte>(bytes).CopyTo(destination);
return true;
}

public abstract bool TryEncode(byte utf8, Span<byte> destination, out int bytesWritten);

public abstract bool TryEncode(ReadOnlySpan<byte> utf8, Span<byte> destination, out int bytesConsumed, out int bytesWritten);

public bool TryParse(ReadOnlySpan<byte> source, out Symbol symbol, out int bytesConsumed)
{
int trieIndex = 0;
int codeUnitIndex = 0;
bytesConsumed = 0;
while (true)
{
if (_parsingTrie[trieIndex].ValueOrNumChildren == 0) // if numChildren == 0, we're on a leaf & we've found our value and completed the code unit
{
symbol = (Symbol)_parsingTrie[trieIndex].IndexOrSymbol; // return the parsed value
if (VerifySuffix(source, codeUnitIndex, symbol))
{
bytesConsumed = _symbols[(int)symbol].Length;
return true;
}
else
{
symbol = 0;
bytesConsumed = 0;
return false;
}
}
else
{
int search = BinarySearch(trieIndex, codeUnitIndex, source[codeUnitIndex]); // we search the _parsingTrie for the nextByte

if (search > 0) // if we found a node
{
trieIndex = _parsingTrie[search].IndexOrSymbol;
bytesConsumed++;
codeUnitIndex++;
}
else
{
symbol = 0;
bytesConsumed = 0;
return false;
}
}
}
}

public abstract bool TryParse(ReadOnlySpan<byte> source, out byte utf8, out int bytesConsumed);

public abstract bool TryParse(ReadOnlySpan<byte> source, Span<byte> utf8, out int bytesConsumed, out int bytesWritten);

#region Public UTF-16 to UTF-8 helpers

public bool TryEncode(ReadOnlySpan<char> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
{
ReadOnlySpan<byte> srcBytes = source.AsBytes();

if (this == SymbolTable.InvariantUtf16)
return TryEncodeUtf16(srcBytes, destination, out bytesConsumed, out bytesWritten);

const int BufferSize = 256;

int srcLength = srcBytes.Length;
if (srcLength <= 0)
{
bytesConsumed = bytesWritten = 0;
return true;
}

Span<byte> temp;
unsafe
{
byte* pTemp = stackalloc byte[BufferSize];
temp = new Span<byte>(pTemp, BufferSize);
}

bytesWritten = 0;
bytesConsumed = 0;
while (srcLength > bytesConsumed)
{
var status = Encoders.Utf16.ToUtf8(srcBytes, temp, out int consumed, out int written);
if (status == Buffers.TransformationStatus.InvalidData)
goto ExitFailed;

srcBytes = srcBytes.Slice(consumed);
bytesConsumed += consumed;

if (!TryEncode(temp.Slice(0, written), destination, out consumed, out written))
goto ExitFailed;

destination = destination.Slice(written);
bytesWritten += written;
}

return true;

ExitFailed:
return false;
}

private bool TryEncodeUtf16(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesConsumed, out int bytesWritten)
{
// NOTE: There is no validation of this UTF-16 encoding. A caller is expected to do any validation on their own if
// they don't trust the data.
bytesConsumed = source.Length;
bytesWritten = destination.Length;
if (bytesConsumed > bytesWritten)
{
source = source.Slice(0, bytesWritten);
bytesConsumed = bytesWritten;
}
else
{
bytesWritten = bytesConsumed;
}

source.CopyTo(destination);
return true;
}

#endregion Public UTF-16 to UTF-8 helpers

#region Private helpers

// This binary search implementation returns an int representing either:
// - the index of the item searched for (if the value is positive)
// - the index of the location where the item should be placed to maintain a sorted list (if the value is negative)
private int BinarySearch(int nodeIndex, int level, byte value)
{
int maxMinLimits = _parsingTrie[nodeIndex].IndexOrSymbol;
if (maxMinLimits != 0 && value > (uint)maxMinLimits >> 24 && value < (uint)(maxMinLimits << 16) >> 24)
{
// See the comments on the struct above for more information about this format
return (int)(nodeIndex + ((uint)(maxMinLimits << 8) >> 24) + value - ((uint)maxMinLimits >> 24));
}

int leftBound = nodeIndex + 1, rightBound = nodeIndex + _parsingTrie[nodeIndex].ValueOrNumChildren;
int midIndex = 0;
while (true)
{
if (leftBound > rightBound) // if the search failed
{
// this loop is necessary because binary search takes the floor
// of the middle, which means it can give incorrect indices for insertion.
// we should never iterate up more than two indices.
while (midIndex < nodeIndex + _parsingTrie[nodeIndex].ValueOrNumChildren
&& _parsingTrie[midIndex].ValueOrNumChildren < value)
{
midIndex++;
}
return -midIndex;
}

midIndex = (leftBound + rightBound) / 2; // find the middle value

byte mValue = _parsingTrie[midIndex].ValueOrNumChildren;

if (mValue < value)
leftBound = midIndex + 1;
else if (mValue > value)
rightBound = midIndex - 1;
else
return midIndex;
}
}

private bool VerifySuffix(ReadOnlySpan<byte> buffer, int codeUnitIndex, Symbol symbol)
{
int codeUnitLength = _symbols[(int)symbol].Length;
if (codeUnitIndex == codeUnitLength - 1)
return true;

for (int i = 0; i < codeUnitLength - codeUnitIndex; i++)
{
if (buffer[i + codeUnitIndex] != _symbols[(int)symbol][i + codeUnitIndex])
return false;
}

return true;
}

#endregion Private helpers
}
}

+ 91
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/FloatFormatter.cs View File

@@ -0,0 +1,91 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Diagnostics;

namespace System.Text
{
internal static class FloatFormatter
{
public static bool TryFormatNumber(double value, bool isSingle, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
Precondition.Require(format.Symbol == 'G' || format.Symbol == 'E' || format.Symbol == 'F');

symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

bytesWritten = 0;
int written;

if (Double.IsNaN(value))
{
return symbolTable.TryEncode(SymbolTable.Symbol.NaN, buffer, out bytesWritten);
}

if (Double.IsInfinity(value))
{
if (Double.IsNegativeInfinity(value))
{
if (!symbolTable.TryEncode(SymbolTable.Symbol.MinusSign, buffer, out written))
{
bytesWritten = 0;
return false;
}
bytesWritten += written;
}
if (!symbolTable.TryEncode(SymbolTable.Symbol.InfinitySign, buffer.Slice(bytesWritten), out written))
{
bytesWritten = 0;
return false;
}
bytesWritten += written;
return true;
}

// TODO: the lines below need to be replaced with properly implemented algorithm
// the problem is the algorithm is complex, so I am commiting a stub for now
var hack = value.ToString(format.Symbol.ToString());
var utf16Bytes = hack.AsSpan().AsBytes();
if (symbolTable == SymbolTable.InvariantUtf8)
{
var status = Encoders.Utf16.ToUtf8(utf16Bytes, buffer, out int consumed, out bytesWritten);
return status == Buffers.TransformationStatus.Done;
}
else if (symbolTable == SymbolTable.InvariantUtf16)
{
bytesWritten = utf16Bytes.Length;
if (utf16Bytes.TryCopyTo(buffer))
return true;

bytesWritten = 0;
return false;
}
else
{
// TODO: This is currently pretty expensive. Can this be done more efficiently?
// Note: removing the hack might solve this problem a very different way.
var status = Encoders.Utf16.ToUtf8Length(utf16Bytes, out int needed);
if (status != Buffers.TransformationStatus.Done)
{
bytesWritten = 0;
return false;
}

Span<byte> temp;
unsafe
{
var buf = stackalloc byte[needed];
temp = new Span<byte>(buf, needed);
}

status = Encoders.Utf16.ToUtf8(utf16Bytes, temp, out int consumed, out written);
if (status != Buffers.TransformationStatus.Done)
{
bytesWritten = 0;
return false;
}

return symbolTable.TryEncode(temp, buffer, out consumed, out bytesWritten);
}
}
}
}

+ 23
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/IBufferFormattable.cs View File

@@ -0,0 +1,23 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.


namespace System.Text
{
public interface IBufferFormattable
{
// I went back and forth between bytesWritten being out and ref. Ref makes it easier to implement the interface.
// Out makes so that callers don't have to trust calees doing the right thing with the bytesWritten value.
// I prefer correctness here over ease of use (as this is a low level API), so I decided on out parameter.

/// <summary>
/// This interface should be implemented by types that want to support allocation-free formatting.
/// </summary>
/// <param name="buffer">The buffer to format the value into</param>
/// <param name="written">This parameter is used to return the number of bytes that were written to the buffer</param>
/// <param name="format">This is a pre-parsed representation of the formatting string. It's preparsed for efficiency.</param>
/// <param name="symbolTable">This object implements the character and symbol encoder.</param>
/// <returns>False if the buffer was to small, otherwise true.</returns>
bool TryFormat(Span<byte> buffer, out int written, ParsedFormat format = default, SymbolTable symbolTable = null);
}
}

+ 208
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/FormattingHelpers.cs View File

@@ -0,0 +1,208 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Diagnostics;
using System.Runtime.CompilerServices;

namespace System.Text
{
// All the helper methods in this class assume that the by-ref is valid and that there is
// enough space to fit the items that will be written into the underlying memory. The calling
// code must have already done all the necessary validation.
internal static class FormattingHelpers
{
// For the purpose of formatting time, the format specifier contains room for
// exactly 7 digits in the fraction portion. See "Round-trip format specifier"
// at the following URL for more information.
// https://msdn.microsoft.com/en-us/library/az4se3k1(v=vs.110).aspx#Roundtrip
private const int FractionDigits = 7;

// A simple lookup table for converting numbers to hex.
private const string HexTable = "0123456789abcdef";

#region UTF-8 Helper methods

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void WriteHexByte(byte value, ref byte buffer, int index)
{
Unsafe.Add(ref buffer, index) = (byte)HexTable[value >> 4];
Unsafe.Add(ref buffer, index + 1) = (byte)HexTable[value & 0xF];
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int WriteFractionDigits(long value, int digitCount, ref byte buffer, int index)
{
for (var i = FractionDigits; i > digitCount; i--)
value /= 10;

return WriteDigits(value, digitCount, ref buffer, index);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int WriteDigits(long value, int digitCount, ref byte buffer, int index)
{
long left = value;

for (var i = digitCount - 1; i >= 0; i--)
{
left = DivMod(left, 10, out long num);
Unsafe.Add(ref buffer, index + i) = (byte)('0' + num);
}

return digitCount;
}

/// <summary>
/// The unsigned long implementation of this method is much slower than the signed version above
/// due to optimization tricks that happen at the IL to ASM stage. Use the signed version unless
/// you definitely need to deal with numbers larger than long.MaxValue.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int WriteDigits(ulong value, int digitCount, ref byte buffer, int index)
{
ulong left = value;

for (var i = digitCount - 1; i >= 0; i--)
{
left = DivMod(left, 10, out ulong num);
Unsafe.Add(ref buffer, index + i) = (byte)('0' + num);
}

return digitCount;
}

#endregion UTF-8 Helper methods

#region UTF-16 Helper methods

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void WriteHexByte(byte value, ref char buffer, int index)
{
Unsafe.Add(ref buffer, index) = HexTable[value >> 4];
Unsafe.Add(ref buffer, index + 1) = HexTable[value & 0xF];
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int WriteFractionDigits(long value, int digitCount, ref char buffer, int index)
{
for (var i = FractionDigits; i > digitCount; i--)
value /= 10;

return WriteDigits(value, digitCount, ref buffer, index);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int WriteDigits(long value, int digitCount, ref char buffer, int index)
{
long left = value;

for (var i = digitCount - 1; i >= 0; i--)
{
left = DivMod(left, 10, out long num);
Unsafe.Add(ref buffer, index + i) = (char)('0' + num);
}

return digitCount;
}

/// <summary>
/// The unsigned long implementation of this method is much slower than the signed version above
/// due to optimization tricks that happen at the IL to ASM stage. Use the signed version unless
/// you definitely need to deal with numbers larger than long.MaxValue.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int WriteDigits(ulong value, int digitCount, ref char buffer, int index)
{
ulong left = value;

for (var i = digitCount - 1; i >= 0; i--)
{
left = DivMod(left, 10, out ulong num);
Unsafe.Add(ref buffer, index + i) = (char)('0' + num);
}

return digitCount;
}

#endregion UTF-16 Helper methods

#region Math Helper methods

/// <summary>
/// We don't have access to Math.DivRem, so this is a copy of the implementation.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static long DivMod(long numerator, long denominator, out long modulo)
{
long div = numerator / denominator;
modulo = numerator - (div * denominator);
return div;
}

/// <summary>
/// We don't have access to Math.DivRem, so this is a copy of the implementation.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ulong DivMod(ulong numerator, ulong denominator, out ulong modulo)
{
ulong div = numerator / denominator;
modulo = numerator - (div * denominator);
return div;
}

#endregion Math Helper methods

#region Character counting helper methods

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int CountDigits(long n)
{
if (n == 0) return 1;

int digits = 0;
while (n != 0)
{
n /= 10;
digits++;
}

return digits;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int CountDigits(ulong n)
{
if (n == 0) return 1;

int digits = 0;
while (n != 0)
{
n /= 10;
digits++;
}

return digits;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int CountFractionDigits(long n)
{
Precondition.Require(n >= 0);

long left = n;
long m = 0;
int count = FractionDigits;

// Remove all the 0 (zero) values from the right.
while (left > 0 && m == 0 && count > 0)
{
left = DivMod(left, 10, out m);
count--;
}

return count + 1;
}

#endregion Character counting helper methods
}
}

+ 92
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/GuidFormatter.Utf16.cs View File

@@ -0,0 +1,92 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Text
{
internal static class InvariantUtf16GuidFormatter
{
#region Constants

private const int GuidChars = 32;

private const char OpenBrace = '{';
private const char CloseBrace = '}';

private const char OpenParen = '(';
private const char CloseParen = ')';

private const char Dash = '-';

#endregion Constants

public static unsafe bool TryFormat(this Guid value, Span<byte> buffer, out int bytesWritten, ParsedFormat format)
{
bool dash = format.Symbol != 'N';
bool bookEnds = (format.Symbol == 'B') || (format.Symbol == 'P');

bytesWritten = (GuidChars + (dash ? 4 : 0) + (bookEnds ? 2 : 0)) * sizeof(char);
if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

Span<char> dst = buffer.NonPortableCast<byte, char>();
ref char utf16Bytes = ref dst.DangerousGetPinnableReference();
byte* bytes = (byte*)&value;
int idx = 0;

if (bookEnds && format.Symbol == 'B')
Unsafe.Add(ref utf16Bytes, idx++) = OpenBrace;
else if (bookEnds && format.Symbol == (byte)'P')
Unsafe.Add(ref utf16Bytes, idx++) = OpenParen;

FormattingHelpers.WriteHexByte(bytes[3], ref utf16Bytes, idx);
FormattingHelpers.WriteHexByte(bytes[2], ref utf16Bytes, idx + 2);
FormattingHelpers.WriteHexByte(bytes[1], ref utf16Bytes, idx + 4);
FormattingHelpers.WriteHexByte(bytes[0], ref utf16Bytes, idx + 6);
idx += 8;

if (dash)
Unsafe.Add(ref utf16Bytes, idx++) = Dash;

FormattingHelpers.WriteHexByte(bytes[5], ref utf16Bytes, idx);
FormattingHelpers.WriteHexByte(bytes[4], ref utf16Bytes, idx + 2);
idx += 4;

if (dash)
Unsafe.Add(ref utf16Bytes, idx++) = Dash;

FormattingHelpers.WriteHexByte(bytes[7], ref utf16Bytes, idx);
FormattingHelpers.WriteHexByte(bytes[6], ref utf16Bytes, idx + 2);
idx += 4;

if (dash)
Unsafe.Add(ref utf16Bytes, idx++) = Dash;

FormattingHelpers.WriteHexByte(bytes[8], ref utf16Bytes, idx);
FormattingHelpers.WriteHexByte(bytes[9], ref utf16Bytes, idx + 2);
idx += 4;

if (dash)
Unsafe.Add(ref utf16Bytes, idx++) = Dash;

FormattingHelpers.WriteHexByte(bytes[10], ref utf16Bytes, idx);
FormattingHelpers.WriteHexByte(bytes[11], ref utf16Bytes, idx + 2);
FormattingHelpers.WriteHexByte(bytes[12], ref utf16Bytes, idx + 4);
FormattingHelpers.WriteHexByte(bytes[13], ref utf16Bytes, idx + 6);
FormattingHelpers.WriteHexByte(bytes[14], ref utf16Bytes, idx + 8);
FormattingHelpers.WriteHexByte(bytes[15], ref utf16Bytes, idx + 10);
idx += 12;

if (bookEnds && format.Symbol == 'B')
Unsafe.Add(ref utf16Bytes, idx++) = CloseBrace;
else if (bookEnds && format.Symbol == 'P')
Unsafe.Add(ref utf16Bytes, idx++) = CloseParen;

return true;
}
}
}

+ 91
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/GuidFormatter.Utf8.cs View File

@@ -0,0 +1,91 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Text
{
internal static class InvariantUtf8GuidFormatter
{
#region Constants

private const int GuidChars = 32;

private const byte OpenBrace = (byte)'{';
private const byte CloseBrace = (byte)'}';

private const byte OpenParen = (byte)'(';
private const byte CloseParen = (byte)')';

private const byte Dash = (byte)'-';

#endregion Constants

public static unsafe bool TryFormat(this Guid value, Span<byte> buffer, out int bytesWritten, ParsedFormat format)
{
bool dash = format.Symbol != 'N';
bool bookEnds = (format.Symbol == 'B') || (format.Symbol == 'P');

bytesWritten = GuidChars + (dash ? 4 : 0) + (bookEnds ? 2 : 0);
if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

ref byte utf8Bytes = ref buffer.DangerousGetPinnableReference();
byte* bytes = (byte*)&value;
int idx = 0;

if (bookEnds && format.Symbol == 'B')
Unsafe.Add(ref utf8Bytes, idx++) = OpenBrace;
else if (bookEnds && format.Symbol == (byte)'P')
Unsafe.Add(ref utf8Bytes, idx++) = OpenParen;

FormattingHelpers.WriteHexByte(bytes[3], ref utf8Bytes, idx);
FormattingHelpers.WriteHexByte(bytes[2], ref utf8Bytes, idx + 2);
FormattingHelpers.WriteHexByte(bytes[1], ref utf8Bytes, idx + 4);
FormattingHelpers.WriteHexByte(bytes[0], ref utf8Bytes, idx + 6);
idx += 8;

if (dash)
Unsafe.Add(ref utf8Bytes, idx++) = Dash;

FormattingHelpers.WriteHexByte(bytes[5], ref utf8Bytes, idx);
FormattingHelpers.WriteHexByte(bytes[4], ref utf8Bytes, idx + 2);
idx += 4;

if (dash)
Unsafe.Add(ref utf8Bytes, idx++) = Dash;

FormattingHelpers.WriteHexByte(bytes[7], ref utf8Bytes, idx);
FormattingHelpers.WriteHexByte(bytes[6], ref utf8Bytes, idx + 2);
idx += 4;

if (dash)
Unsafe.Add(ref utf8Bytes, idx++) = Dash;

FormattingHelpers.WriteHexByte(bytes[8], ref utf8Bytes, idx);
FormattingHelpers.WriteHexByte(bytes[9], ref utf8Bytes, idx + 2);
idx += 4;

if (dash)
Unsafe.Add(ref utf8Bytes, idx++) = Dash;

FormattingHelpers.WriteHexByte(bytes[10], ref utf8Bytes, idx);
FormattingHelpers.WriteHexByte(bytes[11], ref utf8Bytes, idx + 2);
FormattingHelpers.WriteHexByte(bytes[12], ref utf8Bytes, idx + 4);
FormattingHelpers.WriteHexByte(bytes[13], ref utf8Bytes, idx + 6);
FormattingHelpers.WriteHexByte(bytes[14], ref utf8Bytes, idx + 8);
FormattingHelpers.WriteHexByte(bytes[15], ref utf8Bytes, idx + 10);
idx += 12;

if (bookEnds && format.Symbol == 'B')
Unsafe.Add(ref utf8Bytes, idx++) = CloseBrace;
else if (bookEnds && format.Symbol == 'P')
Unsafe.Add(ref utf8Bytes, idx++) = CloseParen;

return true;
}
}
}

+ 262
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/IntegerFormatter.Utf16.cs View File

@@ -0,0 +1,262 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Text
{
internal static class InvariantUtf16IntegerFormatter
{
private const char Minus = '-';
private const char Period = '.';
private const char Seperator = ',';

// Invariant formatting uses groups of 3 for each number group seperated by commas.
// ex. 1,234,567,890
private const int GroupSize = 3;

public static bool TryFormatDecimalInt64(long value, byte precision, Span<byte> buffer, out int bytesWritten)
{
int digitCount = FormattingHelpers.CountDigits(value);
int charsNeeded = digitCount + (int)((value >> 63) & 1);
Span<char> span = buffer.NonPortableCast<byte, char>();

if (span.Length < charsNeeded)
{
bytesWritten = 0;
return false;
}

ref char utf16Bytes = ref span.DangerousGetPinnableReference();
int idx = 0;

if (value < 0)
{
Unsafe.Add(ref utf16Bytes, idx++) = Minus;

// Abs(long.MinValue) == long.MaxValue + 1, so we need to re-route to unsigned to handle value
if (value == long.MinValue)
{
if (!TryFormatDecimalUInt64((ulong)long.MaxValue + 1, precision, buffer.Slice(2), out bytesWritten))
return false;

bytesWritten += sizeof(char); // Add the minus sign
return true;
}

value = -value;
}

if (precision != ParsedFormat.NoPrecision)
{
int leadingZeros = (int)precision - digitCount;
while (leadingZeros-- > 0)
Unsafe.Add(ref utf16Bytes, idx++) = '0';
}

idx += FormattingHelpers.WriteDigits(value, digitCount, ref utf16Bytes, idx);

bytesWritten = idx * sizeof(char);
return true;
}

public static bool TryFormatDecimalUInt64(ulong value, byte precision, Span<byte> buffer, out int bytesWritten)
{
if (value <= long.MaxValue)
return TryFormatDecimalInt64((long)value, precision, buffer, out bytesWritten);

// Remove a single digit from the number. This will get it below long.MaxValue
// Then we call the faster long version and follow-up with writing the last
// digit. This ends up being faster by a factor of 2 than to just do the entire
// operation using the unsigned versions.
value = FormattingHelpers.DivMod(value, 10, out ulong lastDigit);

if (precision != ParsedFormat.NoPrecision && precision > 0)
precision -= 1;

if (!TryFormatDecimalInt64((long)value, precision, buffer, out bytesWritten))
return false;

Span<char> span = buffer.Slice(bytesWritten).NonPortableCast<byte, char>();

if (span.Length < sizeof(char))
{
bytesWritten = 0;
return false;
}

ref char utf16Bytes = ref span.DangerousGetPinnableReference();
FormattingHelpers.WriteDigits(lastDigit, 1, ref utf16Bytes, 0);
bytesWritten += sizeof(char);
return true;
}

public static bool TryFormatNumericInt64(long value, byte precision, Span<byte> buffer, out int bytesWritten)
{
int digitCount = FormattingHelpers.CountDigits(value);
int groupSeperators = (int)FormattingHelpers.DivMod(digitCount, GroupSize, out long firstGroup);
if (firstGroup == 0)
{
firstGroup = 3;
groupSeperators--;
}

int trailingZeros = (precision == ParsedFormat.NoPrecision) ? 2 : precision;
int charsNeeded = (int)((value >> 63) & 1) + digitCount + groupSeperators;
int idx = charsNeeded;

if (trailingZeros > 0)
charsNeeded += trailingZeros + 1; // +1 for period.

Span<char> span = buffer.NonPortableCast<byte, char>();

if (span.Length < charsNeeded)
{
bytesWritten = 0;
return false;
}

ref char utf16Bytes = ref span.DangerousGetPinnableReference();
long v = value;

if (v < 0)
{
Unsafe.Add(ref utf16Bytes, 0) = Minus;

// Abs(long.MinValue) == long.MaxValue + 1, so we need to re-route to unsigned to handle value
if (v == long.MinValue)
{
if (!TryFormatNumericUInt64((ulong)long.MaxValue + 1, precision, buffer.Slice(2), out bytesWritten))
return false;

bytesWritten += sizeof(char); // Add the minus sign
return true;
}

v = -v;
}

// Write out the trailing zeros
if (trailingZeros > 0)
{
Unsafe.Add(ref utf16Bytes, idx) = Period;
FormattingHelpers.WriteDigits(0, trailingZeros, ref utf16Bytes, idx + 1);
}

// Starting from the back, write each group of digits except the first group
while (digitCount > 3)
{
idx -= 3;
v = FormattingHelpers.DivMod(v, 1000, out long groupValue);
FormattingHelpers.WriteDigits(groupValue, 3, ref utf16Bytes, idx);
Unsafe.Add(ref utf16Bytes, --idx) = Seperator;
digitCount -= 3;
}

// Write the first group of digits.
FormattingHelpers.WriteDigits(v, (int)firstGroup, ref utf16Bytes, idx - (int)firstGroup);

bytesWritten = charsNeeded * sizeof(char);
return true;
}

public static bool TryFormatNumericUInt64(ulong value, byte precision, Span<byte> buffer, out int bytesWritten)
{
if (value <= long.MaxValue)
return TryFormatNumericInt64((long)value, precision, buffer, out bytesWritten);

// The ulong path is much slower than the long path here, so we are doing the last group
// inside this method plus the zero padding but routing to the long version for the rest.
value = FormattingHelpers.DivMod(value, 1000, out ulong lastGroup);

if (!TryFormatNumericInt64((long)value, 0, buffer, out bytesWritten))
return false;

if (precision == ParsedFormat.NoPrecision)
precision = 2;

// Since this method routes entirely to the long version if the number is smaller than
// long.MaxValue, we are guaranteed to need to write 3 more digits here before the set
// of trailing zeros.

int extraChars = 4; // 3 digits + group seperator
if (precision > 0)
extraChars += precision + 1; // +1 for period.

Span<char> span = buffer.Slice(bytesWritten).NonPortableCast<byte, char>();

if (span.Length < extraChars)
{
bytesWritten = 0;
return false;
}

ref char utf16Bytes = ref span.DangerousGetPinnableReference();
var idx = 0;

// Write the last group
Unsafe.Add(ref utf16Bytes, idx++) = Seperator;
idx += FormattingHelpers.WriteDigits(lastGroup, 3, ref utf16Bytes, idx);

// Write out the trailing zeros
if (precision > 0)
{
Unsafe.Add(ref utf16Bytes, idx++) = Period;
idx += FormattingHelpers.WriteDigits(0, precision, ref utf16Bytes, idx);
}

bytesWritten += extraChars * sizeof(char);
return true;
}

public static bool TryFormatHexUInt64(ulong value, byte precision, bool useLower, Span<byte> buffer, out int bytesWritten)
{
const string HexTableLower = "0123456789abcdef";
const string HexTableUpper = "0123456789ABCDEF";

var digits = 1;
var v = value;
if (v > 0xFFFFFFFF)
{
digits += 8;
v >>= 0x20;
}
if (v > 0xFFFF)
{
digits += 4;
v >>= 0x10;
}
if (v > 0xFF)
{
digits += 2;
v >>= 0x8;
}
if (v > 0xF) digits++;

int paddingCount = (precision == ParsedFormat.NoPrecision) ? 0 : precision - digits;
if (paddingCount < 0) paddingCount = 0;

int charsNeeded = digits + paddingCount;
Span<char> span = buffer.NonPortableCast<byte, char>();

if (span.Length < charsNeeded)
{
bytesWritten = 0;
return false;
}

string hexTable = useLower ? HexTableLower : HexTableUpper;
ref char utf16Bytes = ref span.DangerousGetPinnableReference();
int idx = charsNeeded;

for (v = value; digits-- > 0; v >>= 4)
Unsafe.Add(ref utf16Bytes, --idx) = hexTable[(int)(v & 0xF)];

while (paddingCount-- > 0)
Unsafe.Add(ref utf16Bytes, --idx) = '0';

bytesWritten = charsNeeded * sizeof(char);
return true;
}
}
}

+ 250
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/IntegerFormatter.Utf8.cs View File

@@ -0,0 +1,250 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Text
{
internal static class InvariantUtf8IntegerFormatter
{
private const byte Minus = (byte)'-';
private const byte Period = (byte)'.';
private const byte Seperator = (byte)',';

// Invariant formatting uses groups of 3 for each number group seperated by commas.
// ex. 1,234,567,890
private const int GroupSize = 3;

public static bool TryFormatDecimalInt64(long value, byte precision, Span<byte> buffer, out int bytesWritten)
{
int digitCount = FormattingHelpers.CountDigits(value);
int bytesNeeded = digitCount + (int)((value >> 63) & 1);

if (buffer.Length < bytesNeeded)
{
bytesWritten = 0;
return false;
}

ref byte utf8Bytes = ref buffer.DangerousGetPinnableReference();
int idx = 0;

if (value < 0)
{
Unsafe.Add(ref utf8Bytes, idx++) = Minus;

// Abs(long.MinValue) == long.MaxValue + 1, so we need to re-route to unsigned to handle value
if (value == long.MinValue)
{
if (!TryFormatDecimalUInt64((ulong)long.MaxValue + 1, precision, buffer.Slice(1), out bytesWritten))
return false;

bytesWritten += 1; // Add the minus sign
return true;
}

value = -value;
}

if (precision != ParsedFormat.NoPrecision)
{
int leadingZeros = (int)precision - digitCount;
while (leadingZeros-- > 0)
Unsafe.Add(ref utf8Bytes, idx++) = (byte)'0';
}

idx += FormattingHelpers.WriteDigits(value, digitCount, ref utf8Bytes, idx);

bytesWritten = idx;
return true;
}

public static bool TryFormatDecimalUInt64(ulong value, byte precision, Span<byte> buffer, out int bytesWritten)
{
if (value <= long.MaxValue)
return TryFormatDecimalInt64((long)value, precision, buffer, out bytesWritten);

// Remove a single digit from the number. This will get it below long.MaxValue
// Then we call the faster long version and follow-up with writing the last
// digit. This ends up being faster by a factor of 2 than to just do the entire
// operation using the unsigned versions.
value = FormattingHelpers.DivMod(value, 10, out ulong lastDigit);

if (precision != ParsedFormat.NoPrecision && precision > 0)
precision -= 1;

if (!TryFormatDecimalInt64((long)value, precision, buffer, out bytesWritten))
return false;

if (buffer.Length - 1 < bytesWritten)
{
bytesWritten = 0;
return false;
}

ref byte utf8Bytes = ref buffer.DangerousGetPinnableReference();
bytesWritten += FormattingHelpers.WriteDigits(lastDigit, 1, ref utf8Bytes, bytesWritten);
return true;
}

public static bool TryFormatNumericInt64(long value, byte precision, Span<byte> buffer, out int bytesWritten)
{
int digitCount = FormattingHelpers.CountDigits(value);
int groupSeperators = (int)FormattingHelpers.DivMod(digitCount, GroupSize, out long firstGroup);
if (firstGroup == 0)
{
firstGroup = 3;
groupSeperators--;
}

int trailingZeros = (precision == ParsedFormat.NoPrecision) ? 2 : precision;
int idx = (int)((value >> 63) & 1) + digitCount + groupSeperators;

bytesWritten = idx;
if (trailingZeros > 0)
bytesWritten += trailingZeros + 1; // +1 for period.

if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

ref byte utf8Bytes = ref buffer.DangerousGetPinnableReference();
long v = value;

if (v < 0)
{
Unsafe.Add(ref utf8Bytes, 0) = Minus;

// Abs(long.MinValue) == long.MaxValue + 1, so we need to re-route to unsigned to handle value
if (v == long.MinValue)
{
if (!TryFormatNumericUInt64((ulong)long.MaxValue + 1, precision, buffer.Slice(1), out bytesWritten))
return false;

bytesWritten += 1; // Add the minus sign
return true;
}

v = -v;
}

// Write out the trailing zeros
if (trailingZeros > 0)
{
Unsafe.Add(ref utf8Bytes, idx) = Period;
FormattingHelpers.WriteDigits(0, trailingZeros, ref utf8Bytes, idx + 1);
}

// Starting from the back, write each group of digits except the first group
while (digitCount > 3)
{
digitCount -= 3;
idx -= 3;
v = FormattingHelpers.DivMod(v, 1000, out long groupValue);
FormattingHelpers.WriteDigits(groupValue, 3, ref utf8Bytes, idx);
Unsafe.Add(ref utf8Bytes, --idx) = Seperator;
}

// Write the first group of digits.
FormattingHelpers.WriteDigits(v, (int)firstGroup, ref utf8Bytes, idx - (int)firstGroup);

return true;
}

public static bool TryFormatNumericUInt64(ulong value, byte precision, Span<byte> buffer, out int bytesWritten)
{
if (value <= long.MaxValue)
return TryFormatNumericInt64((long)value, precision, buffer, out bytesWritten);

// The ulong path is much slower than the long path here, so we are doing the last group
// inside this method plus the zero padding but routing to the long version for the rest.
value = FormattingHelpers.DivMod(value, 1000, out ulong lastGroup);

if (!TryFormatNumericInt64((long)value, 0, buffer, out bytesWritten))
return false;

if (precision == ParsedFormat.NoPrecision)
precision = 2;

int idx = bytesWritten;

// Since this method routes entirely to the long version if the number is smaller than
// long.MaxValue, we are guaranteed to need to write 3 more digits here before the set
// of trailing zeros.

bytesWritten += 4; // 3 digits + group seperator
if (precision > 0)
bytesWritten += precision + 1; // +1 for period.

if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

ref byte utf8Bytes = ref buffer.DangerousGetPinnableReference();

// Write the last group
Unsafe.Add(ref utf8Bytes, idx++) = Seperator;
idx += FormattingHelpers.WriteDigits(lastGroup, 3, ref utf8Bytes, idx);

// Write out the trailing zeros
if (precision > 0)
{
Unsafe.Add(ref utf8Bytes, idx) = Period;
FormattingHelpers.WriteDigits(0, precision, ref utf8Bytes, idx + 1);
}

return true;
}

public static bool TryFormatHexUInt64(ulong value, byte precision, bool useLower, Span<byte> buffer, out int bytesWritten)
{
const string HexTableLower = "0123456789abcdef";
const string HexTableUpper = "0123456789ABCDEF";

var digits = 1;
var v = value;
if (v > 0xFFFFFFFF)
{
digits += 8;
v >>= 0x20;
}
if (v > 0xFFFF)
{
digits += 4;
v >>= 0x10;
}
if (v > 0xFF)
{
digits += 2;
v >>= 0x8;
}
if (v > 0xF) digits++;

int paddingCount = (precision == ParsedFormat.NoPrecision) ? 0 : precision - digits;
if (paddingCount < 0) paddingCount = 0;

bytesWritten = digits + paddingCount;
if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

string hexTable = useLower ? HexTableLower : HexTableUpper;
ref byte utf8Bytes = ref buffer.DangerousGetPinnableReference();
int idx = bytesWritten;

for (v = value; digits-- > 0; v >>= 4)
Unsafe.Add(ref utf8Bytes, --idx) = (byte)hexTable[(int)(v & 0xF)];

while (paddingCount-- > 0)
Unsafe.Add(ref utf8Bytes, --idx) = (byte)'0';

return true;
}
}
}

+ 497
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/IntegerFormatter.cs View File

@@ -0,0 +1,497 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Diagnostics;

namespace System.Text
{
internal static class IntegerFormatter
{
internal static bool TryFormatInt64(long value, ulong mask, Span<byte> buffer, out int bytesWritten, ParsedFormat format, SymbolTable symbolTable)
{
if (value >= 0)
{
return TryFormatUInt64(unchecked((ulong)value), buffer, out bytesWritten, format, symbolTable);
}
else if (format.Symbol == 'x' || format.Symbol == 'X')
{
return TryFormatUInt64(unchecked((ulong)value) & mask, buffer, out bytesWritten, format, symbolTable);
}
else
{
int minusSignBytes = 0;
if (!symbolTable.TryEncode(SymbolTable.Symbol.MinusSign, buffer, out minusSignBytes))
{
bytesWritten = 0;
return false;
}

int digitBytes = 0;
if (!TryFormatUInt64(unchecked((ulong)-value), buffer.Slice(minusSignBytes), out digitBytes, format, symbolTable))
{
bytesWritten = 0;
return false;
}
bytesWritten = digitBytes + minusSignBytes;
return true;
}
}

internal static bool TryFormatUInt64(ulong value, Span<byte> buffer, out int bytesWritten, ParsedFormat format, SymbolTable symbolTable)
{
switch (format.Symbol)
{
case 'x':
case 'X':
if (symbolTable == SymbolTable.InvariantUtf8)
return TryFormatHexadecimalInvariantCultureUtf8(value, buffer, out bytesWritten, format);
else if (symbolTable == SymbolTable.InvariantUtf16)
return TryFormatHexadecimalInvariantCultureUtf16(value, buffer, out bytesWritten, format);
else
throw new NotSupportedException();

case 'd':
case 'D':
case 'g':
case 'G':
if (symbolTable == SymbolTable.InvariantUtf8)
return TryFormatDecimalInvariantCultureUtf8(value, buffer, out bytesWritten, format);
else if (symbolTable == SymbolTable.InvariantUtf16)
return TryFormatDecimalInvariantCultureUtf16(value, buffer, out bytesWritten, format);
else
return TryFormatDecimal(value, buffer, out bytesWritten, format, symbolTable);

case 'n':
case 'N':
return TryFormatDecimal(value, buffer, out bytesWritten, format, symbolTable);

default:
throw new FormatException();
}
}

private static bool TryFormatDecimalInvariantCultureUtf16(ulong value, Span<byte> buffer, out int bytesWritten, ParsedFormat format)
{
char symbol = char.ToUpperInvariant(format.Symbol);
Precondition.Require(symbol == 'D' || symbol == 'G');

// Count digits
var valueToCountDigits = value;
var digitsCount = 1;
while (valueToCountDigits >= 10UL)
{
valueToCountDigits = valueToCountDigits / 10UL;
digitsCount++;
}

var index = 0;
var bytesCount = digitsCount * 2;

// If format is D and precision is greater than digits count, append leading zeros
if ((symbol == 'D') && format.HasPrecision)
{
var leadingZerosCount = format.Precision - digitsCount;
if (leadingZerosCount > 0)
{
bytesCount += leadingZerosCount * 2;
}

if (bytesCount > buffer.Length)
{
bytesWritten = 0;
return false;
}

while (leadingZerosCount-- > 0)
{
buffer[index++] = (byte)'0';
buffer[index++] = 0;
}
}
else if (bytesCount > buffer.Length)
{
bytesWritten = 0;
return false;
}

index = bytesCount;
while (digitsCount-- > 0)
{
ulong digit = value % 10UL;
value /= 10UL;
buffer[--index] = 0;
buffer[--index] = (byte)(digit + (ulong)'0');
}

bytesWritten = bytesCount;
return true;
}

private static bool TryFormatDecimalInvariantCultureUtf8(ulong value, Span<byte> buffer, out int bytesWritten, ParsedFormat format)
{
char symbol = char.ToUpperInvariant(format.Symbol);
Precondition.Require(symbol == 'D' || symbol == 'G');

// Count digits
var valueToCountDigits = value;
var digitsCount = 1;
while (valueToCountDigits >= 10UL)
{
valueToCountDigits = valueToCountDigits / 10UL;
digitsCount++;
}

var index = 0;
var bytesCount = digitsCount;

// If format is D and precision is greater than digits count, append leading zeros
if ((symbol == 'D') && format.HasPrecision)
{
var leadingZerosCount = format.Precision - digitsCount;
if (leadingZerosCount > 0)
{
bytesCount += leadingZerosCount;
}

if (bytesCount > buffer.Length)
{
bytesWritten = 0;
return false;
}

while (leadingZerosCount-- > 0)
{
buffer[index++] = (byte)'0';
}
}
else if (bytesCount > buffer.Length)
{
bytesWritten = 0;
return false;
}

index = bytesCount;
while (digitsCount-- > 0)
{
ulong digit = value % 10UL;
value /= 10UL;
buffer[--index] = (byte)(digit + (ulong)'0');
}

bytesWritten = bytesCount;
return true;
}

private static bool TryFormatHexadecimalInvariantCultureUtf16(ulong value, Span<byte> buffer, out int bytesWritten, ParsedFormat format)
{
Precondition.Require(format.Symbol == 'X' || format.Symbol == 'x');

byte firstDigitOffset = (byte)'0';
byte firstHexCharOffset = format.Symbol == 'x' ? (byte)'a' : (byte)'A';
firstHexCharOffset -= 10;

// Count amount of hex digits
var hexDigitsCount = 1;
ulong valueToCount = value;
if (valueToCount > 0xFFFFFFFF)
{
hexDigitsCount += 8;
valueToCount >>= 0x20;
}
if (valueToCount > 0xFFFF)
{
hexDigitsCount += 4;
valueToCount >>= 0x10;
}
if (valueToCount > 0xFF)
{
hexDigitsCount += 2;
valueToCount >>= 0x8;
}
if (valueToCount > 0xF)
{
hexDigitsCount++;
}

var bytesCount = hexDigitsCount * 2;

// Count leading zeros
var leadingZerosCount = format.HasPrecision ? format.Precision - hexDigitsCount : 0;
bytesCount += leadingZerosCount > 0 ? leadingZerosCount * 2 : 0;

if (bytesCount > buffer.Length)
{
bytesWritten = 0;
return false;
}

var index = bytesCount;
while (hexDigitsCount-- > 0)
{
byte digit = (byte)(value & 0xF);
value >>= 0x4;
digit += digit < 10 ? firstDigitOffset : firstHexCharOffset;

buffer[--index] = 0;
buffer[--index] = digit;
}

// Write leading zeros if any
while (leadingZerosCount-- > 0)
{
buffer[--index] = 0;
buffer[--index] = firstDigitOffset;
}

bytesWritten = bytesCount;
return true;
}

private static bool TryFormatHexadecimalInvariantCultureUtf8(ulong value, Span<byte> buffer, out int bytesWritten, ParsedFormat format)
{
Precondition.Require(format.Symbol == 'X' || format.Symbol == 'x');

byte firstDigitOffset = (byte)'0';
byte firstHexCharOffset = format.Symbol == 'X' ? (byte)'A' : (byte)'a';
firstHexCharOffset -= 10;

// Count amount of hex digits
var hexDigitsCount = 1;
ulong valueToCount = value;
if (valueToCount > 0xFFFFFFFF)
{
hexDigitsCount += 8;
valueToCount >>= 0x20;
}
if (valueToCount > 0xFFFF)
{
hexDigitsCount += 4;
valueToCount >>= 0x10;
}
if (valueToCount > 0xFF)
{
hexDigitsCount += 2;
valueToCount >>= 0x8;
}
if (valueToCount > 0xF)
{
hexDigitsCount++;
}

var bytesCount = hexDigitsCount;

// Count leading zeros
var leadingZerosCount = format.HasPrecision ? format.Precision - hexDigitsCount : 0;
bytesCount += leadingZerosCount > 0 ? leadingZerosCount : 0;

if (bytesCount > buffer.Length)
{
bytesWritten = 0;
return false;
}

var index = bytesCount;
while (hexDigitsCount-- > 0)
{
byte digit = (byte)(value & 0xF);
value >>= 0x4;
digit += digit < 10 ? firstDigitOffset : firstHexCharOffset;
buffer[--index] = digit;
}

// Write leading zeros if any
while (leadingZerosCount-- > 0)
{
buffer[--index] = firstDigitOffset;
}

bytesWritten = bytesCount;
return true;
}

// TODO: this whole routine is too slow. It does div and mod twice, which are both costly (especially that some JITs cannot optimize it).
// It does it twice to avoid reversing the formatted buffer, which can be tricky given it should handle arbitrary cultures.
// One optimization I thought we could do is to do div/mod once and store digits in a temp buffer (but that would allocate). Modification to the idea would be to store the digits in a local struct
// Another idea possibly worth tying would be to special case cultures that have constant digit size, and go back to the format + reverse buffer approach.
private static bool TryFormatDecimal(ulong value, Span<byte> buffer, out int bytesWritten, ParsedFormat format, SymbolTable symbolTable)
{
char symbol = char.ToUpperInvariant(format.Symbol);
Precondition.Require(symbol == 'D' || format.Symbol == 'G' || format.Symbol == 'N');

// Reverse value on decimal basis, count digits and trailing zeros before the decimal separator
ulong reversedValueExceptFirst = 0;
var digitsCount = 1;
var trailingZerosCount = 0;

// We reverse the digits in numeric form because reversing encoded digits is hard and/or costly.
// If value contains 20 digits, its reversed value will not fit into ulong size.
// So reverse it till last digit (reversedValueExceptFirst will have all the digits except the first one).
while (value >= 10)
{
var digit = value % 10UL;
value = value / 10UL;

if (reversedValueExceptFirst == 0 && digit == 0)
{
trailingZerosCount++;
}
else
{
reversedValueExceptFirst = reversedValueExceptFirst * 10UL + digit;
digitsCount++;
}
}

bytesWritten = 0;
int digitBytes;
// If format is D and precision is greater than digitsCount + trailingZerosCount, append leading zeros
if (symbol == 'D' && format.HasPrecision)
{
var leadingZerosCount = format.Precision - digitsCount - trailingZerosCount;
while (leadingZerosCount-- > 0)
{
if (!symbolTable.TryEncode(SymbolTable.Symbol.D0, buffer.Slice(bytesWritten), out digitBytes))
{
bytesWritten = 0;
return false;
}
bytesWritten += digitBytes;
}
}

// Append first digit
if (!symbolTable.TryEncode((SymbolTable.Symbol)value, buffer.Slice(bytesWritten), out digitBytes))
{
bytesWritten = 0;
return false;
}
bytesWritten += digitBytes;
digitsCount--;

if (symbol == 'N')
{
const int GroupSize = 3;

// Count amount of digits before first group separator. It will be reset to groupSize every time digitsLeftInGroup == zero
var digitsLeftInGroup = (digitsCount + trailingZerosCount) % GroupSize;
if (digitsLeftInGroup == 0)
{
if (digitsCount + trailingZerosCount > 0)
{
// There is a new group immediately after the first digit
if (!symbolTable.TryEncode(SymbolTable.Symbol.GroupSeparator, buffer.Slice(bytesWritten), out digitBytes))
{
bytesWritten = 0;
return false;
}
bytesWritten += digitBytes;
}
digitsLeftInGroup = GroupSize;
}

// Append digits
while (reversedValueExceptFirst > 0)
{
if (digitsLeftInGroup == 0)
{
if (!symbolTable.TryEncode(SymbolTable.Symbol.GroupSeparator, buffer.Slice(bytesWritten), out digitBytes))
{
bytesWritten = 0;
return false;
}
bytesWritten += digitBytes;
digitsLeftInGroup = GroupSize;
}

var nextDigit = reversedValueExceptFirst % 10UL;
reversedValueExceptFirst = reversedValueExceptFirst / 10UL;

if (!symbolTable.TryEncode((SymbolTable.Symbol)nextDigit, buffer.Slice(bytesWritten), out digitBytes))
{
bytesWritten = 0;
return false;
}
bytesWritten += digitBytes;
digitsLeftInGroup--;
}

// Append trailing zeros if any
while (trailingZerosCount-- > 0)
{
if (digitsLeftInGroup == 0)
{
if (!symbolTable.TryEncode(SymbolTable.Symbol.GroupSeparator, buffer.Slice(bytesWritten), out digitBytes))
{
bytesWritten = 0;
return false;
}
bytesWritten += digitBytes;
digitsLeftInGroup = GroupSize;
}

if (!symbolTable.TryEncode(SymbolTable.Symbol.D0, buffer.Slice(bytesWritten), out digitBytes))
{
bytesWritten = 0;
return false;
}
bytesWritten += digitBytes;
digitsLeftInGroup--;
}
}
else
{
while (reversedValueExceptFirst > 0)
{
var bufferSlice = buffer.Slice(bytesWritten);
var nextDigit = reversedValueExceptFirst % 10UL;
reversedValueExceptFirst = reversedValueExceptFirst / 10UL;
if (!symbolTable.TryEncode((SymbolTable.Symbol)nextDigit, bufferSlice, out digitBytes))
{
bytesWritten = 0;
return false;
}
bytesWritten += digitBytes;
}

// Append trailing zeros if any
while (trailingZerosCount-- > 0)
{
if (!symbolTable.TryEncode(SymbolTable.Symbol.D0, buffer.Slice(bytesWritten), out digitBytes))
{
bytesWritten = 0;
return false;
}
bytesWritten += digitBytes;
}
}

// If format is N and precision is not defined or is greater than zero, append trailing zeros after decimal point
if (symbol == 'N')
{
int trailingZerosAfterDecimalCount = format.HasPrecision ? format.Precision : 2;

if (trailingZerosAfterDecimalCount > 0)
{
if (!symbolTable.TryEncode(SymbolTable.Symbol.DecimalSeparator, buffer.Slice(bytesWritten), out digitBytes))
{
bytesWritten = 0;
return false;
}
bytesWritten += digitBytes;

while (trailingZerosAfterDecimalCount-- > 0)
{
if (!symbolTable.TryEncode(SymbolTable.Symbol.D0, buffer.Slice(bytesWritten), out digitBytes))
{
bytesWritten = 0;
return false;
}
bytesWritten += digitBytes;
}
}
}

return true;
}
}
}

+ 402
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/TimeFormatter.Utf16.cs View File

@@ -0,0 +1,402 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Text
{
internal static class InvariantUtf16TimeFormatter
{
#region Constants

private const int DefaultFractionDigits = 7;

private const char Colon = ':';
private const char Comma = ',';
private const char Minus = '-';
private const char Period = '.';
private const char Plus = '+';
private const char Slash = '/';
private const char Space = ' ';

private const char TimeMarker = 'T';
private const char UtcMarker = 'Z';

private const char GMT1 = 'G';
private const char GMT2 = 'M';
private const char GMT3 = 'T';

private const char GMT1Lowercase = 'g';
private const char GMT2Lowercase = 'm';
private const char GMT3Lowercase = 't';

private static readonly char[][] DayAbbreviations = new char[][]
{
new char[] { 'S', 'u', 'n' },
new char[] { 'M', 'o', 'n' },
new char[] { 'T', 'u', 'e' },
new char[] { 'W', 'e', 'd' },
new char[] { 'T', 'h', 'u' },
new char[] { 'F', 'r', 'i' },
new char[] { 'S', 'a', 't' },
};

private static readonly char[][] MonthAbbreviations = new char[][]
{
new char[] { 'J', 'a', 'n' },
new char[] { 'F', 'e', 'b' },
new char[] { 'M', 'a', 'r' },
new char[] { 'A', 'p', 'r' },
new char[] { 'M', 'a', 'y' },
new char[] { 'J', 'u', 'n' },
new char[] { 'J', 'u', 'l' },
new char[] { 'A', 'u', 'g' },
new char[] { 'S', 'e', 'p' },
new char[] { 'O', 'c', 't' },
new char[] { 'N', 'o', 'v' },
new char[] { 'D', 'e', 'c' },
};

private static readonly char[][] DayAbbreviationsLowercase = new char[][]
{
new char[] { 's', 'u', 'n' },
new char[] { 'm', 'o', 'n' },
new char[] { 't', 'u', 'e' },
new char[] { 'w', 'e', 'd' },
new char[] { 't', 'h', 'u' },
new char[] { 'f', 'r', 'i' },
new char[] { 's', 'a', 't' },
};

private static readonly char[][] MonthAbbreviationsLowercase = new char[][]
{
new char[] { 'j', 'a', 'n' },
new char[] { 'f', 'e', 'b' },
new char[] { 'm', 'a', 'r' },
new char[] { 'a', 'p', 'r' },
new char[] { 'm', 'a', 'y' },
new char[] { 'j', 'u', 'n' },
new char[] { 'j', 'u', 'l' },
new char[] { 'a', 'u', 'g' },
new char[] { 's', 'e', 'p' },
new char[] { 'o', 'c', 't' },
new char[] { 'n', 'o', 'v' },
new char[] { 'd', 'e', 'c' },
};

#endregion Constants

public static bool TryFormatG(DateTime value, TimeSpan offset, Span<byte> buffer, out int bytesWritten)
{
const int MinimumCharsNeeded = 19;

int charsNeeded = MinimumCharsNeeded;
if (offset != PrimitiveFormatter.NullOffset)
{
charsNeeded += 7; // Space['+'|'-']hh:ss
}

bytesWritten = charsNeeded * sizeof(char);
if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

Span<char> dst = buffer.NonPortableCast<byte, char>();
ref char utf16Bytes = ref dst.DangerousGetPinnableReference();

FormattingHelpers.WriteDigits(value.Month, 2, ref utf16Bytes, 0);
Unsafe.Add(ref utf16Bytes, 2) = Slash;

FormattingHelpers.WriteDigits(value.Day, 2, ref utf16Bytes, 3);
Unsafe.Add(ref utf16Bytes, 5) = Slash;

FormattingHelpers.WriteDigits(value.Year, 4, ref utf16Bytes, 6);
Unsafe.Add(ref utf16Bytes, 10) = Space;

FormattingHelpers.WriteDigits(value.Hour, 2, ref utf16Bytes, 11);
Unsafe.Add(ref utf16Bytes, 13) = Colon;

FormattingHelpers.WriteDigits(value.Minute, 2, ref utf16Bytes, 14);
Unsafe.Add(ref utf16Bytes, 16) = Colon;

FormattingHelpers.WriteDigits(value.Second, 2, ref utf16Bytes, 17);

if (offset != PrimitiveFormatter.NullOffset)
{
Unsafe.Add(ref utf16Bytes, 19) = Space;

long ticks = value.Ticks;
if (ticks < 0)
{
Unsafe.Add(ref utf16Bytes, 20) = Minus;
ticks = -ticks;
}
else
{
Unsafe.Add(ref utf16Bytes, 20) = Plus;
}

FormattingHelpers.WriteDigits(offset.Hours, 2, ref utf16Bytes, 21);
Unsafe.Add(ref utf16Bytes, 23) = Colon;
FormattingHelpers.WriteDigits(offset.Minutes, 2, ref utf16Bytes, 24);
}

return true;
}

public static bool TryFormatO(DateTime value, TimeSpan offset, Span<byte> buffer, out int bytesWritten)
{
const int MinimumCharsNeeded = 27;

int charsNeeded = MinimumCharsNeeded;
DateTimeKind kind = DateTimeKind.Local;

if (offset == PrimitiveFormatter.NullOffset)
{
kind = value.Kind;
if (kind == DateTimeKind.Local)
{
offset = TimeZoneInfo.Local.GetUtcOffset(value);
charsNeeded += 6;
}
else if (kind == DateTimeKind.Utc)
{
charsNeeded += 1;
}
}
else
{
charsNeeded += 6;
}

bytesWritten = charsNeeded * sizeof(char);
if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

Span<char> dst = buffer.NonPortableCast<byte, char>();
ref char utf16Bytes = ref dst.DangerousGetPinnableReference();

FormattingHelpers.WriteDigits(value.Year, 4, ref utf16Bytes, 0);
Unsafe.Add(ref utf16Bytes, 4) = Minus;

FormattingHelpers.WriteDigits(value.Month, 2, ref utf16Bytes, 5);
Unsafe.Add(ref utf16Bytes, 7) = Minus;

FormattingHelpers.WriteDigits(value.Day, 2, ref utf16Bytes, 8);
Unsafe.Add(ref utf16Bytes, 10) = TimeMarker;

FormattingHelpers.WriteDigits(value.Hour, 2, ref utf16Bytes, 11);
Unsafe.Add(ref utf16Bytes, 13) = Colon;

FormattingHelpers.WriteDigits(value.Minute, 2, ref utf16Bytes, 14);
Unsafe.Add(ref utf16Bytes, 16) = Colon;

FormattingHelpers.WriteDigits(value.Second, 2, ref utf16Bytes, 17);
Unsafe.Add(ref utf16Bytes, 19) = Period;

FormattingHelpers.DivMod(value.Ticks, TimeSpan.TicksPerSecond, out long fraction);
FormattingHelpers.WriteFractionDigits(fraction, DefaultFractionDigits, ref utf16Bytes, 20);

if (kind == DateTimeKind.Local)
{
int hours = offset.Hours;
char sign = Plus;

if (offset.Hours < 0)
{
hours = -offset.Hours;
sign = Minus;
}

Unsafe.Add(ref utf16Bytes, 27) = sign;
FormattingHelpers.WriteDigits(hours, 2, ref utf16Bytes, 28);
Unsafe.Add(ref utf16Bytes, 30) = Colon;
FormattingHelpers.WriteDigits(offset.Minutes, 2, ref utf16Bytes, 31);
}
else if (kind == DateTimeKind.Utc)
{
Unsafe.Add(ref utf16Bytes, 27) = UtcMarker;
}

return true;
}

public static bool TryFormatRfc1123(DateTime value, Span<byte> buffer, out int bytesWritten)
{
const int CharsNeeded = 29;

bytesWritten = CharsNeeded * sizeof(char);
if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

Span<char> dst = buffer.NonPortableCast<byte, char>();
ref char utf16Bytes = ref dst.DangerousGetPinnableReference();

var dayAbbrev = DayAbbreviations[(int)value.DayOfWeek];
Unsafe.Add(ref utf16Bytes, 0) = dayAbbrev[0];
Unsafe.Add(ref utf16Bytes, 1) = dayAbbrev[1];
Unsafe.Add(ref utf16Bytes, 2) = dayAbbrev[2];
Unsafe.Add(ref utf16Bytes, 3) = Comma;
Unsafe.Add(ref utf16Bytes, 4) = Space;

FormattingHelpers.WriteDigits(value.Day, 2, ref utf16Bytes, 5);
Unsafe.Add(ref utf16Bytes, 7) = ' ';

var monthAbbrev = MonthAbbreviations[value.Month - 1];
Unsafe.Add(ref utf16Bytes, 8) = monthAbbrev[0];
Unsafe.Add(ref utf16Bytes, 9) = monthAbbrev[1];
Unsafe.Add(ref utf16Bytes, 10) = monthAbbrev[2];
Unsafe.Add(ref utf16Bytes, 11) = Space;

FormattingHelpers.WriteDigits(value.Year, 4, ref utf16Bytes, 12);
Unsafe.Add(ref utf16Bytes, 16) = Space;

FormattingHelpers.WriteDigits(value.Hour, 2, ref utf16Bytes, 17);
Unsafe.Add(ref utf16Bytes, 19) = Colon;

FormattingHelpers.WriteDigits(value.Minute, 2, ref utf16Bytes, 20);
Unsafe.Add(ref utf16Bytes, 22) = Colon;

FormattingHelpers.WriteDigits(value.Second, 2, ref utf16Bytes, 23);
Unsafe.Add(ref utf16Bytes, 25) = Space;

Unsafe.Add(ref utf16Bytes, 26) = GMT1;
Unsafe.Add(ref utf16Bytes, 27) = GMT2;
Unsafe.Add(ref utf16Bytes, 28) = GMT3;

return true;
}

public static bool TryFormatRfc1123Lowercase(DateTime value, Span<byte> buffer, out int bytesWritten)
{
const int CharsNeeded = 29;

bytesWritten = CharsNeeded * sizeof(char);
if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

Span<char> dst = buffer.NonPortableCast<byte, char>();
ref char utf16Bytes = ref dst.DangerousGetPinnableReference();

var dayAbbrev = DayAbbreviationsLowercase[(int)value.DayOfWeek];
Unsafe.Add(ref utf16Bytes, 0) = dayAbbrev[0];
Unsafe.Add(ref utf16Bytes, 1) = dayAbbrev[1];
Unsafe.Add(ref utf16Bytes, 2) = dayAbbrev[2];
Unsafe.Add(ref utf16Bytes, 3) = Comma;
Unsafe.Add(ref utf16Bytes, 4) = Space;

FormattingHelpers.WriteDigits(value.Day, 2, ref utf16Bytes, 5);
Unsafe.Add(ref utf16Bytes, 7) = ' ';

var monthAbbrev = MonthAbbreviationsLowercase[value.Month - 1];
Unsafe.Add(ref utf16Bytes, 8) = monthAbbrev[0];
Unsafe.Add(ref utf16Bytes, 9) = monthAbbrev[1];
Unsafe.Add(ref utf16Bytes, 10) = monthAbbrev[2];
Unsafe.Add(ref utf16Bytes, 11) = Space;

FormattingHelpers.WriteDigits(value.Year, 4, ref utf16Bytes, 12);
Unsafe.Add(ref utf16Bytes, 16) = Space;

FormattingHelpers.WriteDigits(value.Hour, 2, ref utf16Bytes, 17);
Unsafe.Add(ref utf16Bytes, 19) = Colon;

FormattingHelpers.WriteDigits(value.Minute, 2, ref utf16Bytes, 20);
Unsafe.Add(ref utf16Bytes, 22) = Colon;

FormattingHelpers.WriteDigits(value.Second, 2, ref utf16Bytes, 23);
Unsafe.Add(ref utf16Bytes, 25) = Space;

Unsafe.Add(ref utf16Bytes, 26) = GMT1Lowercase;
Unsafe.Add(ref utf16Bytes, 27) = GMT2Lowercase;
Unsafe.Add(ref utf16Bytes, 28) = GMT3Lowercase;

return true;
}

public static bool TryFormat(TimeSpan value, char format, Span<byte> buffer, out int bytesWritten)
{
bool longForm = (format == 'G');
bool constant = (format == 't' || format == 'T' || format == 'c');

long ticks = value.Ticks;
int days = (int)FormattingHelpers.DivMod(ticks, TimeSpan.TicksPerDay, out long timeLeft);

bool showSign = false;
if (ticks < 0)
{
showSign = true;
days = -days;
timeLeft = -timeLeft;
}

int hours = (int)FormattingHelpers.DivMod(timeLeft, TimeSpan.TicksPerHour, out timeLeft);
int minutes = (int)FormattingHelpers.DivMod(timeLeft, TimeSpan.TicksPerMinute, out timeLeft);
int seconds = (int)FormattingHelpers.DivMod(timeLeft, TimeSpan.TicksPerSecond, out long fraction);

int dayDigits = 0;
int hourDigits = (constant || longForm || hours > 9) ? 2 : 1;
int fractionDigits = 0;

bytesWritten = hourDigits + 6; // [h]h:mm:ss
if (showSign)
bytesWritten += 1; // [-]
if (longForm || days > 0)
{
dayDigits = FormattingHelpers.CountDigits(days);
bytesWritten += dayDigits + 1; // [d'.']
}
if (longForm || fraction > 0)
{
fractionDigits = (longForm || constant) ? DefaultFractionDigits : FormattingHelpers.CountFractionDigits(fraction);
bytesWritten += fractionDigits + 1; // ['.'fffffff] or ['.'FFFFFFF] for short-form
}

bytesWritten *= sizeof(char);
if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

Span<char> dst = buffer.NonPortableCast<byte, char>();
ref char utf16Bytes = ref dst.DangerousGetPinnableReference();
int idx = 0;

if (showSign)
Unsafe.Add(ref utf16Bytes, idx++) = Minus;

if (dayDigits > 0)
{
idx += FormattingHelpers.WriteDigits(days, dayDigits, ref utf16Bytes, idx);
Unsafe.Add(ref utf16Bytes, idx++) = constant ? Period : Colon;
}

idx += FormattingHelpers.WriteDigits(hours, hourDigits, ref utf16Bytes, idx);
Unsafe.Add(ref utf16Bytes, idx++) = Colon;

idx += FormattingHelpers.WriteDigits(minutes, 2, ref utf16Bytes, idx);
Unsafe.Add(ref utf16Bytes, idx++) = Colon;

idx += FormattingHelpers.WriteDigits(seconds, 2, ref utf16Bytes, idx);

if (fractionDigits > 0)
{
Unsafe.Add(ref utf16Bytes, idx++) = Period;
idx += FormattingHelpers.WriteFractionDigits(fraction, fractionDigits, ref utf16Bytes, idx);
}

return true;
}
}
}

+ 394
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/Internal/TimeFormatter.Utf8.cs View File

@@ -0,0 +1,394 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Text
{
internal static class InvariantUtf8TimeFormatter
{
#region Constants

private const int DefaultFractionDigits = 7;

private const byte Colon = (byte)':';
private const byte Comma = (byte)',';
private const byte Minus = (byte)'-';
private const byte Period = (byte)'.';
private const byte Plus = (byte)'+';
private const byte Slash = (byte)'/';
private const byte Space = (byte)' ';

private const byte TimeMarker = (byte)'T';
private const byte UtcMarker = (byte)'Z';

private const byte GMT1 = (byte)'G';
private const byte GMT2 = (byte)'M';
private const byte GMT3 = (byte)'T';

private const byte GMT1Lowercase = (byte)'g';
private const byte GMT2Lowercase = (byte)'m';
private const byte GMT3Lowercase = (byte)'t';

private static readonly byte[][] DayAbbreviations = new byte[][]
{
new byte[] { (byte)'S', (byte)'u', (byte)'n' },
new byte[] { (byte)'M', (byte)'o', (byte)'n' },
new byte[] { (byte)'T', (byte)'u', (byte)'e' },
new byte[] { (byte)'W', (byte)'e', (byte)'d' },
new byte[] { (byte)'T', (byte)'h', (byte)'u' },
new byte[] { (byte)'F', (byte)'r', (byte)'i' },
new byte[] { (byte)'S', (byte)'a', (byte)'t' },
};

private static readonly byte[][] DayAbbreviationsLowercase = new byte[][]
{
new byte[] { (byte)'s', (byte)'u', (byte)'n' },
new byte[] { (byte)'m', (byte)'o', (byte)'n' },
new byte[] { (byte)'t', (byte)'u', (byte)'e' },
new byte[] { (byte)'w', (byte)'e', (byte)'d' },
new byte[] { (byte)'t', (byte)'h', (byte)'u' },
new byte[] { (byte)'f', (byte)'r', (byte)'i' },
new byte[] { (byte)'s', (byte)'a', (byte)'t' },
};

private static readonly byte[][] MonthAbbreviations = new byte[][]
{
new byte[] { (byte)'J', (byte)'a', (byte)'n' },
new byte[] { (byte)'F', (byte)'e', (byte)'b' },
new byte[] { (byte)'M', (byte)'a', (byte)'r' },
new byte[] { (byte)'A', (byte)'p', (byte)'r' },
new byte[] { (byte)'M', (byte)'a', (byte)'y' },
new byte[] { (byte)'J', (byte)'u', (byte)'n' },
new byte[] { (byte)'J', (byte)'u', (byte)'l' },
new byte[] { (byte)'A', (byte)'u', (byte)'g' },
new byte[] { (byte)'S', (byte)'e', (byte)'p' },
new byte[] { (byte)'O', (byte)'c', (byte)'t' },
new byte[] { (byte)'N', (byte)'o', (byte)'v' },
new byte[] { (byte)'D', (byte)'e', (byte)'c' },
};

private static readonly byte[][] MonthAbbreviationsLowercase = new byte[][]
{
new byte[] { (byte)'j', (byte)'a', (byte)'n' },
new byte[] { (byte)'f', (byte)'e', (byte)'b' },
new byte[] { (byte)'m', (byte)'a', (byte)'r' },
new byte[] { (byte)'a', (byte)'p', (byte)'r' },
new byte[] { (byte)'m', (byte)'a', (byte)'y' },
new byte[] { (byte)'j', (byte)'u', (byte)'n' },
new byte[] { (byte)'j', (byte)'u', (byte)'l' },
new byte[] { (byte)'a', (byte)'u', (byte)'g' },
new byte[] { (byte)'s', (byte)'e', (byte)'p' },
new byte[] { (byte)'o', (byte)'c', (byte)'t' },
new byte[] { (byte)'n', (byte)'o', (byte)'v' },
new byte[] { (byte)'d', (byte)'e', (byte)'c' },
};

#endregion Constants

public static bool TryFormatG(DateTime value, TimeSpan offset, Span<byte> buffer, out int bytesWritten)
{
const int MinimumBytesNeeded = 19;

bytesWritten = MinimumBytesNeeded;
if (offset != PrimitiveFormatter.NullOffset)
{
bytesWritten += 7; // Space['+'|'-']hh:ss
}

if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

ref byte utf8Bytes = ref buffer.DangerousGetPinnableReference();

FormattingHelpers.WriteDigits(value.Month, 2, ref utf8Bytes, 0);
Unsafe.Add(ref utf8Bytes, 2) = Slash;

FormattingHelpers.WriteDigits(value.Day, 2, ref utf8Bytes, 3);
Unsafe.Add(ref utf8Bytes, 5) = Slash;

FormattingHelpers.WriteDigits(value.Year, 4, ref utf8Bytes, 6);
Unsafe.Add(ref utf8Bytes, 10) = Space;

FormattingHelpers.WriteDigits(value.Hour, 2, ref utf8Bytes, 11);
Unsafe.Add(ref utf8Bytes, 13) = Colon;

FormattingHelpers.WriteDigits(value.Minute, 2, ref utf8Bytes, 14);
Unsafe.Add(ref utf8Bytes, 16) = Colon;

FormattingHelpers.WriteDigits(value.Second, 2, ref utf8Bytes, 17);

if (offset != PrimitiveFormatter.NullOffset)
{
Unsafe.Add(ref utf8Bytes, 19) = Space;

long ticks = value.Ticks;
if (ticks < 0)
{
Unsafe.Add(ref utf8Bytes, 20) = Minus;
ticks = -ticks;
}
else
{
Unsafe.Add(ref utf8Bytes, 20) = Plus;
}

FormattingHelpers.WriteDigits(offset.Hours, 2, ref utf8Bytes, 21);
Unsafe.Add(ref utf8Bytes, 23) = Colon;
FormattingHelpers.WriteDigits(offset.Minutes, 2, ref utf8Bytes, 24);
}

return true;
}

public static bool TryFormatO(DateTime value, TimeSpan offset, Span<byte> buffer, out int bytesWritten)
{
const int MinimumBytesNeeded = 27;

bytesWritten = MinimumBytesNeeded;
DateTimeKind kind = DateTimeKind.Local;

if (offset == PrimitiveFormatter.NullOffset)
{
kind = value.Kind;
if (kind == DateTimeKind.Local)
{
offset = TimeZoneInfo.Local.GetUtcOffset(value);
bytesWritten += 6;
}
else if (kind == DateTimeKind.Utc)
{
bytesWritten += 1;
}
}
else
{
bytesWritten += 6;
}

if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

ref byte utf8Bytes = ref buffer.DangerousGetPinnableReference();

FormattingHelpers.WriteDigits(value.Year, 4, ref utf8Bytes, 0);
Unsafe.Add(ref utf8Bytes, 4) = Minus;

FormattingHelpers.WriteDigits(value.Month, 2, ref utf8Bytes, 5);
Unsafe.Add(ref utf8Bytes, 7) = Minus;

FormattingHelpers.WriteDigits(value.Day, 2, ref utf8Bytes, 8);
Unsafe.Add(ref utf8Bytes, 10) = TimeMarker;

FormattingHelpers.WriteDigits(value.Hour, 2, ref utf8Bytes, 11);
Unsafe.Add(ref utf8Bytes, 13) = Colon;

FormattingHelpers.WriteDigits(value.Minute, 2, ref utf8Bytes, 14);
Unsafe.Add(ref utf8Bytes, 16) = Colon;

FormattingHelpers.WriteDigits(value.Second, 2, ref utf8Bytes, 17);
Unsafe.Add(ref utf8Bytes, 19) = Period;

FormattingHelpers.DivMod(value.Ticks, TimeSpan.TicksPerSecond, out long fraction);
FormattingHelpers.WriteDigits(fraction, DefaultFractionDigits, ref utf8Bytes, 20);

if (kind == DateTimeKind.Local)
{
int hours = offset.Hours;
byte sign = Plus;

if (offset.Hours < 0)
{
hours = -offset.Hours;
sign = Minus;
}

Unsafe.Add(ref utf8Bytes, 27) = sign;
FormattingHelpers.WriteDigits(hours, 2, ref utf8Bytes, 28);
Unsafe.Add(ref utf8Bytes, 30) = Colon;
FormattingHelpers.WriteDigits(offset.Minutes, 2, ref utf8Bytes, 31);
}
else if (kind == DateTimeKind.Utc)
{
Unsafe.Add(ref utf8Bytes, 27) = UtcMarker;
}

return true;
}

public static bool TryFormatRfc1123(DateTime value, Span<byte> buffer, out int bytesWritten)
{
const int BytesNeeded = 29;

bytesWritten = BytesNeeded;
if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

ref byte utf8Bytes = ref buffer.DangerousGetPinnableReference();

var dayAbbrev = DayAbbreviations[(int)value.DayOfWeek];
Unsafe.Add(ref utf8Bytes, 0) = dayAbbrev[0];
Unsafe.Add(ref utf8Bytes, 1) = dayAbbrev[1];
Unsafe.Add(ref utf8Bytes, 2) = dayAbbrev[2];
Unsafe.Add(ref utf8Bytes, 3) = Comma;
Unsafe.Add(ref utf8Bytes, 4) = Space;

FormattingHelpers.WriteDigits(value.Day, 2, ref utf8Bytes, 5);
Unsafe.Add(ref utf8Bytes, 7) = (byte)' ';

var monthAbbrev = MonthAbbreviations[value.Month - 1];
Unsafe.Add(ref utf8Bytes, 8) = monthAbbrev[0];
Unsafe.Add(ref utf8Bytes, 9) = monthAbbrev[1];
Unsafe.Add(ref utf8Bytes, 10) = monthAbbrev[2];
Unsafe.Add(ref utf8Bytes, 11) = Space;

FormattingHelpers.WriteDigits(value.Year, 4, ref utf8Bytes, 12);
Unsafe.Add(ref utf8Bytes, 16) = Space;

FormattingHelpers.WriteDigits(value.Hour, 2, ref utf8Bytes, 17);
Unsafe.Add(ref utf8Bytes, 19) = Colon;

FormattingHelpers.WriteDigits(value.Minute, 2, ref utf8Bytes, 20);
Unsafe.Add(ref utf8Bytes, 22) = Colon;

FormattingHelpers.WriteDigits(value.Second, 2, ref utf8Bytes, 23);
Unsafe.Add(ref utf8Bytes, 25) = Space;

Unsafe.Add(ref utf8Bytes, 26) = GMT1;
Unsafe.Add(ref utf8Bytes, 27) = GMT2;
Unsafe.Add(ref utf8Bytes, 28) = GMT3;

return true;
}

public static bool TryFormatRfc1123Lowercase(DateTime value, Span<byte> buffer, out int bytesWritten)
{
const int BytesNeeded = 29;

bytesWritten = BytesNeeded;
if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

ref byte utf8Bytes = ref buffer.DangerousGetPinnableReference();

var dayAbbrev = DayAbbreviationsLowercase[(int)value.DayOfWeek];
Unsafe.Add(ref utf8Bytes, 0) = dayAbbrev[0];
Unsafe.Add(ref utf8Bytes, 1) = dayAbbrev[1];
Unsafe.Add(ref utf8Bytes, 2) = dayAbbrev[2];
Unsafe.Add(ref utf8Bytes, 3) = Comma;
Unsafe.Add(ref utf8Bytes, 4) = Space;

FormattingHelpers.WriteDigits(value.Day, 2, ref utf8Bytes, 5);
Unsafe.Add(ref utf8Bytes, 7) = (byte)' ';

var monthAbbrev = MonthAbbreviationsLowercase[value.Month - 1];
Unsafe.Add(ref utf8Bytes, 8) = monthAbbrev[0];
Unsafe.Add(ref utf8Bytes, 9) = monthAbbrev[1];
Unsafe.Add(ref utf8Bytes, 10) = monthAbbrev[2];
Unsafe.Add(ref utf8Bytes, 11) = Space;

FormattingHelpers.WriteDigits(value.Year, 4, ref utf8Bytes, 12);
Unsafe.Add(ref utf8Bytes, 16) = Space;

FormattingHelpers.WriteDigits(value.Hour, 2, ref utf8Bytes, 17);
Unsafe.Add(ref utf8Bytes, 19) = Colon;

FormattingHelpers.WriteDigits(value.Minute, 2, ref utf8Bytes, 20);
Unsafe.Add(ref utf8Bytes, 22) = Colon;

FormattingHelpers.WriteDigits(value.Second, 2, ref utf8Bytes, 23);
Unsafe.Add(ref utf8Bytes, 25) = Space;

Unsafe.Add(ref utf8Bytes, 26) = GMT1Lowercase;
Unsafe.Add(ref utf8Bytes, 27) = GMT2Lowercase;
Unsafe.Add(ref utf8Bytes, 28) = GMT3Lowercase;

return true;
}

public static bool TryFormat(TimeSpan value, char format, Span<byte> buffer, out int bytesWritten)
{
bool longForm = (format == 'G');
bool constant = (format == 't' || format == 'T' || format == 'c');

long ticks = value.Ticks;
int days = (int)FormattingHelpers.DivMod(ticks, TimeSpan.TicksPerDay, out long timeLeft);

bool showSign = false;
if (ticks < 0)
{
showSign = true;
days = -days;
timeLeft = -timeLeft;
}

int hours = (int)FormattingHelpers.DivMod(timeLeft, TimeSpan.TicksPerHour, out timeLeft);
int minutes = (int)FormattingHelpers.DivMod(timeLeft, TimeSpan.TicksPerMinute, out timeLeft);
int seconds = (int)FormattingHelpers.DivMod(timeLeft, TimeSpan.TicksPerSecond, out long fraction);

int dayDigits = 0;
int hourDigits = (constant || longForm || hours > 9) ? 2 : 1;
int fractionDigits = 0;

bytesWritten = hourDigits + 6; // [h]h:mm:ss
if (showSign)
bytesWritten += 1; // [-]
if (longForm || days > 0)
{
dayDigits = FormattingHelpers.CountDigits(days);
bytesWritten += dayDigits + 1; // [d'.']
}
if (longForm || fraction > 0)
{
fractionDigits = (longForm || constant) ? DefaultFractionDigits : FormattingHelpers.CountFractionDigits(fraction);
bytesWritten += fractionDigits + 1; // ['.'fffffff] or ['.'FFFFFFF] for short-form
}

if (buffer.Length < bytesWritten)
{
bytesWritten = 0;
return false;
}

ref byte utf8Bytes = ref buffer.DangerousGetPinnableReference();
int idx = 0;

if (showSign)
Unsafe.Add(ref utf8Bytes, idx++) = Minus;

if (dayDigits > 0)
{
idx += FormattingHelpers.WriteDigits(days, dayDigits, ref utf8Bytes, idx);
Unsafe.Add(ref utf8Bytes, idx++) = constant ? Period : Colon;
}

idx += FormattingHelpers.WriteDigits(hours, hourDigits, ref utf8Bytes, idx);
Unsafe.Add(ref utf8Bytes, idx++) = Colon;

idx += FormattingHelpers.WriteDigits(minutes, 2, ref utf8Bytes, idx);
Unsafe.Add(ref utf8Bytes, idx++) = Colon;

idx += FormattingHelpers.WriteDigits(seconds, 2, ref utf8Bytes, idx);

if (fractionDigits > 0)
{
Unsafe.Add(ref utf8Bytes, idx++) = Period;
idx += FormattingHelpers.WriteFractionDigits(fraction, fractionDigits, ref utf8Bytes, idx);
}

return true;
}
}
}

+ 20
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/PrimitiveFormatter.Guid.cs View File

@@ -0,0 +1,20 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace System.Text
{
public static partial class PrimitiveFormatter
{
public static bool TryFormat(this Guid value, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

if (symbolTable == SymbolTable.InvariantUtf8)
return InvariantUtf8GuidFormatter.TryFormat(value, buffer, out bytesWritten, format);
else if (symbolTable == SymbolTable.InvariantUtf16)
return InvariantUtf16GuidFormatter.TryFormat(value, buffer, out bytesWritten, format);
else
throw new NotImplementedException();
}
}
}

+ 187
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/PrimitiveFormatter.Integer.cs View File

@@ -0,0 +1,187 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace System.Text
{
/// <summary>
/// Pseudo-implementations of IBufferFormattable interface for primitive types
/// </summary>
/// <remarks>
/// Holds extension methods for formatting types that cannot implement IBufferFormattable for layering reasons.
/// </remarks>
public static partial class PrimitiveFormatter
{
public static bool TryFormat(this byte value, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
return TryFormatCore(value, buffer, out bytesWritten, format, symbolTable);
}

public static bool TryFormat(this sbyte value, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
return TryFormatCore(value, 0xff, buffer, out bytesWritten, format, symbolTable);
}

public static bool TryFormat(this ushort value, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
return TryFormatCore(value, buffer, out bytesWritten, format, symbolTable);
}

public static bool TryFormat(this short value, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
return TryFormatCore(value, 0xffff, buffer, out bytesWritten, format, symbolTable);
}

public static bool TryFormat(this uint value, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
return TryFormatCore(value, buffer, out bytesWritten, format, symbolTable);
}

public static bool TryFormat(this int value, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
return TryFormatCore(value, 0xffffffff, buffer, out bytesWritten, format, symbolTable);
}

public static bool TryFormat(this ulong value, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
return TryFormatCore(value, buffer, out bytesWritten, format, symbolTable);
}

public static bool TryFormat(this long value, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
return TryFormatCore(value, 0xffffffffffffffff, buffer, out bytesWritten, format, symbolTable);
}

static bool TryFormatCore(long value, ulong mask, Span<byte> buffer, out int bytesWritten, ParsedFormat format, SymbolTable symbolTable)
{
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

if (format.IsDefault)
{
format = 'G';
}

if (symbolTable == SymbolTable.InvariantUtf8)
return TryFormatInvariantUtf8(value, mask, buffer, out bytesWritten, format);
else if (symbolTable == SymbolTable.InvariantUtf16)
return TryFormatInvariantUtf16(value, mask, buffer, out bytesWritten, format);
else
return IntegerFormatter.TryFormatInt64(value, mask, buffer, out bytesWritten, format, symbolTable);
}

static bool TryFormatCore(ulong value, Span<byte> buffer, out int bytesWritten, ParsedFormat format, SymbolTable symbolTable)
{
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

if (symbolTable == SymbolTable.InvariantUtf8)
return TryFormatInvariantUtf8(value, buffer, out bytesWritten, format);
else if (symbolTable == SymbolTable.InvariantUtf16)
return TryFormatInvariantUtf16(value, buffer, out bytesWritten, format);
else
return IntegerFormatter.TryFormatUInt64(value, buffer, out bytesWritten, format, symbolTable);
}

static bool TryFormatInvariantUtf8(long value, ulong mask, Span<byte> buffer, out int bytesWritten, ParsedFormat format)
{
switch (format.Symbol)
{
case (char)0:
case 'd':
case 'D':
case 'G':
case 'g':
return InvariantUtf8IntegerFormatter.TryFormatDecimalInt64(value, format.Precision, buffer, out bytesWritten);

case 'n':
case 'N':
return InvariantUtf8IntegerFormatter.TryFormatNumericInt64(value, format.Precision, buffer, out bytesWritten);

case 'x':
return InvariantUtf8IntegerFormatter.TryFormatHexUInt64((ulong)value & mask, format.Precision, true, buffer, out bytesWritten);

case 'X':
return InvariantUtf8IntegerFormatter.TryFormatHexUInt64((ulong)value & mask, format.Precision, false, buffer, out bytesWritten);

default:
throw new NotSupportedException();
}
}

static bool TryFormatInvariantUtf8(ulong value, Span<byte> buffer, out int bytesWritten, ParsedFormat format)
{
switch (format.Symbol)
{
case (char)0:
case 'd':
case 'D':
case 'G':
case 'g':
return InvariantUtf8IntegerFormatter.TryFormatDecimalUInt64(value, format.Precision, buffer, out bytesWritten);

case 'n':
case 'N':
return InvariantUtf8IntegerFormatter.TryFormatNumericUInt64(value, format.Precision, buffer, out bytesWritten);

case 'x':
return InvariantUtf8IntegerFormatter.TryFormatHexUInt64(value, format.Precision, true, buffer, out bytesWritten);

case 'X':
return InvariantUtf8IntegerFormatter.TryFormatHexUInt64(value, format.Precision, false, buffer, out bytesWritten);

default:
throw new NotSupportedException();
}
}

static bool TryFormatInvariantUtf16(long value, ulong mask, Span<byte> buffer, out int bytesWritten, ParsedFormat format)
{
switch (format.Symbol)
{
case (char)0:
case 'd':
case 'D':
case 'G':
case 'g':
return InvariantUtf16IntegerFormatter.TryFormatDecimalInt64(value, format.Precision, buffer, out bytesWritten);

case 'n':
case 'N':
return InvariantUtf16IntegerFormatter.TryFormatNumericInt64(value, format.Precision, buffer, out bytesWritten);

case 'x':
return InvariantUtf16IntegerFormatter.TryFormatHexUInt64((ulong)value & mask, format.Precision, true, buffer, out bytesWritten);

case 'X':
return InvariantUtf16IntegerFormatter.TryFormatHexUInt64((ulong)value & mask, format.Precision, false, buffer, out bytesWritten);

default:
throw new NotSupportedException();
}
}

static bool TryFormatInvariantUtf16(ulong value, Span<byte> buffer, out int bytesWritten, ParsedFormat format)
{
switch (format.Symbol)
{
case (char)0:
case 'd':
case 'D':
case 'G':
case 'g':
return InvariantUtf16IntegerFormatter.TryFormatDecimalUInt64(value, format.Precision, buffer, out bytesWritten);

case 'n':
case 'N':
return InvariantUtf16IntegerFormatter.TryFormatNumericUInt64(value, format.Precision, buffer, out bytesWritten);

case 'x':
return InvariantUtf16IntegerFormatter.TryFormatHexUInt64(value, format.Precision, true, buffer, out bytesWritten);

case 'X':
return InvariantUtf16IntegerFormatter.TryFormatHexUInt64(value, format.Precision, false, buffer, out bytesWritten);

default:
throw new NotSupportedException();
}
}
}
}

+ 160
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/PrimitiveFormatter.Time.cs View File

@@ -0,0 +1,160 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Diagnostics;
using System.Runtime.CompilerServices;

namespace System.Text
{
public static partial class PrimitiveFormatter
{
internal static readonly TimeSpan NullOffset = TimeSpan.MinValue;

public static bool TryFormat(this DateTimeOffset value, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
TimeSpan offset = NullOffset;
char symbol = format.Symbol;
if (format.IsDefault)
{
symbol = 'G';
offset = value.Offset;
}

symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

switch (symbol)
{
case 'R':
return TryFormatDateTimeRfc1123(value.UtcDateTime, buffer, out bytesWritten, symbolTable);

case 'l':
return TryFormatDateTimeRfc1123Lowercase(value.UtcDateTime, buffer, out bytesWritten, symbolTable);

case 'O':
return TryFormatDateTimeFormatO(value.DateTime, value.Offset, buffer, out bytesWritten, symbolTable);

case 'G':
return TryFormatDateTimeFormatG(value.DateTime, offset, buffer, out bytesWritten, symbolTable);

default:
ThrowNotImplemented();
bytesWritten = 0;
return false;
}
}

public static bool TryFormat(this DateTime value, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
char symbol = format.IsDefault ? 'G' : format.Symbol;

symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

switch (symbol)
{
case 'R':
return TryFormatDateTimeRfc1123(value, buffer, out bytesWritten, symbolTable);

case 'l':
return TryFormatDateTimeRfc1123Lowercase(value, buffer, out bytesWritten, symbolTable);

case 'O':
return TryFormatDateTimeFormatO(value, NullOffset, buffer, out bytesWritten, symbolTable);

case 'G':
return TryFormatDateTimeFormatG(value, NullOffset, buffer, out bytesWritten, symbolTable);

default:
ThrowNotImplemented();
bytesWritten = 0;
return false;
}
}

public static bool TryFormat(this TimeSpan value, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
char symbol = format.IsDefault ? 'c' : format.Symbol;

Precondition.Require(symbol == 'G' || symbol == 'g' || symbol == 'c' || symbol == 't' || symbol == 'T');

symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

return TryFormatTimeSpan(value, symbol, buffer, out bytesWritten, symbolTable);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
static bool TryFormatDateTimeFormatG(DateTime value, TimeSpan offset, Span<byte> buffer, out int bytesWritten, SymbolTable symbolTable)
{
// for now it only works for invariant culture
if (symbolTable == SymbolTable.InvariantUtf8)
return InvariantUtf8TimeFormatter.TryFormatG(value, offset, buffer, out bytesWritten);
else if (symbolTable == SymbolTable.InvariantUtf16)
return InvariantUtf16TimeFormatter.TryFormatG(value, offset, buffer, out bytesWritten);

ThrowNotImplemented();
bytesWritten = 0;
return false;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
static bool TryFormatDateTimeFormatO(DateTime value, TimeSpan offset, Span<byte> buffer, out int bytesWritten, SymbolTable symbolTable)
{
// for now it only works for invariant culture
if (symbolTable == SymbolTable.InvariantUtf8)
return InvariantUtf8TimeFormatter.TryFormatO(value, offset, buffer, out bytesWritten);
else if (symbolTable == SymbolTable.InvariantUtf16)
return InvariantUtf16TimeFormatter.TryFormatO(value, offset, buffer, out bytesWritten);

ThrowNotImplemented();
bytesWritten = 0;
return false;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
static bool TryFormatDateTimeRfc1123(DateTime value, Span<byte> buffer, out int bytesWritten, SymbolTable symbolTable)
{
// for now it only works for invariant culture
if (symbolTable == SymbolTable.InvariantUtf8)
return InvariantUtf8TimeFormatter.TryFormatRfc1123(value, buffer, out bytesWritten);
else if (symbolTable == SymbolTable.InvariantUtf16)
return InvariantUtf16TimeFormatter.TryFormatRfc1123(value, buffer, out bytesWritten);

ThrowNotImplemented();
bytesWritten = 0;
return false;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
static bool TryFormatDateTimeRfc1123Lowercase(DateTime value, Span<byte> buffer, out int bytesWritten, SymbolTable symbolTable)
{
// for now it only works for invariant culture
if (symbolTable == SymbolTable.InvariantUtf8)
return InvariantUtf8TimeFormatter.TryFormatRfc1123Lowercase(value, buffer, out bytesWritten);
else if (symbolTable == SymbolTable.InvariantUtf16)
return InvariantUtf16TimeFormatter.TryFormatRfc1123Lowercase(value, buffer, out bytesWritten);

ThrowNotImplemented();
bytesWritten = 0;
return false;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
static bool TryFormatTimeSpan(TimeSpan value, char format, Span<byte> buffer, out int bytesWritten, SymbolTable symbolTable)
{
// for now it only works for invariant culture
if (symbolTable == SymbolTable.InvariantUtf8)
return InvariantUtf8TimeFormatter.TryFormat(value, format, buffer, out bytesWritten);
else if (symbolTable == SymbolTable.InvariantUtf16)
return InvariantUtf16TimeFormatter.TryFormat(value, format, buffer, out bytesWritten);

ThrowNotImplemented();
bytesWritten = 0;
return false;
}

// Methods won't be inlined if they contain a throw, so we factor out the throw to a separate method.
static void ThrowNotImplemented()
{
throw new NotImplementedException();
}
}
}

+ 32
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Formatting/PrimitiveFormatter_float.cs View File

@@ -0,0 +1,32 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Diagnostics;

namespace System.Text
{
public static partial class PrimitiveFormatter
{
public static bool TryFormat(this double value, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
if (format.IsDefault)
{
format = 'G';
}
Precondition.Require(format.Symbol == 'G');
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;
return FloatFormatter.TryFormatNumber(value, false, buffer, out bytesWritten, format, symbolTable);
}

public static bool TryFormat(this float value, Span<byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
if (format.IsDefault)
{
format = 'G';
}
Precondition.Require(format.Symbol == 'G');
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;
return FloatFormatter.TryFormatNumber(value, true, buffer, out bytesWritten, format, symbolTable);
}
}
}

+ 62
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/ParsedFormat.cs View File

@@ -0,0 +1,62 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace System.Text
{
public struct ParsedFormat
{
public const byte NoPrecision = byte.MaxValue;
public const byte MaxPrecision = 99;

private byte _format;
private byte _precision;

public char Symbol => (char)_format;
public byte Precision => _precision;
public bool HasPrecision => _precision != NoPrecision;
public bool IsDefault => _format == 0 && _precision == 0;

public ParsedFormat(char symbol, byte precision = NoPrecision)
{
if (precision != NoPrecision && precision > MaxPrecision)
throw new ArgumentOutOfRangeException("precision");
if (symbol != (byte)symbol)
throw new ArgumentOutOfRangeException("symbol");

_format = (byte)symbol;
_precision = precision;
}

public static implicit operator ParsedFormat(char symbol) => new ParsedFormat(symbol);

public static ParsedFormat Parse(ReadOnlySpan<char> format)
{
if (format.IsEmpty)
return default;

char specifier = format[0];
byte precision = NoPrecision;

if (format.Length > 1)
{
var span = format.Slice(1);

if (!PrimitiveParser.InvariantUtf16.TryParseByte(span, out precision))
throw new FormatException("format");

if (precision > MaxPrecision)
throw new FormatException("precision");
}

return new ParsedFormat(specifier, precision);
}

public static ParsedFormat Parse(string format)
{
if (string.IsNullOrEmpty(format))
return default;

return Parse(format.AsSpan());
}
}
}

+ 268
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/InvariantBool.cs View File

@@ -0,0 +1,268 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.


// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace System.Text
{
public static partial class PrimitiveParser
{
public static partial class InvariantUtf8
{
public unsafe static bool TryParseBoolean(byte* text, int length, out bool value)
{
if (length >= 4)
{
if ((text[0] == 'T' || text[0] == 't') &&
(text[1] == 'R' || text[1] == 'r') &&
(text[2] == 'U' || text[2] == 'u') &&
(text[3] == 'E' || text[3] == 'e'))
{
// No need to set consumed
value = true;
return true;
}
if (length >= 5)
{
if ((text[0] == 'F' || text[0] == 'f') &&
(text[1] == 'A' || text[1] == 'a') &&
(text[2] == 'L' || text[2] == 'l') &&
(text[3] == 'S' || text[3] == 's') &&
(text[4] == 'E' || text[4] == 'e'))
{
// No need to set consumed
value = false;
return true;
}
}
}
// No need to set consumed
value = default;
return false;
}
public unsafe static bool TryParseBoolean(byte* text, int length, out bool value, out int bytesConsumed)
{
if (length >= 4)
{
if ((text[0] == 'T' || text[0] == 't') &&
(text[1] == 'R' || text[1] == 'r') &&
(text[2] == 'U' || text[2] == 'u') &&
(text[3] == 'E' || text[3] == 'e'))
{
bytesConsumed = 4;
value = true;
return true;
}
if (length >= 5)
{
if ((text[0] == 'F' || text[0] == 'f') &&
(text[1] == 'A' || text[1] == 'a') &&
(text[2] == 'L' || text[2] == 'l') &&
(text[3] == 'S' || text[3] == 's') &&
(text[4] == 'E' || text[4] == 'e'))
{
bytesConsumed = 5;
value = false;
return true;
}
}
}
bytesConsumed = 0;
value = default;
return false;
}
public static bool TryParseBoolean(ReadOnlySpan<byte> text, out bool value)
{
if (text.Length >= 4)
{
if ((text[0] == 'T' || text[0] == 't') &&
(text[1] == 'R' || text[1] == 'r') &&
(text[2] == 'U' || text[2] == 'u') &&
(text[3] == 'E' || text[3] == 'e'))
{
// No need to set consumed
value = true;
return true;
}
if (text.Length >= 5)
{
if ((text[0] == 'F' || text[0] == 'f') &&
(text[1] == 'A' || text[1] == 'a') &&
(text[2] == 'L' || text[2] == 'l') &&
(text[3] == 'S' || text[3] == 's') &&
(text[4] == 'E' || text[4] == 'e'))
{
// No need to set consumed
value = false;
return true;
}
}
}
// No need to set consumed
value = default;
return false;
}
public static bool TryParseBoolean(ReadOnlySpan<byte> text, out bool value, out int bytesConsumed)
{
if (text.Length >= 4)
{
if ((text[0] == 'T' || text[0] == 't') &&
(text[1] == 'R' || text[1] == 'r') &&
(text[2] == 'U' || text[2] == 'u') &&
(text[3] == 'E' || text[3] == 'e'))
{
bytesConsumed = 4;
value = true;
return true;
}
if (text.Length >= 5)
{
if ((text[0] == 'F' || text[0] == 'f') &&
(text[1] == 'A' || text[1] == 'a') &&
(text[2] == 'L' || text[2] == 'l') &&
(text[3] == 'S' || text[3] == 's') &&
(text[4] == 'E' || text[4] == 'e'))
{
bytesConsumed = 5;
value = false;
return true;
}
}
}
bytesConsumed = 0;
value = default;
return false;
}
}
public static partial class InvariantUtf16
{
public unsafe static bool TryParseBoolean(char* text, int length, out bool value)
{
if (length >= 4)
{
if ((text[0] == 'T' || text[0] == 't') &&
(text[1] == 'R' || text[1] == 'r') &&
(text[2] == 'U' || text[2] == 'u') &&
(text[3] == 'E' || text[3] == 'e'))
{
// No need to set consumed
value = true;
return true;
}
if (length >= 5)
{
if ((text[0] == 'F' || text[0] == 'f') &&
(text[1] == 'A' || text[1] == 'a') &&
(text[2] == 'L' || text[2] == 'l') &&
(text[3] == 'S' || text[3] == 's') &&
(text[4] == 'E' || text[4] == 'e'))
{
// No need to set consumed
value = false;
return true;
}
}
}
// No need to set consumed
value = default;
return false;
}
public unsafe static bool TryParseBoolean(char* text, int length, out bool value, out int charsConsumed)
{
if (length >= 4)
{
if ((text[0] == 'T' || text[0] == 't') &&
(text[1] == 'R' || text[1] == 'r') &&
(text[2] == 'U' || text[2] == 'u') &&
(text[3] == 'E' || text[3] == 'e'))
{
charsConsumed = 4;
value = true;
return true;
}
if (length >= 5)
{
if ((text[0] == 'F' || text[0] == 'f') &&
(text[1] == 'A' || text[1] == 'a') &&
(text[2] == 'L' || text[2] == 'l') &&
(text[3] == 'S' || text[3] == 's') &&
(text[4] == 'E' || text[4] == 'e'))
{
charsConsumed = 5;
value = false;
return true;
}
}
}
charsConsumed = 0;
value = default;
return false;
}
public static bool TryParseBoolean(ReadOnlySpan<char> text, out bool value)
{
if (text.Length >= 4)
{
if ((text[0] == 'T' || text[0] == 't') &&
(text[1] == 'R' || text[1] == 'r') &&
(text[2] == 'U' || text[2] == 'u') &&
(text[3] == 'E' || text[3] == 'e'))
{
// No need to set consumed
value = true;
return true;
}
if (text.Length >= 5)
{
if ((text[0] == 'F' || text[0] == 'f') &&
(text[1] == 'A' || text[1] == 'a') &&
(text[2] == 'L' || text[2] == 'l') &&
(text[3] == 'S' || text[3] == 's') &&
(text[4] == 'E' || text[4] == 'e'))
{
// No need to set consumed
value = false;
return true;
}
}
}
// No need to set consumed
value = default;
return false;
}
public static bool TryParseBoolean(ReadOnlySpan<char> text, out bool value, out int charsConsumed)
{
if (text.Length >= 4)
{
if ((text[0] == 'T' || text[0] == 't') &&
(text[1] == 'R' || text[1] == 'r') &&
(text[2] == 'U' || text[2] == 'u') &&
(text[3] == 'E' || text[3] == 'e'))
{
charsConsumed = 4;
value = true;
return true;
}
if (text.Length >= 5)
{
if ((text[0] == 'F' || text[0] == 'f') &&
(text[1] == 'A' || text[1] == 'a') &&
(text[2] == 'L' || text[2] == 'l') &&
(text[3] == 'S' || text[3] == 's') &&
(text[4] == 'E' || text[4] == 'e'))
{
charsConsumed = 5;
value = false;
return true;
}
}
}
charsConsumed = 0;
value = default;
return false;
}
}
}
}

+ 2395
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/InvariantSigned.cs
File diff suppressed because it is too large
View File


+ 2594
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/InvariantSignedHex.cs
File diff suppressed because it is too large
View File


+ 2332
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/InvariantUnsigned.cs
File diff suppressed because it is too large
View File


+ 2594
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/InvariantUnsignedHex.cs
File diff suppressed because it is too large
View File


+ 111
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/InvariantUtf16_decimal.cs View File

@@ -0,0 +1,111 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.


namespace System.Text
{
public static partial class PrimitiveParser
{
public static partial class InvariantUtf16
{
public unsafe static bool TryParseDecimal(char* text, int length, out decimal value)
{
int consumed;
var span = new ReadOnlySpan<char>(text, length);
return TryParseDecimal(span, out value, out consumed);
}
public unsafe static bool TryParseDecimal(char* text, int length, out decimal value, out int charactersConsumed)
{
var span = new ReadOnlySpan<char>(text, length);
return TryParseDecimal(span, out value, out charactersConsumed);
}
public static bool TryParseDecimal(ReadOnlySpan<char> text, out decimal value)
{
int consumed;
return TryParseDecimal(text, out value, out consumed);
}
public static bool TryParseDecimal(ReadOnlySpan<char> text, out decimal value, out int charactersConsumed)
{
// Precondition replacement
if (text.Length < 1)
{
value = 0;
charactersConsumed = 0;
return false;
}

value = 0.0M;
charactersConsumed = 0;
string decimalString = "";
bool decimalPlace = false, signed = false;

int indexOfFirstDigit = 0;
if (text[0] == '-' || text[0] == '+')
{
signed = true;
decimalString += text[0];
indexOfFirstDigit = 1;
charactersConsumed++;
}

for (int charIndex = indexOfFirstDigit; charIndex < text.Length; charIndex++)
{
char nextChar = text[charIndex];
char nextCharVal = (char)(nextChar - '0');

if (nextCharVal > 9)
{
if (!decimalPlace && nextChar == '.')
{
charactersConsumed++;
decimalPlace = true;
decimalString += nextChar;
}
else if ((decimalPlace && signed && charactersConsumed == 2) || ((signed || decimalPlace) && charactersConsumed == 1))
{
value = 0;
charactersConsumed = 0;
return false;
}
else
{
if (decimal.TryParse(decimalString, out value))
{
return true;
}
else
{
charactersConsumed = 0;
return false;
}
}
}
else
{
charactersConsumed++;
decimalString += nextChar;
}
}

if ((decimalPlace && signed && charactersConsumed == 2) || ((signed || decimalPlace) && charactersConsumed == 1))
{
value = 0;
charactersConsumed = 0;
return false;
}
else
{
if (decimal.TryParse(decimalString, out value))
{
return true;
}
else
{
charactersConsumed = 0;
return false;
}
}
}
}
}
}

+ 111
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/InvariantUtf8_decimal.cs View File

@@ -0,0 +1,111 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.


namespace System.Text
{
public static partial class PrimitiveParser
{
public static partial class InvariantUtf8
{
public unsafe static bool TryParseDecimal(byte* text, int length, out decimal value)
{
int consumed;
var span = new ReadOnlySpan<byte>(text, length);
return TryParseDecimal(span, out value, out consumed);
}
public unsafe static bool TryParseDecimal(byte* text, int length, out decimal value, out int bytesConsumed)
{
var span = new ReadOnlySpan<byte>(text, length);
return TryParseDecimal(span, out value, out bytesConsumed);
}
public static bool TryParseDecimal(ReadOnlySpan<byte> text, out decimal value)
{
int consumed;
return TryParseDecimal(text, out value, out consumed);
}
public static bool TryParseDecimal(ReadOnlySpan<byte> text, out decimal value, out int bytesConsumed)
{
// Precondition replacement
if (text.Length < 1)
{
value = 0;
bytesConsumed = 0;
return false;
}

value = 0.0M;
bytesConsumed = 0;
string decimalString = "";
bool decimalPlace = false, signed = false;

int indexOfFirstDigit = 0;
if (text[0] == '-' || text[0] == '+')
{
signed = true;
decimalString += (char)text[0];
indexOfFirstDigit = 1;
bytesConsumed++;
}

for (int byteIndex = indexOfFirstDigit; byteIndex < text.Length; byteIndex++)
{
byte nextByte = text[byteIndex];
byte nextByteVal = (byte)(nextByte - '0');

if (nextByteVal > 9)
{
if (!decimalPlace && nextByte == '.')
{
bytesConsumed++;
decimalPlace = true;
decimalString += (char)nextByte;
}
else if ((decimalPlace && signed && bytesConsumed == 2) || ((signed || decimalPlace) && bytesConsumed == 1))
{
value = 0;
bytesConsumed = 0;
return false;
}
else
{
if (decimal.TryParse(decimalString, out value))
{
return true;
}
else
{
bytesConsumed = 0;
return false;
}
}
}
else
{
bytesConsumed++;
decimalString += (char)nextByte;
}
}

if ((decimalPlace && signed && bytesConsumed == 2) || ((signed || decimalPlace) && bytesConsumed == 1))
{
value = 0;
bytesConsumed = 0;
return false;
}
else
{
if (decimal.TryParse(decimalString, out value))
{
return true;
}
else
{
bytesConsumed = 0;
return false;
}
}
}
}
}
}

+ 54
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/PrimitiveParser.cs View File

@@ -0,0 +1,54 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Text
{
public static partial class PrimitiveParser
{
const int ByteOverflowLength = 3;
const int ByteOverflowLengthHex = 2;
const int UInt16OverflowLength = 5;
const int UInt16OverflowLengthHex = 4;
const int UInt32OverflowLength = 10;
const int UInt32OverflowLengthHex = 8;
const int UInt64OverflowLength = 20;
const int UInt64OverflowLengthHex = 16;
const int SByteOverflowLength = 3;
const int SByteOverflowLengthHex = 2;
const int Int16OverflowLength = 5;
const int Int16OverflowLengthHex = 4;
const int Int32OverflowLength = 10;
const int Int32OverflowLengthHex = 8;
const int Int64OverflowLength = 19;
const int Int64OverflowLengthHex = 16;

static readonly byte[] s_HexLookup =
{
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 15
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 31
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 47
0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 63
0xFF, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 79
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 95
0xFF, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 111
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 127
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 143
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 159
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 175
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 191
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 207
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 223
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 239
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // 255
};

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsHexFormat(ParsedFormat format)
{
return format.Symbol == 'X' || format.Symbol == 'x';
}
}
}

+ 32
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/PrimitiveParser_Boolean.cs View File

@@ -0,0 +1,32 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.


namespace System.Text
{
public static partial class PrimitiveParser
{
public static bool TryParseBoolean(ReadOnlySpan<byte> text, out bool value, out int bytesConsumed, SymbolTable symbolTable = null)
{
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

bytesConsumed = 0;
value = default;

if (symbolTable == SymbolTable.InvariantUtf8)
{
return InvariantUtf8.TryParseBoolean(text, out value, out bytesConsumed);
}
if (symbolTable == SymbolTable.InvariantUtf16)
{
ReadOnlySpan<char> textChars = text.NonPortableCast<byte, char>();
int charactersConsumed;
bool result = InvariantUtf16.TryParseBoolean(textChars, out value, out charactersConsumed);
bytesConsumed = charactersConsumed * sizeof(char);
return result;
}

return false;
}
}
}

+ 32
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/PrimitiveParser_Decimal.cs View File

@@ -0,0 +1,32 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.


namespace System.Text
{
public static partial class PrimitiveParser
{
public static bool TryParseDecimal(ReadOnlySpan<byte> text, out decimal value, out int bytesConsumed, SymbolTable symbolTable = null)
{
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

bytesConsumed = 0;
value = default;

if (symbolTable == SymbolTable.InvariantUtf8)
{
return InvariantUtf8.TryParseDecimal(text, out value, out bytesConsumed);
}
else if (symbolTable == SymbolTable.InvariantUtf16)
{
ReadOnlySpan<char> textChars = text.NonPortableCast<byte, char>();
int charactersConsumed;
bool result = InvariantUtf16.TryParseDecimal(textChars, out value, out charactersConsumed);
bytesConsumed = charactersConsumed * sizeof(char);
return result;
}

return false;
}
}
}

+ 528
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/Signed.cs View File

@@ -0,0 +1,528 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Text
{
public static partial class PrimitiveParser
{
#region Helpers

private const sbyte maxValueSbyteDiv10 = sbyte.MaxValue / 10;
private const short maxValueShortDiv10 = short.MaxValue / 10;
private const int maxValueIntDiv10 = int.MaxValue / 10;
private const long maxValueLongDiv10 = long.MaxValue / 10;

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsDigit(int i)
{
return (uint)(i - '0') <= ('9' - '0');
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsValid(SymbolTable.Symbol symbol)
{
return symbol <= SymbolTable.Symbol.D9;
}

// If parsedValue > (sbyte.MaxValue / 10), any more appended digits will cause overflow.
// if parsedValue == (sbyte.MaxValue / 10), any nextDigit greater than 7 or 8 (depending on sign) implies overflow.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool WillOverFlow(sbyte value, int nextDigit, int sign)
{
bool nextDigitTooLarge = nextDigit > 8 || (sign > 0 && nextDigit > 7);
return (value > maxValueSbyteDiv10 || (value == maxValueSbyteDiv10 && nextDigitTooLarge));
}

// If parsedValue > (short.MaxValue / 10), any more appended digits will cause overflow.
// if parsedValue == (short.MaxValue / 10), any nextDigit greater than 7 or 8 (depending on sign) implies overflow.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool WillOverFlow(short value, int nextDigit, int sign)
{
bool nextDigitTooLarge = nextDigit > 8 || (sign > 0 && nextDigit > 7);
return (value > maxValueShortDiv10 || (value == maxValueShortDiv10 && nextDigitTooLarge));
}

// If parsedValue > (int.MaxValue / 10), any more appended digits will cause overflow.
// if parsedValue == (int.MaxValue / 10), any nextDigit greater than 7 or 8 (depending on sign) implies overflow.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool WillOverFlow(int value, int nextDigit, int sign)
{
bool nextDigitTooLarge = nextDigit > 8 || (sign > 0 && nextDigit > 7);
return (value > maxValueIntDiv10 || (value == maxValueIntDiv10 && nextDigitTooLarge));
}

// If parsedValue > (long.MaxValue / 10), any more appended digits will cause overflow.
// if parsedValue == (long.MaxValue / 10), any nextDigit greater than 7 or 8 (depending on sign) implies overflow.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool WillOverFlow(long value, int nextDigit, int sign)
{
bool nextDigitTooLarge = nextDigit > 8 || (sign > 0 && nextDigit > 7);
return (value > maxValueLongDiv10 || (value == maxValueLongDiv10 && nextDigitTooLarge));
}

#endregion

public static bool TryParseSByte(ReadOnlySpan<byte> text, out sbyte value, out int bytesConsumed, ParsedFormat format = default, SymbolTable symbolTable = null)
{
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

if (!format.IsDefault && format.HasPrecision)
{
throw new NotImplementedException("Format with precision not supported.");
}

if (symbolTable == SymbolTable.InvariantUtf8)
{
if (IsHexFormat(format))
{
return InvariantUtf8.Hex.TryParseSByte(text, out value, out bytesConsumed);
}
else
{
return InvariantUtf8.TryParseSByte(text, out value, out bytesConsumed);
}
}
else if (symbolTable == SymbolTable.InvariantUtf16)
{
ReadOnlySpan<char> utf16Text = text.NonPortableCast<byte, char>();
int charsConsumed;
bool result;
if (IsHexFormat(format))
{
result = InvariantUtf16.Hex.TryParseSByte(utf16Text, out value, out charsConsumed);
}
else
{
result = InvariantUtf16.TryParseSByte(utf16Text, out value, out charsConsumed);
}
bytesConsumed = charsConsumed * sizeof(char);
return result;
}

if (IsHexFormat(format))
{
throw new NotImplementedException("The only supported encodings for hexadecimal parsing are InvariantUtf8 and InvariantUtf16.");
}

if (!(format.IsDefault || format.Symbol == 'G' || format.Symbol == 'g'))
{
throw new NotImplementedException(String.Format("Format '{0}' not supported.", format.Symbol));
}

SymbolTable.Symbol nextSymbol;
int thisSymbolConsumed;
if (!symbolTable.TryParse(text, out nextSymbol, out thisSymbolConsumed))
{
value = default;
bytesConsumed = 0;
return false;
}

int sign = 1;
if (nextSymbol == SymbolTable.Symbol.MinusSign)
{
sign = -1;
}

int signConsumed = 0;
if (nextSymbol == SymbolTable.Symbol.PlusSign || nextSymbol == SymbolTable.Symbol.MinusSign)
{
signConsumed = thisSymbolConsumed;
if (!symbolTable.TryParse(text.Slice(signConsumed), out nextSymbol, out thisSymbolConsumed))
{
value = default;
bytesConsumed = 0;
return false;
}
}

if (nextSymbol > SymbolTable.Symbol.D9)
{
value = default;
bytesConsumed = 0;
return false;
}

int parsedValue = (int)nextSymbol;
int index = signConsumed + thisSymbolConsumed;

while (index < text.Length)
{
bool success = symbolTable.TryParse(text.Slice(index), out nextSymbol, out thisSymbolConsumed);
if (!success || nextSymbol > SymbolTable.Symbol.D9)
{
bytesConsumed = index;
value = (sbyte)(parsedValue * sign);
return true;
}

// If parsedValue > (sbyte.MaxValue / 10), any more appended digits will cause overflow.
// if parsedValue == (sbyte.MaxValue / 10), any nextDigit greater than 7 or 8 (depending on sign) implies overflow.
bool positive = sign > 0;
bool nextDigitTooLarge = nextSymbol > SymbolTable.Symbol.D8 || (positive && nextSymbol > SymbolTable.Symbol.D7);
if (parsedValue > sbyte.MaxValue / 10 || (parsedValue == sbyte.MaxValue / 10 && nextDigitTooLarge))
{
bytesConsumed = 0;
value = default;
return false;
}

index += thisSymbolConsumed;
parsedValue = parsedValue * 10 + (int)nextSymbol;
}

bytesConsumed = text.Length;
value = (sbyte)(parsedValue * sign);
return true;
}

public static bool TryParseInt16(ReadOnlySpan<byte> text, out short value, out int bytesConsumed, ParsedFormat format = default, SymbolTable symbolTable = null)
{
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

if (!format.IsDefault && format.HasPrecision)
{
throw new NotImplementedException("Format with precision not supported.");
}

if (symbolTable == SymbolTable.InvariantUtf8)
{
if (IsHexFormat(format))
{
return InvariantUtf8.Hex.TryParseInt16(text, out value, out bytesConsumed);
}
else
{
return InvariantUtf8.TryParseInt16(text, out value, out bytesConsumed);
}
}
else if (symbolTable == SymbolTable.InvariantUtf16)
{
ReadOnlySpan<char> utf16Text = text.NonPortableCast<byte, char>();
int charsConsumed;
bool result;
if (IsHexFormat(format))
{
result = InvariantUtf16.Hex.TryParseInt16(utf16Text, out value, out charsConsumed);
}
else
{
result = InvariantUtf16.TryParseInt16(utf16Text, out value, out charsConsumed);
}
bytesConsumed = charsConsumed * sizeof(char);
return result;
}

if (IsHexFormat(format))
{
throw new NotImplementedException("The only supported encodings for hexadecimal parsing are InvariantUtf8 and InvariantUtf16.");
}

if (!(format.IsDefault || format.Symbol == 'G' || format.Symbol == 'g'))
{
throw new NotImplementedException(String.Format("Format '{0}' not supported.", format.Symbol));
}

SymbolTable.Symbol nextSymbol;
int thisSymbolConsumed;
if (!symbolTable.TryParse(text, out nextSymbol, out thisSymbolConsumed))
{
value = default;
bytesConsumed = 0;
return false;
}

int sign = 1;
if ((SymbolTable.Symbol)nextSymbol == SymbolTable.Symbol.MinusSign)
{
sign = -1;
}

int signConsumed = 0;
if (nextSymbol == SymbolTable.Symbol.PlusSign || nextSymbol == SymbolTable.Symbol.MinusSign)
{
signConsumed = thisSymbolConsumed;
if (!symbolTable.TryParse(text.Slice(signConsumed), out nextSymbol, out thisSymbolConsumed))
{
value = default;
bytesConsumed = 0;
return false;
}
}

if (nextSymbol > SymbolTable.Symbol.D9)
{
value = default;
bytesConsumed = 0;
return false;
}

int parsedValue = (int)nextSymbol;
int index = signConsumed + thisSymbolConsumed;

while (index < text.Length)
{
bool success = symbolTable.TryParse(text.Slice(index), out nextSymbol, out thisSymbolConsumed);
if (!success || nextSymbol > SymbolTable.Symbol.D9)
{
bytesConsumed = index;
value = (short)(parsedValue * sign);
return true;
}

// If parsedValue > (short.MaxValue / 10), any more appended digits will cause overflow.
// if parsedValue == (short.MaxValue / 10), any nextDigit greater than 7 or 8 (depending on sign) implies overflow.
bool positive = sign > 0;
bool nextDigitTooLarge = nextSymbol > SymbolTable.Symbol.D8 || (positive && nextSymbol > SymbolTable.Symbol.D7);
if (parsedValue > short.MaxValue / 10 || (parsedValue == short.MaxValue / 10 && nextDigitTooLarge))
{
bytesConsumed = 0;
value = default;
return false;
}

index += thisSymbolConsumed;
parsedValue = parsedValue * 10 + (int)nextSymbol;
}

bytesConsumed = text.Length;
value = (short)(parsedValue * sign);
return true;
}

public static bool TryParseInt32(ReadOnlySpan<byte> text, out int value, out int bytesConsumed, ParsedFormat format = default, SymbolTable symbolTable = null)
{
bool isDefault = format.IsDefault;
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

if (!isDefault && format.HasPrecision)
{
throw new NotImplementedException("Format with precision not supported.");
}

bool isHex = IsHexFormat(format);

if (symbolTable == SymbolTable.InvariantUtf8)
{
return isHex ? InvariantUtf8.Hex.TryParseInt32(text, out value, out bytesConsumed) :
InvariantUtf8.TryParseInt32(text, out value, out bytesConsumed);
}
else if (symbolTable == SymbolTable.InvariantUtf16)
{
/*return isHex ? InvariantUtf16.Hex.TryParseInt32(text, out value, out bytesConsumed) :
InvariantUtf16.TryParseInt32(text, out value, out bytesConsumed);*/
ReadOnlySpan<char> utf16Text = text.NonPortableCast<byte, char>();
bool result = isHex ? InvariantUtf16.Hex.TryParseInt32(utf16Text, out value, out int charsConsumed) :
InvariantUtf16.TryParseInt32(utf16Text, out value, out charsConsumed);
bytesConsumed = charsConsumed * sizeof(char);
return result;
}

if (isHex)
{
throw new NotImplementedException("The only supported encodings for hexadecimal parsing are InvariantUtf8 and InvariantUtf16.");
}

if (!(isDefault || format.Symbol == 'G' || format.Symbol == 'g'))
{
throw new NotImplementedException(String.Format("Format '{0}' not supported.", format.Symbol));
}

int textLength = text.Length;
if (textLength < 1) goto FalseExit;

if (!symbolTable.TryParse(text, out SymbolTable.Symbol symbol, out int consumed)) goto FalseExit;

sbyte sign = 1;
int index = 0;
if (symbol == SymbolTable.Symbol.MinusSign)
{
sign = -1;
index += consumed;
if (index >= textLength) goto FalseExit;
if (!symbolTable.TryParse(text.Slice(index), out symbol, out consumed)) goto FalseExit;
}
else if (symbol == SymbolTable.Symbol.PlusSign)
{
index += consumed;
if (index >= textLength) goto FalseExit;
if (!symbolTable.TryParse(text.Slice(index), out symbol, out consumed)) goto FalseExit;
}

int answer = 0;
if (IsValid(symbol))
{
int numBytes = consumed;
if (symbol == SymbolTable.Symbol.D0)
{
do
{
index += consumed;
if (index >= textLength) goto Done;
if (!symbolTable.TryParse(text.Slice(index), out symbol, out consumed)) goto Done;
} while (symbol == SymbolTable.Symbol.D0);
if (!IsValid(symbol)) goto Done;
}

int firstNonZeroDigitIndex = index;
if (textLength - firstNonZeroDigitIndex < Int32OverflowLength * numBytes)
{
do
{
answer = answer * 10 + (int)symbol;
index += consumed;
if (index >= textLength) goto Done;
if (!symbolTable.TryParse(text.Slice(index), out symbol, out consumed)) goto Done;
} while (IsValid(symbol));
}
else
{
do
{
answer = answer * 10 + (int)symbol;
index += consumed;
if (index - firstNonZeroDigitIndex == (Int32OverflowLength - 1) * numBytes)
{
if (!symbolTable.TryParse(text.Slice(index), out symbol, out consumed)) goto Done;
if (IsValid(symbol))
{
if (WillOverFlow(answer, (int)symbol, sign)) goto FalseExit;
answer = answer * 10 + (int)symbol;
index += consumed;
}
goto Done;
}
if (!symbolTable.TryParse(text.Slice(index), out symbol, out consumed)) goto Done;
} while (IsValid(symbol));
}
goto Done;
}

FalseExit:
bytesConsumed = 0;
value = 0;
return false;

Done:
bytesConsumed = index;
value = answer * sign;
return true;
}

public static bool TryParseInt64(ReadOnlySpan<byte> text, out long value, out int bytesConsumed, ParsedFormat format = default, SymbolTable symbolTable = null)
{
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

if (!format.IsDefault && format.HasPrecision)
{
throw new NotImplementedException("Format with precision not supported.");
}

if (symbolTable == SymbolTable.InvariantUtf8)
{
if (IsHexFormat(format))
{
return InvariantUtf8.Hex.TryParseInt64(text, out value, out bytesConsumed);
}
else
{
return InvariantUtf8.TryParseInt64(text, out value, out bytesConsumed);
}
}
else if (symbolTable == SymbolTable.InvariantUtf16)
{
ReadOnlySpan<char> utf16Text = text.NonPortableCast<byte, char>();
int charsConsumed;
bool result;
if (IsHexFormat(format))
{
result = InvariantUtf16.Hex.TryParseInt64(utf16Text, out value, out charsConsumed);
}
else
{
result = InvariantUtf16.TryParseInt64(utf16Text, out value, out charsConsumed);
}
bytesConsumed = charsConsumed * sizeof(char);
return result;
}

if (IsHexFormat(format))
{
throw new NotImplementedException("The only supported encodings for hexadecimal parsing are InvariantUtf8 and InvariantUtf16.");
}

if (!(format.IsDefault || format.Symbol == 'G' || format.Symbol == 'g'))
{
throw new NotImplementedException(String.Format("Format '{0}' not supported.", format.Symbol));
}

SymbolTable.Symbol nextSymbol;
int thisSymbolConsumed;
if (!symbolTable.TryParse(text, out nextSymbol, out thisSymbolConsumed))
{
value = default;
bytesConsumed = 0;
return false;
}

int sign = 1;
if (nextSymbol == SymbolTable.Symbol.MinusSign)
{
sign = -1;
}

int signConsumed = 0;
if (nextSymbol == SymbolTable.Symbol.PlusSign || nextSymbol == SymbolTable.Symbol.MinusSign)
{
signConsumed = thisSymbolConsumed;
if (!symbolTable.TryParse(text.Slice(signConsumed), out nextSymbol, out thisSymbolConsumed))
{
value = default;
bytesConsumed = 0;
return false;
}
}

if (nextSymbol > SymbolTable.Symbol.D9)
{
value = default;
bytesConsumed = 0;
return false;
}

long parsedValue = (long)nextSymbol;
int index = signConsumed + thisSymbolConsumed;

while (index < text.Length)
{
bool success = symbolTable.TryParse(text.Slice(index), out nextSymbol, out thisSymbolConsumed);
if (!success || nextSymbol > SymbolTable.Symbol.D9)
{
bytesConsumed = index;
value = (long)(parsedValue * sign);
return true;
}

// If parsedValue > (long.MaxValue / 10), any more appended digits will cause overflow.
// if parsedValue == (long.MaxValue / 10), any nextDigit greater than 7 or 8 (depending on sign) implies overflow.
bool positive = sign > 0;
bool nextDigitTooLarge = nextSymbol > SymbolTable.Symbol.D8 || (positive && nextSymbol > SymbolTable.Symbol.D7);
if (parsedValue > long.MaxValue / 10 || (parsedValue == long.MaxValue / 10 && nextDigitTooLarge))
{
bytesConsumed = 0;
value = default;
return false;
}

index += thisSymbolConsumed;
parsedValue = parsedValue * 10 + (long)nextSymbol;
}

bytesConsumed = text.Length;
value = (long)(parsedValue * sign);
return true;
}
}
}

+ 385
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Primitives/System/Text/Parsing/Unsigned.cs View File

@@ -0,0 +1,385 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace System.Text
{
public static partial class PrimitiveParser
{

public static bool TryParseByte(ReadOnlySpan<byte> text, out byte value, out int bytesConsumed, ParsedFormat format = default, SymbolTable symbolTable = null)
{
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

if (!format.IsDefault && format.HasPrecision)
{
throw new NotImplementedException("Format with precision not supported.");
}

if (symbolTable == SymbolTable.InvariantUtf8)
{
if (IsHexFormat(format))
{
return InvariantUtf8.Hex.TryParseByte(text, out value, out bytesConsumed);
}
else
{
return InvariantUtf8.TryParseByte(text, out value, out bytesConsumed);
}
}
else if (symbolTable == SymbolTable.InvariantUtf16)
{
ReadOnlySpan<char> utf16Text = text.NonPortableCast<byte, char>();
int charsConsumed;
bool result;
if (IsHexFormat(format))
{
result = InvariantUtf16.Hex.TryParseByte(utf16Text, out value, out charsConsumed);
}
else
{
result = InvariantUtf16.TryParseByte(utf16Text, out value, out charsConsumed);
}
bytesConsumed = charsConsumed * sizeof(char);
return result;
}

if (IsHexFormat(format))
{
throw new NotImplementedException("The only supported encodings for hexadecimal parsing are InvariantUtf8 and InvariantUtf16.");
}

if (!(format.IsDefault || format.Symbol == 'G' || format.Symbol == 'g'))
{
throw new NotImplementedException(String.Format("Format '{0}' not supported.", format.Symbol));
}

SymbolTable.Symbol nextSymbol;
int thisSymbolConsumed;
if (!symbolTable.TryParse(text, out nextSymbol, out thisSymbolConsumed))
{
value = default;
bytesConsumed = 0;
return false;
}

if (nextSymbol > SymbolTable.Symbol.D9)
{
value = default;
bytesConsumed = 0;
return false;
}

uint parsedValue = (uint)nextSymbol;
int index = thisSymbolConsumed;

while (index < text.Length)
{
bool success = symbolTable.TryParse(text.Slice(index), out nextSymbol, out thisSymbolConsumed);
if (!success || nextSymbol > SymbolTable.Symbol.D9)
{
bytesConsumed = index;
value = (byte) parsedValue;
return true;
}

// If parsedValue > (byte.MaxValue / 10), any more appended digits will cause overflow.
// if parsedValue == (byte.MaxValue / 10), any nextDigit greater than 5 implies overflow.
if (parsedValue > byte.MaxValue / 10 || (parsedValue == byte.MaxValue / 10 && nextSymbol > SymbolTable.Symbol.D5))
{
bytesConsumed = 0;
value = default;
return false;
}

index += thisSymbolConsumed;
parsedValue = parsedValue * 10 + (uint)nextSymbol;
}

bytesConsumed = text.Length;
value = (byte) parsedValue;
return true;
}

public static bool TryParseUInt16(ReadOnlySpan<byte> text, out ushort value, out int bytesConsumed, ParsedFormat format = default, SymbolTable symbolTable = null)
{
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

if (!format.IsDefault && format.HasPrecision)
{
throw new NotImplementedException("Format with precision not supported.");
}

if (symbolTable == SymbolTable.InvariantUtf8)
{
if (IsHexFormat(format))
{
return InvariantUtf8.Hex.TryParseUInt16(text, out value, out bytesConsumed);
}
else
{
return InvariantUtf8.TryParseUInt16(text, out value, out bytesConsumed);
}
}
else if (symbolTable == SymbolTable.InvariantUtf16)
{
ReadOnlySpan<char> utf16Text = text.NonPortableCast<byte, char>();
int charsConsumed;
bool result;
if (IsHexFormat(format))
{
result = InvariantUtf16.Hex.TryParseUInt16(utf16Text, out value, out charsConsumed);
}
else
{
result = InvariantUtf16.TryParseUInt16(utf16Text, out value, out charsConsumed);
}
bytesConsumed = charsConsumed * sizeof(char);
return result;
}

if (IsHexFormat(format))
{
throw new NotImplementedException("The only supported encodings for hexadecimal parsing are InvariantUtf8 and InvariantUtf16.");
}

if (!(format.IsDefault || format.Symbol == 'G' || format.Symbol == 'g'))
{
throw new NotImplementedException(String.Format("Format '{0}' not supported.", format.Symbol));
}

SymbolTable.Symbol nextSymbol;
int thisSymbolConsumed;
if (!symbolTable.TryParse(text, out nextSymbol, out thisSymbolConsumed))
{
value = default;
bytesConsumed = 0;
return false;
}

if (nextSymbol > SymbolTable.Symbol.D9)
{
value = default;
bytesConsumed = 0;
return false;
}

uint parsedValue = (uint)nextSymbol;
int index = thisSymbolConsumed;

while (index < text.Length)
{
bool success = symbolTable.TryParse(text.Slice(index), out nextSymbol, out thisSymbolConsumed);
if (!success || nextSymbol > SymbolTable.Symbol.D9)
{
bytesConsumed = index;
value = (ushort) parsedValue;
return true;
}

// If parsedValue > (ushort.MaxValue / 10), any more appended digits will cause overflow.
// if parsedValue == (ushort.MaxValue / 10), any nextDigit greater than 5 implies overflow.
if (parsedValue > ushort.MaxValue / 10 || (parsedValue == ushort.MaxValue / 10 && nextSymbol > SymbolTable.Symbol.D5))
{
bytesConsumed = 0;
value = default;
return false;
}

index += thisSymbolConsumed;
parsedValue = parsedValue * 10 + (uint)nextSymbol;
}

bytesConsumed = text.Length;
value = (ushort) parsedValue;
return true;
}

public static bool TryParseUInt32(ReadOnlySpan<byte> text, out uint value, out int bytesConsumed, ParsedFormat format = default, SymbolTable symbolTable = null)
{
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

if (!format.IsDefault && format.HasPrecision)
{
throw new NotImplementedException("Format with precision not supported.");
}

if (symbolTable == SymbolTable.InvariantUtf8)
{
if (IsHexFormat(format))
{
return InvariantUtf8.Hex.TryParseUInt32(text, out value, out bytesConsumed);
}
else
{
return InvariantUtf8.TryParseUInt32(text, out value, out bytesConsumed);
}
}
else if (symbolTable == SymbolTable.InvariantUtf16)
{
ReadOnlySpan<char> utf16Text = text.NonPortableCast<byte, char>();
int charsConsumed;
bool result;
if (IsHexFormat(format))
{
result = InvariantUtf16.Hex.TryParseUInt32(utf16Text, out value, out charsConsumed);
}
else
{
result = InvariantUtf16.TryParseUInt32(utf16Text, out value, out charsConsumed);
}
bytesConsumed = charsConsumed * sizeof(char);
return result;
}

if (IsHexFormat(format))
{
throw new NotImplementedException("The only supported encodings for hexadecimal parsing are InvariantUtf8 and InvariantUtf16.");
}

if (!(format.IsDefault || format.Symbol == 'G' || format.Symbol == 'g'))
{
throw new NotImplementedException(String.Format("Format '{0}' not supported.", format.Symbol));
}

SymbolTable.Symbol nextSymbol;
int thisSymbolConsumed;
if (!symbolTable.TryParse(text, out nextSymbol, out thisSymbolConsumed))
{
value = default;
bytesConsumed = 0;
return false;
}

if (nextSymbol > SymbolTable.Symbol.D9)
{
value = default;
bytesConsumed = 0;
return false;
}

uint parsedValue = (uint)nextSymbol;
int index = thisSymbolConsumed;

while (index < text.Length)
{
bool success = symbolTable.TryParse(text.Slice(index), out nextSymbol, out thisSymbolConsumed);
if (!success || nextSymbol > SymbolTable.Symbol.D9)
{
bytesConsumed = index;
value = (uint) parsedValue;
return true;
}

// If parsedValue > (uint.MaxValue / 10), any more appended digits will cause overflow.
// if parsedValue == (uint.MaxValue / 10), any nextDigit greater than 5 implies overflow.
if (parsedValue > uint.MaxValue / 10 || (parsedValue == uint.MaxValue / 10 && nextSymbol > SymbolTable.Symbol.D5))
{
bytesConsumed = 0;
value = default;
return false;
}

index += thisSymbolConsumed;
parsedValue = parsedValue * 10 + (uint)nextSymbol;
}

bytesConsumed = text.Length;
value = (uint) parsedValue;
return true;
}

public static bool TryParseUInt64(ReadOnlySpan<byte> text, out ulong value, out int bytesConsumed, ParsedFormat format = default, SymbolTable symbolTable = null)
{
symbolTable = symbolTable ?? SymbolTable.InvariantUtf8;

if (!format.IsDefault && format.HasPrecision)
{
throw new NotImplementedException("Format with precision not supported.");
}

if (symbolTable == SymbolTable.InvariantUtf8)
{
if (IsHexFormat(format))
{
return InvariantUtf8.Hex.TryParseUInt64(text, out value, out bytesConsumed);
}
else
{
return InvariantUtf8.TryParseUInt64(text, out value, out bytesConsumed);
}
}
else if (symbolTable == SymbolTable.InvariantUtf16)
{
ReadOnlySpan<char> utf16Text = text.NonPortableCast<byte, char>();
int charsConsumed;
bool result;
if (IsHexFormat(format))
{
result = InvariantUtf16.Hex.TryParseUInt64(utf16Text, out value, out charsConsumed);
}
else
{
result = InvariantUtf16.TryParseUInt64(utf16Text, out value, out charsConsumed);
}
bytesConsumed = charsConsumed * sizeof(char);
return result;
}

if (IsHexFormat(format))
{
throw new NotImplementedException("The only supported encodings for hexadecimal parsing are InvariantUtf8 and InvariantUtf16.");
}

if (!(format.IsDefault || format.Symbol == 'G' || format.Symbol == 'g'))
{
throw new NotImplementedException(String.Format("Format '{0}' not supported.", format.Symbol));
}

SymbolTable.Symbol nextSymbol;
int thisSymbolConsumed;
if (!symbolTable.TryParse(text, out nextSymbol, out thisSymbolConsumed))
{
value = default;
bytesConsumed = 0;
return false;
}

if (nextSymbol > SymbolTable.Symbol.D9)
{
value = default;
bytesConsumed = 0;
return false;
}

ulong parsedValue = (uint)nextSymbol;
int index = thisSymbolConsumed;

while (index < text.Length)
{
bool success = symbolTable.TryParse(text.Slice(index), out nextSymbol, out thisSymbolConsumed);
if (!success || nextSymbol > SymbolTable.Symbol.D9)
{
bytesConsumed = index;
value = (ulong) parsedValue;
return true;
}

// If parsedValue > (ulong.MaxValue / 10), any more appended digits will cause overflow.
// if parsedValue == (ulong.MaxValue / 10), any nextDigit greater than 5 implies overflow.
if (parsedValue > ulong.MaxValue / 10 || (parsedValue == ulong.MaxValue / 10 && nextSymbol > SymbolTable.Symbol.D5))
{
bytesConsumed = 0;
value = default;
return false;
}

index += thisSymbolConsumed;
parsedValue = parsedValue * 10 + (uint)nextSymbol;
}

bytesConsumed = text.Length;
value = (ulong) parsedValue;
return true;
}
}
}

+ 35
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf16LittleEndianCodePointEnumerable.cs View File

@@ -0,0 +1,35 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Collections;
using System.Collections.Generic;

namespace System.Text.Utf16
{
// TODO: Should this and Utf8 code point enumerators/enumerable be subclasses of Utf8/16Encoder?
internal struct Utf16LittleEndianCodePointEnumerable : IEnumerable<uint>, IEnumerable
{
private string _s;

public Utf16LittleEndianCodePointEnumerable(string s)
{
_s = s;
}

public Utf16LittleEndianCodePointEnumerator GetEnumerator()
{
return new Utf16LittleEndianCodePointEnumerator(_s);
}

IEnumerator<uint> IEnumerable<uint>.GetEnumerator()
{
return GetEnumerator();
}

IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}

+ 86
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf16LittleEndianCodePointEnumerator.cs View File

@@ -0,0 +1,86 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Collections;
using System.Collections.Generic;

namespace System.Text.Utf16
{
internal struct Utf16LittleEndianCodePointEnumerator : IEnumerator<uint>, IEnumerator
{
string _s;
int _index;

int _encodedChars;
uint _codePoint;

public Utf16LittleEndianCodePointEnumerator(string s)
{
_s = s;
_index = -1;
_encodedChars = 0;
_codePoint = default;
}

public uint Current
{
get
{
if (_encodedChars != 0)
{
return _codePoint;
}

if (_index < 0 || _index >= _s.Length)
{
throw new InvalidOperationException("Enumerator is on invalid position");
}

if (!Utf8Helper.TryDecodeCodePointFromString(_s, _index, out _codePoint, out _encodedChars))
{
_codePoint = default;
_encodedChars = 0;
// or index outside of string
throw new InvalidOperationException("Invalid characters in the string");
}

if (_encodedChars <= 0)
{
// TODO: Change exception type
throw new Exception("Internal error: CodePoint is decoded but number of characters read is 0 or negative");
}

return _codePoint;
}
}

public void Reset()
{
_index = -1;
_encodedChars = 0;
_codePoint = default;
}

public bool MoveNext()
{
if (_index == -1)
{
_index = 0;
_encodedChars = 0;
}
else
{
uint dummy = Current;
_index += _encodedChars;
_encodedChars = 0;
}

return _index < _s.Length;
}

object IEnumerator.Current { get { return Current; } }

void IDisposable.Dispose() { }
}
}

+ 232
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf8Helper.cs View File

@@ -0,0 +1,232 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.CompilerServices;

namespace System.Text
{
static class Utf8Helper
{
#region Constants

// TODO: Make this immutable and let them be strong typed
// http://unicode.org/cldr/utility/list-unicodeset.jsp?a=\p{whitespace}&g=&i=
private static readonly uint[] SortedWhitespaceCodePoints = new uint[]
{
0x0009, 0x000A, 0x000B, 0x000C, 0x000D,
0x0020,
0x0085,
0x00A0,
0x1680,
0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006,
0x2007,
0x2008, 0x2009, 0x200A,
0x2028, 0x2029,
0x202F,
0x205F,
0x3000
};

// To get this to compile with dotnet cli, we need to temporarily un-binary the magic values
private const byte b0000_0111U = 0x07; //7
private const byte b0000_1111U = 0x0F; //15
private const byte b0001_1111U = 0x1F; //31
private const byte b0011_1111U = 0x3F; //63
private const byte b0111_1111U = 0x7F; //127
private const byte b1000_0000U = 0x80; //128
private const byte b1100_0000U = 0xC0; //192
private const byte b1110_0000U = 0xE0; //224
private const byte b1111_0000U = 0xF0; //240
private const byte b1111_1000U = 0xF8; //248

private const byte NonFirstByteInCodePointValue = 0x80;
private const byte NonFirstByteInCodePointMask = 0xC0;

public const int MaxCodeUnitsPerCodePoint = 4;

#endregion Constants

public static bool TryDecodeCodePoint(ReadOnlySpan<byte> utf8, int index, out uint codePoint, out int bytesConsumed)
{
if (index >= utf8.Length)
{
codePoint = default;
bytesConsumed = 0;
return false;
}

var first = utf8[index];

bytesConsumed = GetEncodedBytes(first);
if (bytesConsumed == 0 || utf8.Length - index < bytesConsumed)
{
bytesConsumed = 0;
codePoint = default;
return false;
}

switch (bytesConsumed)
{
case 1:
codePoint = first;
break;

case 2:
codePoint = (uint)(first & b0001_1111U);
break;

case 3:
codePoint = (uint)(first & b0000_1111U);
break;

case 4:
codePoint = (uint)(first & b0000_0111U);
break;

default:
codePoint = default;
bytesConsumed = 0;
return false;
}

for (var i = 1; i < bytesConsumed; i++)
{
uint current = utf8[index + i];
if ((current & b1100_0000U) != b1000_0000U)
{
bytesConsumed = 0;
codePoint = default;
return false;
}

codePoint = (codePoint << 6) | (b0011_1111U & current);
}

return true;
}

private static int GetEncodedBytes(byte b)
{
if ((b & b1000_0000U) == 0)
return 1;

if ((b & b1110_0000U) == b1100_0000U)
return 2;

if ((b & b1111_0000U) == b1110_0000U)
return 3;

if ((b & b1111_1000U) == b1111_0000U)
return 4;

return 0;
}

public static int GetNumberOfEncodedBytes(uint codePoint)
{
if (codePoint <= 0x7F)
return 1;

if (codePoint <= 0x7FF)
return 2;

if (codePoint <= 0xFFFF)
return 3;

if (codePoint <= 0x10FFFF)
return 4;

return 0;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsFirstCodeUnitInEncodedCodePoint(byte codeUnit)
{
return (codeUnit & NonFirstByteInCodePointMask) != NonFirstByteInCodePointValue;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryFindEncodedCodePointBytesCountGoingBackwards(ReadOnlySpan<byte> buffer, out int encodedBytes)
{
encodedBytes = 1;
ReadOnlySpan<byte> it = buffer;
// TODO: Should we have something like: Span<byte>.(Slice from the back)
for (; encodedBytes <= MaxCodeUnitsPerCodePoint; encodedBytes++, it = it.Slice(0, it.Length - 1))
{
if (it.Length == 0)
{
encodedBytes = default;
return false;
}

// TODO: Should we have Span<byte>.Last?
if (IsFirstCodeUnitInEncodedCodePoint(it[it.Length - 1]))
{
// output: encodedBytes
return true;
}
}

// Invalid unicode character or stream prematurely ended (which is still invalid character in that stream)
encodedBytes = default;
return false;
}

// TODO: Name TBD
// TODO: optimize?
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool TryDecodeCodePointBackwards(ReadOnlySpan<byte> buffer, out uint codePoint, out int encodedBytes)
{
if (TryFindEncodedCodePointBytesCountGoingBackwards(buffer, out encodedBytes))
{
int realEncodedBytes;
// TODO: Inline decoding, as the invalid surrogate check can be done faster
bool ret = TryDecodeCodePoint(buffer, buffer.Length - encodedBytes, out codePoint, out realEncodedBytes);
if (ret && encodedBytes != realEncodedBytes)
{
// invalid surrogate character
// we know the character length by iterating on surrogate characters from the end
// but the first byte of the character has also encoded length
// seems like the lengths don't match
codePoint = default;
return false;
}

return true;
}

codePoint = default;
encodedBytes = default;
return false;
}

// TODO: Should we rewrite this to not use char.ConvertToUtf32 or is it fast enough?
public static bool TryDecodeCodePointFromString(string s, int index, out uint codePoint, out int encodedChars)
{
if (index < 0 || index >= s.Length)
{
codePoint = default;
encodedChars = 0;
return false;
}

if (index == s.Length - 1 && char.IsSurrogate(s[index]))
{
codePoint = default;
encodedChars = 0;
return false;
}

encodedChars = char.IsHighSurrogate(s[index]) ? 2 : 1;
codePoint = unchecked((uint)char.ConvertToUtf32(s, index));

return true;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsWhitespace(uint codePoint)
{
return Array.BinarySearch<uint>(SortedWhitespaceCodePoints, codePoint) >= 0;
}
}
}

+ 45
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf8String.CodePointEnumerable.cs View File

@@ -0,0 +1,45 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.


namespace System.Text.Utf8
{
partial struct Utf8String
{
public struct CodePointEnumerable
{
private ReadOnlySpan<byte> _buffer;

public CodePointEnumerable(byte[] bytes, int index, int length)
{
_buffer = new ReadOnlySpan<byte>(bytes, index, length);
}

public unsafe CodePointEnumerable(ReadOnlySpan<byte> buffer)
{
_buffer = buffer;
}

public CodePointEnumerator GetEnumerator()
{
return new CodePointEnumerator(_buffer);
}

public CodePointReverseEnumerator GetReverseEnumerator()
{
return new CodePointReverseEnumerator(_buffer);
}

public int Count()
{
int result = 0;
foreach (var cp in this)
{
result++;
}

return result;
}
}
}
}

+ 120
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf8String.CodePointEnumerator.cs View File

@@ -0,0 +1,120 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.


namespace System.Text.Utf8
{
partial struct Utf8String
{
public struct CodePointEnumerator
{
private ReadOnlySpan<byte> _buffer;
private int _index;
private int _currentLenCache;
private const int ResetIndex = -Utf8Helper.MaxCodeUnitsPerCodePoint - 1;

public unsafe CodePointEnumerator(ReadOnlySpan<byte> buffer) : this()
{
_buffer = buffer;

Reset();
}

// TODO: Name TBD
public int PositionInCodeUnits
{
get
{
if (IsOnResetPosition())
{
return -1;
}

return _index;
}
}

public unsafe uint Current
{
get
{
if (IsOnResetPosition())
{
throw new InvalidOperationException("MoveNext() needs to be called at least once");
}

if (!HasValue())
{
throw new InvalidOperationException("Current does not exist");
}

uint codePoint;
bool succeeded = Utf8Helper.TryDecodeCodePoint(_buffer, _index, out codePoint, out _currentLenCache);

if (!succeeded || _currentLenCache == 0)
{
// TODO: Change exception type
throw new Exception("Invalid code point!");
}

return codePoint;
}
}

public bool MoveNext()
{
if (!HasValue())
{
return false;
}
if (IsOnResetPosition())
{
MoveToFirstPosition();
return HasValue();
}

if (_currentLenCache == 0)
{
uint codePointDummy = Current;
if (_currentLenCache == 0)
{
throw new Exception("Invalid UTF-8 character (badly encoded)");
}
}

_index += _currentLenCache;
_currentLenCache = 0;

return HasValue();
}

// This is different than Reset, it goes to the first element not before first
private void MoveToFirstPosition()
{
_index = 0;
}

private bool IsOnResetPosition()
{
return _index == ResetIndex;
}

private bool HasValue()
{
if (IsOnResetPosition())
{
return true;
}

return _index < _buffer.Length;
}

// This is different than MoveToFirstPosition, this actually goes before anything
public void Reset()
{
_index = ResetIndex;
_currentLenCache = 0;
}
}
}
}

+ 122
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf8String.CodePointReverseEnumerator.cs View File

@@ -0,0 +1,122 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.


namespace System.Text.Utf8
{
partial struct Utf8String
{
// TODO: Name TBD
public struct CodePointReverseEnumerator
{
private ReadOnlySpan<byte> _buffer;
private int _index;
private int _currentLenCache;
private const int ResetIndex = -Utf8Helper.MaxCodeUnitsPerCodePoint - 1;

public unsafe CodePointReverseEnumerator(ReadOnlySpan<byte> buffer) : this()
{
_buffer = buffer;

Reset();
}

// TODO: Name TBD
public int PositionInCodeUnits
{
get
{
if (IsOnResetPosition())
{
return -1;
}

return _index;
}
}

public unsafe uint Current
{
get
{
if (IsOnResetPosition())
{
throw new InvalidOperationException("MoveNext() needs to be called at least once");
}

if (!HasValue())
{
throw new InvalidOperationException("Current does not exist");
}

ReadOnlySpan<byte> buffer = _buffer.Slice(0, _index);
uint ret;
bool succeeded = Utf8Helper.TryDecodeCodePointBackwards(buffer, out ret, out _currentLenCache);

if (!succeeded || _currentLenCache == 0)
{
// TODO: Change exception type
throw new Exception("Invalid code point!");
}

return ret;
}
}

public bool MoveNext()
{
if (!HasValue())
{
return false;
}
if (IsOnResetPosition())
{
MoveToFirstPosition();
return HasValue();
}

if (_currentLenCache == 0)
{
uint codePointDummy = Current;
if (_currentLenCache == 0)
{
throw new Exception("Invalid UTF-8 character (badly encoded)");
}
}

_index -= _currentLenCache;
_currentLenCache = 0;

return HasValue();
}

// This is different than Reset, it goes to the first element not before first
private void MoveToFirstPosition()
{
_index = _buffer.Length;
}

private bool IsOnResetPosition()
{
return _index == ResetIndex;
}

private bool HasValue()
{
if (IsOnResetPosition())
{
return true;
}

return _index > 0;
}

// This is different than MoveToFirstPosition, this actually goes before anything
public void Reset()
{
_index = ResetIndex;
_currentLenCache = 0;
}
}
}
}

+ 41
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf8String.Enumerator.cs View File

@@ -0,0 +1,41 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.


namespace System.Text.Utf8
{
partial struct Utf8String
{
public struct Enumerator
{
private readonly ReadOnlySpan<byte> _buffer;
private readonly int _length;
private int _index;

internal Enumerator(ReadOnlySpan<byte> buffer)
{
_buffer = buffer;
_length = buffer.Length;
_index = -1;
}

public byte Current
{
get
{
return _buffer[_index];
}
}

public bool MoveNext()
{
return ++_index < _length;
}

public void Reset()
{
_index = -1;
}
}
}
}

+ 641
- 0
src/Discord.Net.Core/_corefxlab/System.Text.Utf8String/System/Text/Utf8String.cs View File

@@ -0,0 +1,641 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Text.Utf16;

namespace System.Text.Utf8
{
[DebuggerDisplay("{ToString()}u8")]
public partial struct Utf8String
{
private readonly ReadOnlySpan<byte> _buffer;

private const int StringNotFound = -1;

static Utf8String s_empty => default;

// TODO: Validate constructors, When should we copy? When should we just use the underlying array?
// TODO: Should we be immutable/readonly?
public Utf8String(ReadOnlySpan<byte> buffer)
{
_buffer = buffer;
}

public Utf8String(byte[] utf8bytes)
{
_buffer = new ReadOnlySpan<byte>(utf8bytes);
}

public Utf8String(byte[] utf8bytes, int index, int length)
{
_buffer = new ReadOnlySpan<byte>(utf8bytes, index, length);
}

public Utf8String(string s)
{
if (s == null)
{
throw new ArgumentNullException("s", "String cannot be null");
}

if (s == string.Empty)
{
_buffer = ReadOnlySpan<byte>.Empty;
}
else
{
_buffer = new ReadOnlySpan<byte>(GetUtf8BytesFromString(s));
}
}

/// <summary>
/// This constructor is for use by the compiler.
/// </summary>
[EditorBrowsable(EditorBrowsableState.Never)]
public Utf8String(RuntimeFieldHandle utf8Data, int length) : this(CreateArrayFromFieldHandle(utf8Data, length))
{
}

public static explicit operator Utf8String(ArraySegment<byte> utf8Bytes)
{
return new Utf8String(utf8Bytes);
}

static byte[] CreateArrayFromFieldHandle(RuntimeFieldHandle utf8Data, int length)
{
var array = new byte[length];
RuntimeHelpers.InitializeArray(array, utf8Data);
return array;
}

public static Utf8String Empty { get { return s_empty; } }

/// <summary>
/// Returns length of the string in UTF-8 code units (bytes)
/// </summary>
public int Length
{
get
{
return _buffer.Length;
}
}

public Enumerator GetEnumerator()
{
return new Enumerator(_buffer);
}

public CodePointEnumerable CodePoints
{
get
{
return new CodePointEnumerable(_buffer);
}
}

public byte this[int i]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
{
// there is no need to check the boundaries -> Span is going to do this on it's own
return (byte)_buffer[i];
}
}

public static implicit operator ReadOnlySpan<byte>(Utf8String utf8)
{
return utf8.Bytes;
}

public static explicit operator Utf8String(string s)
{
return new Utf8String(s);
}

public static explicit operator string(Utf8String s)
{
return s.ToString();
}

public ReadOnlySpan<byte> Bytes => _buffer;

public override string ToString()
{
var status = Encoders.Utf8.ToUtf16Length(this.Bytes, out int needed);
if (status != Buffers.TransformationStatus.Done)
return string.Empty;

// UTF-16 is 2 bytes per char
var chars = new char[needed >> 1];
var utf16 = new Span<char>(chars).AsBytes();
status = Encoders.Utf8.ToUtf16(this.Bytes, utf16, out int consumed, out int written);
if (status != Buffers.TransformationStatus.Done)
return string.Empty;

return new string(chars);
}

public bool ReferenceEquals(Utf8String other)
{
return _buffer == other._buffer;
}

public bool Equals(Utf8String other)
{
return _buffer.SequenceEqual(other._buffer);
}

public bool Equals(string other)
{
CodePointEnumerator thisEnumerator = GetCodePointEnumerator();
Utf16LittleEndianCodePointEnumerator otherEnumerator = new Utf16LittleEndianCodePointEnumerator(other);

while (true)
{
bool hasNext = thisEnumerator.MoveNext();
if (hasNext != otherEnumerator.MoveNext())
{
return false;
}

if (!hasNext)
{
return true;
}

if (thisEnumerator.Current != otherEnumerator.Current)
{
return false;
}
}
}

public static bool operator ==(Utf8String left, Utf8String right)
{
return left.Equals(right);
}

public static bool operator !=(Utf8String left, Utf8String right)
{
return !left.Equals(right);
}

public static bool operator ==(Utf8String left, string right)
{
return left.Equals(right);
}

public static bool operator !=(Utf8String left, string right)
{
return !left.Equals(right);
}

public static bool operator ==(string left, Utf8String right)
{
return right.Equals(left);
}

public static bool operator !=(string left, Utf8String right)
{
return !right.Equals(left);
}

public int CompareTo(Utf8String other)
{
throw new NotImplementedException();
}

public int CompareTo(string other)
{
throw new NotImplementedException();
}

/// <summary>
///
/// </summary>
/// <param name="index">Index in UTF-8 code units (bytes)</param>
/// <returns>Length in UTF-8 code units (bytes)</returns>
public Utf8String Substring(int index)
{
return Substring(index, Length - index);
}

/// <summary>
///
/// </summary>
/// <param name="index">Index in UTF-8 code units (bytes)</param>
/// <returns>Length in UTF-8 code units (bytes)</returns>
public Utf8String Substring(int index, int length)
{
if (index < 0)
{
throw new ArgumentOutOfRangeException("index");
}

if (length < 0)
{
// TODO: Should we support that?
throw new ArgumentOutOfRangeException("length");
}

if (length == 0)
{
return Empty;
}

if (length == Length)
{
return this;
}

if (index + length > Length)
{
// TODO: Should this be index or length?
throw new ArgumentOutOfRangeException("index");
}

return new Utf8String(_buffer.Slice(index, length));
}

// TODO: Naive algorithm, reimplement faster
// TODO: Should this be public?
public int IndexOf(Utf8String value)
{
if (value.Length == 0)
{
// TODO: Is this the right answer?
// TODO: Does this even make sense?
return 0;
}

if (Length == 0)
{
return StringNotFound;
}

Utf8String restOfTheString = this;
for (int i = 0; restOfTheString.Length <= Length; restOfTheString = Substring(++i))
{
int pos = restOfTheString.IndexOf(value[0]);
if (pos == StringNotFound)
{
return StringNotFound;
}
i += pos;
if (IsSubstringAt(i, value))
{
return i;
}
}

return StringNotFound;
}

// TODO: Should this be public?
public int IndexOf(byte codeUnit)
{
// TODO: _buffer.IndexOf(codeUnit.Value); when Span has it

for (int i = 0; i < Length; i++)
{
if (codeUnit == this[i])
{
return i;
}
}

return StringNotFound;
}

// TODO: Should this be public?
public int IndexOf(uint codePoint)
{
CodePointEnumerator it = GetCodePointEnumerator();
while (it.MoveNext())
{
if (it.Current == codePoint)
{
return it.PositionInCodeUnits;
}
}

return StringNotFound;
}

// TODO: Re-evaluate all Substring family methods and check their parameters name
public bool TrySubstringFrom(Utf8String value, out Utf8String result)
{
int idx = IndexOf(value);

if (idx == StringNotFound)
{
result = default;
return false;
}

result = Substring(idx);
return true;
}

public bool TrySubstringFrom(byte codeUnit, out Utf8String result)
{
int idx = IndexOf(codeUnit);

if (idx == StringNotFound)
{
result = default;
return false;
}

result = Substring(idx);
return true;
}

public bool TrySubstringFrom(uint codePoint, out Utf8String result)
{
int idx = IndexOf(codePoint);

if (idx == StringNotFound)
{
result = default;
return false;
}

result = Substring(idx);
return true;
}

public bool TrySubstringTo(Utf8String value, out Utf8String result)
{
int idx = IndexOf(value);

if (idx == StringNotFound)
{
result = default;
return false;
}

result = Substring(0, idx);
return true;
}

public bool TrySubstringTo(byte codeUnit, out Utf8String result)
{
int idx = IndexOf(codeUnit);

if (idx == StringNotFound)
{
result = default;
return false;
}

result = Substring(0, idx);
return true;
}

public bool TrySubstringTo(uint codePoint, out Utf8String result)
{
int idx = IndexOf(codePoint);

if (idx == StringNotFound)
{
result = default;
return false;
}

result = Substring(0, idx);
return true;
}

public bool IsSubstringAt(int index, Utf8String s)
{
if (index < 0 || index + s.Length > Length)
{
return false;
}

return Substring(index, s.Length).Equals(s);
}

public void CopyTo(Span<byte> buffer)
{
_buffer.CopyTo(buffer);
}

public void CopyTo(byte[] buffer)
{
_buffer.CopyTo(buffer);
}

// TODO: write better hashing function
// TODO: span.GetHashCode() + some constant?
public override int GetHashCode()
{
unchecked
{
if (Length <= 4)
{
int hash = Length;
for (int i = 0; i < Length; i++)
{
hash <<= 8;
hash ^= (byte)this[i];
}
return hash;
}
else
{
int hash = Length;
hash ^= (byte)this[0];
hash <<= 8;
hash ^= (byte)this[1];
hash <<= 8;
hash ^= (byte)this[Length - 2];
hash <<= 8;
hash ^= (byte)this[Length - 1];
return hash;
}
}
}

public override bool Equals(object obj)
{
if (obj is Utf8String)
{
return Equals((Utf8String)obj);
}
if (obj is string)
{
return Equals((string)obj);
}

return false;
}

private CodePointEnumerator GetCodePointEnumerator()
{
return new CodePointEnumerator(_buffer);
}

public bool StartsWith(uint codePoint)
{
CodePointEnumerator e = GetCodePointEnumerator();
if (!e.MoveNext())
{
return false;
}

return e.Current == codePoint;
}

public bool StartsWith(byte codeUnit)
{
if (Length == 0)
{
return false;
}

return this[0] == codeUnit;
}

public bool StartsWith(Utf8String value)
{
if(value.Length > this.Length)
{
return false;
}

return this.Substring(0, value.Length).Equals(value);
}

public bool EndsWith(byte codeUnit)
{
if (Length == 0)
{
return false;
}

return this[Length - 1] == codeUnit;
}

public bool EndsWith(Utf8String value)
{
if (Length < value.Length)
{
return false;
}

return this.Substring(Length - value.Length, value.Length).Equals(value);
}

public bool EndsWith(uint codePoint)
{
throw new NotImplementedException();
}

private static int GetUtf8LengthInBytes(IEnumerable<uint> codePoints)
{
int len = 0;
foreach (var codePoint in codePoints)
{
len += Utf8Helper.GetNumberOfEncodedBytes(codePoint);
}

return len;
}

// TODO: This should return Utf16CodeUnits which should wrap byte[]/Span<byte>, same for other encoders
private static byte[] GetUtf8BytesFromString(string str)
{
var utf16 = str.AsSpan().AsBytes();
var status = Encoders.Utf16.ToUtf8Length(utf16, out int needed);
if (status != Buffers.TransformationStatus.Done)
return null;

var utf8 = new byte[needed];
status = Encoders.Utf16.ToUtf8(utf16, utf8, out int consumed, out int written);
if (status != Buffers.TransformationStatus.Done)
// This shouldn't happen...
return null;

return utf8;
}

public Utf8String TrimStart()
{
CodePointEnumerator it = GetCodePointEnumerator();
while (it.MoveNext() && Utf8Helper.IsWhitespace(it.Current))
{
}

return Substring(it.PositionInCodeUnits);
}

public Utf8String TrimStart(uint[] trimCodePoints)
{
throw new NotImplementedException();
}

public Utf8String TrimStart(byte[] trimCodeUnits)
{
throw new NotImplementedException();
}

public Utf8String TrimEnd()
{
CodePointReverseEnumerator it = CodePoints.GetReverseEnumerator();
while (it.MoveNext() && Utf8Helper.IsWhitespace(it.Current))
{
}

return Substring(0, it.PositionInCodeUnits);
}

public Utf8String TrimEnd(uint[] trimCodePoints)
{
throw new NotImplementedException();
}

public Utf8String TrimEnd(byte[] trimCodeUnits)
{
throw new NotImplementedException();
}

public Utf8String Trim()
{
return TrimStart().TrimEnd();
}

public Utf8String Trim(uint[] trimCodePoints)
{
throw new NotImplementedException();
}

public Utf8String Trim(byte[] trimCodeUnits)
{
throw new NotImplementedException();
}

// TODO: Name TBD, CopyArray? GetBytes?
public byte[] CopyBytes()
{
return _buffer.ToArray();
}

public byte[] CopyCodeUnits()
{
throw new NotImplementedException();
}

public static bool IsWhiteSpace(byte codePoint)
{
return codePoint == ' ' || codePoint == '\n' || codePoint == '\r' || codePoint == '\t';
}
}
}

+ 13
- 7
src/Discord.Net.Providers.WS4Net/WS4NetClient.cs View File

@@ -3,6 +3,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Utf8;
using System.Threading;
using System.Threading.Tasks;
using WebSocket4Net;
@@ -12,8 +13,7 @@ namespace Discord.Net.Providers.WS4Net
{
internal class WS4NetClient : IWebSocketClient, IDisposable
{
public event Func<byte[], int, int, Task> BinaryMessage;
public event Func<string, Task> TextMessage;
public event Func<ReadOnlyBuffer<byte>, bool, Task> Message;
public event Func<Exception, Task> Closed;

private readonly SemaphoreSlim _lock;
@@ -129,15 +129,20 @@ namespace Discord.Net.Providers.WS4Net
_cancelToken = CancellationTokenSource.CreateLinkedTokenSource(_parentToken, _cancelTokenSource.Token).Token;
}

public async Task SendAsync(byte[] data, int index, int count, bool isText)
public async Task SendAsync(ReadOnlyBuffer<byte> data, bool isText)
{
await _lock.WaitAsync(_cancelToken).ConfigureAwait(false);
try
{
if (isText)
_client.Send(Encoding.UTF8.GetString(data, index, count));
_client.Send(new Utf8String(data.Span).ToString());
else
_client.Send(data, index, count);
{
if (data.DangerousTryGetArray(out var array))
_client.Send(array.Array, 0, data.Length);
else
_client.Send(data.ToArray(), 0, data.Length);
}
}
finally
{
@@ -147,11 +152,12 @@ namespace Discord.Net.Providers.WS4Net

private void OnTextMessage(object sender, MessageReceivedEventArgs e)
{
TextMessage(e.Message).GetAwaiter().GetResult();
//TODO: Inefficient, but were dropping this plugin ASAP
Message(new ReadOnlyBuffer<byte>(Encoding.UTF8.GetBytes(e.Message)), true).GetAwaiter().GetResult();
}
private void OnBinaryMessage(object sender, DataReceivedEventArgs e)
{
BinaryMessage(e.Data, 0, e.Data.Count()).GetAwaiter().GetResult();
Message(new ReadOnlyBuffer<byte>(e.Data, 0, e.Data.Count()), false).GetAwaiter().GetResult();
}
private void OnConnected(object sender, object e)
{


+ 8
- 8
src/Discord.Net.Rest/API/Common/Application.cs View File

@@ -1,24 +1,24 @@
#pragma warning disable CS1591
using Newtonsoft.Json;
using Discord.Serialization;

namespace Discord.API
{
internal class Application
{
[JsonProperty("description")]
[ModelProperty("description")]
public string Description { get; set; }
[JsonProperty("rpc_origins")]
[ModelProperty("rpc_origins")]
public string[] RPCOrigins { get; set; }
[JsonProperty("name")]
[ModelProperty("name")]
public string Name { get; set; }
[JsonProperty("id")]
[ModelProperty("id")]
public ulong Id { get; set; }
[JsonProperty("icon")]
[ModelProperty("icon")]
public string Icon { get; set; }

[JsonProperty("flags"), Int53]
[ModelProperty("flags"), Int53]
public Optional<ulong> Flags { get; set; }
[JsonProperty("owner")]
[ModelProperty("owner")]
public Optional<User> Owner { get; set; }
}
}

+ 8
- 8
src/Discord.Net.Rest/API/Common/Attachment.cs View File

@@ -1,23 +1,23 @@
#pragma warning disable CS1591
using Newtonsoft.Json;
using Discord.Serialization;

namespace Discord.API
{
internal class Attachment
{
[JsonProperty("id")]
[ModelProperty("id")]
public ulong Id { get; set; }
[JsonProperty("filename")]
[ModelProperty("filename")]
public string Filename { get; set; }
[JsonProperty("size")]
[ModelProperty("size")]
public int Size { get; set; }
[JsonProperty("url")]
[ModelProperty("url")]
public string Url { get; set; }
[JsonProperty("proxy_url")]
[ModelProperty("proxy_url")]
public string ProxyUrl { get; set; }
[JsonProperty("height")]
[ModelProperty("height")]
public Optional<int> Height { get; set; }
[JsonProperty("width")]
[ModelProperty("width")]
public Optional<int> Width { get; set; }
}
}

+ 3
- 3
src/Discord.Net.Rest/API/Common/Ban.cs View File

@@ -1,13 +1,13 @@
#pragma warning disable CS1591
using Newtonsoft.Json;
using Discord.Serialization;

namespace Discord.API
{
internal class Ban
{
[JsonProperty("user")]
[ModelProperty("user")]
public User User { get; set; }
[JsonProperty("reason")]
[ModelProperty("reason")]
public string Reason { get; set; }
}
}

+ 14
- 14
src/Discord.Net.Rest/API/Common/Channel.cs View File

@@ -1,5 +1,5 @@
#pragma warning disable CS1591
using Newtonsoft.Json;
using Discord.Serialization;
using System;

namespace Discord.API
@@ -7,41 +7,41 @@ namespace Discord.API
internal class Channel
{
//Shared
[JsonProperty("id")]
[ModelProperty("id")]
public ulong Id { get; set; }
[JsonProperty("type")]
[ModelProperty("type")]
public ChannelType Type { get; set; }
[JsonProperty("last_message_id")]
[ModelProperty("last_message_id")]
public ulong? LastMessageId { get; set; }

//GuildChannel
[JsonProperty("guild_id")]
[ModelProperty("guild_id")]
public Optional<ulong> GuildId { get; set; }
[JsonProperty("name")]
[ModelProperty("name")]
public Optional<string> Name { get; set; }
[JsonProperty("position")]
[ModelProperty("position")]
public Optional<int> Position { get; set; }
[JsonProperty("permission_overwrites")]
[ModelProperty("permission_overwrites")]
public Optional<Overwrite[]> PermissionOverwrites { get; set; }

//TextChannel
[JsonProperty("topic")]
[ModelProperty("topic")]
public Optional<string> Topic { get; set; }
[JsonProperty("last_pin_timestamp")]
[ModelProperty("last_pin_timestamp")]
public Optional<DateTimeOffset?> LastPinTimestamp { get; set; }

//VoiceChannel
[JsonProperty("bitrate")]
[ModelProperty("bitrate")]
public Optional<int> Bitrate { get; set; }
[JsonProperty("user_limit")]
[ModelProperty("user_limit")]
public Optional<int> UserLimit { get; set; }

//PrivateChannel
[JsonProperty("recipients")]
[ModelProperty("recipients")]
public Optional<User[]> Recipients { get; set; }

//GroupChannel
[JsonProperty("icon")]
[ModelProperty("icon")]
public Optional<string> Icon { get; set; }
}
}

+ 6
- 6
src/Discord.Net.Rest/API/Common/Connection.cs View File

@@ -1,21 +1,21 @@
#pragma warning disable CS1591
using Newtonsoft.Json;
using Discord.Serialization;
using System.Collections.Generic;

namespace Discord.API
{
internal class Connection
{
[JsonProperty("id")]
[ModelProperty("id")]
public string Id { get; set; }
[JsonProperty("type")]
[ModelProperty("type")]
public string Type { get; set; }
[JsonProperty("name")]
[ModelProperty("name")]
public string Name { get; set; }
[JsonProperty("revoked")]
[ModelProperty("revoked")]
public bool Revoked { get; set; }

[JsonProperty("integrations")]
[ModelProperty("integrations")]
public IReadOnlyCollection<ulong> Integrations { get; set; }
}
}

+ 14
- 15
src/Discord.Net.Rest/API/Common/Embed.cs View File

@@ -1,37 +1,36 @@
#pragma warning disable CS1591
using System;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Discord.Serialization;

namespace Discord.API
{
internal class Embed
{
[JsonProperty("title")]
[ModelProperty("title")]
public string Title { get; set; }
[JsonProperty("description")]
[ModelProperty("description")]
public string Description { get; set; }
[JsonProperty("url")]
[ModelProperty("url")]
public string Url { get; set; }
[JsonProperty("color")]
[ModelProperty("color")]
public uint? Color { get; set; }
[JsonProperty("type"), JsonConverter(typeof(StringEnumConverter))]
[ModelProperty("type")]
public EmbedType Type { get; set; }
[JsonProperty("timestamp")]
[ModelProperty("timestamp")]
public DateTimeOffset? Timestamp { get; set; }
[JsonProperty("author")]
[ModelProperty("author")]
public Optional<EmbedAuthor> Author { get; set; }
[JsonProperty("footer")]
[ModelProperty("footer")]
public Optional<EmbedFooter> Footer { get; set; }
[JsonProperty("video")]
[ModelProperty("video")]
public Optional<EmbedVideo> Video { get; set; }
[JsonProperty("thumbnail")]
[ModelProperty("thumbnail")]
public Optional<EmbedThumbnail> Thumbnail { get; set; }
[JsonProperty("image")]
[ModelProperty("image")]
public Optional<EmbedImage> Image { get; set; }
[JsonProperty("provider")]
[ModelProperty("provider")]
public Optional<EmbedProvider> Provider { get; set; }
[JsonProperty("fields")]
[ModelProperty("fields")]
public Optional<EmbedField[]> Fields { get; set; }
}
}

+ 5
- 5
src/Discord.Net.Rest/API/Common/EmbedAuthor.cs View File

@@ -1,17 +1,17 @@
using System;
using Newtonsoft.Json;
using Discord.Serialization;

namespace Discord.API
{
internal class EmbedAuthor
{
[JsonProperty("name")]
[ModelProperty("name")]
public string Name { get; set; }
[JsonProperty("url")]
[ModelProperty("url")]
public string Url { get; set; }
[JsonProperty("icon_url")]
[ModelProperty("icon_url")]
public string IconUrl { get; set; }
[JsonProperty("proxy_icon_url")]
[ModelProperty("proxy_icon_url")]
public string ProxyIconUrl { get; set; }
}
}

+ 4
- 4
src/Discord.Net.Rest/API/Common/EmbedField.cs View File

@@ -1,14 +1,14 @@
using Newtonsoft.Json;
using Discord.Serialization;

namespace Discord.API
{
internal class EmbedField
{
[JsonProperty("name")]
[ModelProperty("name")]
public string Name { get; set; }
[JsonProperty("value")]
[ModelProperty("value")]
public string Value { get; set; }
[JsonProperty("inline")]
[ModelProperty("inline")]
public bool Inline { get; set; }
}
}

+ 4
- 4
src/Discord.Net.Rest/API/Common/EmbedFooter.cs View File

@@ -1,15 +1,15 @@
using System;
using Newtonsoft.Json;
using Discord.Serialization;

namespace Discord.API
{
internal class EmbedFooter
{
[JsonProperty("text")]
[ModelProperty("text")]
public string Text { get; set; }
[JsonProperty("icon_url")]
[ModelProperty("icon_url")]
public string IconUrl { get; set; }
[JsonProperty("proxy_icon_url")]
[ModelProperty("proxy_icon_url")]
public string ProxyIconUrl { get; set; }
}
}

+ 5
- 5
src/Discord.Net.Rest/API/Common/EmbedImage.cs View File

@@ -1,18 +1,18 @@
#pragma warning disable CS1591
using System;
using Newtonsoft.Json;
using Discord.Serialization;

namespace Discord.API
{
internal class EmbedImage
{
[JsonProperty("url")]
[ModelProperty("url")]
public string Url { get; set; }
[JsonProperty("proxy_url")]
[ModelProperty("proxy_url")]
public string ProxyUrl { get; set; }
[JsonProperty("height")]
[ModelProperty("height")]
public Optional<int> Height { get; set; }
[JsonProperty("width")]
[ModelProperty("width")]
public Optional<int> Width { get; set; }
}
}

+ 3
- 3
src/Discord.Net.Rest/API/Common/EmbedProvider.cs View File

@@ -1,14 +1,14 @@
#pragma warning disable CS1591
using System;
using Newtonsoft.Json;
using Discord.Serialization;

namespace Discord.API
{
internal class EmbedProvider
{
[JsonProperty("name")]
[ModelProperty("name")]
public string Name { get; set; }
[JsonProperty("url")]
[ModelProperty("url")]
public string Url { get; set; }
}
}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save