Added support for strings containing escapes

src/Discord.Net.Serialization/Discord.Net.Serialization.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <Import Project="../../Discord.Net.targets" />
   <PropertyGroup>
     <AssemblyName>Discord.Net.Serialization</AssemblyName>
@@ -8,11 +8,11 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="System.Buffers" Version="4.4.0-preview2-25405-01" />
-    <PackageReference Include="System.Collections.Immutable" Version="1.3.1" />
+    <PackageReference Include="System.Buffers" Version="4.4.0" />
+    <PackageReference Include="System.Collections.Immutable" Version="1.4.0" />
     <PackageReference Include="System.Interactive.Async" Version="3.1.1" />
     <PackageReference Include="System.Memory" Version="4.4.0-preview2-25405-01" />
-    <PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="4.4.0-preview2-25405-01" />
-    <PackageReference Include="System.ValueTuple" Version="4.3.1" />
+    <PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="4.4.0" />
+    <PackageReference Include="System.ValueTuple" Version="4.4.0" />
   </ItemGroup>
-</Project>
+</Project>

src/Discord.Net.Serialization/Extensions/BufferExtensions.cs

@@ -107,14 +107,17 @@ namespace Discord.Serialization
         {
             int index = 0;
             bytesConsumed = 0;
-            if (!TryParseDateParts(text, ref index, ref bytesConsumed, out int year, out int month, out int day) ||
-                !TryParseTimeParts(text, ref index, ref bytesConsumed, out int hour, out int min, out int sec, out int milli) ||
-                !TryParseTimezoneParts(text, ref index, ref bytesConsumed, out var offset))
+            if (!TryParseDateParts(text, ref index, out int year, out int month, out int day) ||
+                !TryParseTimeParts(text, ref index, out int hour, out int min, out int sec, out int milli, out int milliLength) ||
+                !TryParseTimezoneParts(text, ref index, out var offset))
             {
                 value = default;
                 return false;
             }
 
+            if (milliLength == 6)
+                milli /= 1000;
+
             value = new DateTime(year, month, day, hour, min, sec, milli, DateTimeKind.Utc);
             if (offset != TimeSpan.Zero)
                 value -= offset;
@@ -124,19 +127,23 @@ namespace Discord.Serialization
         {
             int index = 0;
             bytesConsumed = 0;
-            if (!TryParseDateParts(text, ref index, ref bytesConsumed, out int year, out int month, out int day) ||
-                !TryParseTimeParts(text, ref index, ref bytesConsumed, out int hour, out int min, out int sec, out int milli) ||
-                !TryParseTimezoneParts(text, ref index, ref bytesConsumed, out var offset))
+            if (!TryParseDateParts(text, ref index, out int year, out int month, out int day) ||
+                !TryParseTimeParts(text, ref index, out int hour, out int min, out int sec, out int milli, out int milliLength) ||
+                !TryParseTimezoneParts(text, ref index, out var offset))
             {
                 value = default;
                 return false;
             }
+
+            if (milliLength == 6)
+                milli /= 1000;
+
             value = new DateTimeOffset(year, month, day, hour, min, sec, milli, offset);
             return true;
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static bool TryParseDateParts(ReadOnlySpan<byte> text, ref int index, ref int bytesConsumed,
+        private static bool TryParseDateParts(ReadOnlySpan<byte> text, ref int index, 
             out int year, out int month, out int day)
         {
             year = 0;
@@ -145,65 +152,54 @@ namespace Discord.Serialization
 
             //Format: YYYY-MM-DD
             if (text.Length < 10 ||
-                !TryParseNumericPart(text, ref index, out year, ref bytesConsumed, 4) ||
+                !TryParseNumericPart(text, ref index, out year, out var ignored, 4) ||
                 text[index++] != (byte)'-' ||
-                !TryParseNumericPart(text, ref index, out month, ref bytesConsumed, 2) ||
+                !TryParseNumericPart(text, ref index, out month, out ignored, 2) ||
                 text[index++] != (byte)'-' ||
-                !TryParseNumericPart(text, ref index, out day, ref bytesConsumed, 2))
-            {
-                bytesConsumed = 0;
+                !TryParseNumericPart(text, ref index, out day, out ignored, 2))
                 return false;
-            }
             return true;
         }
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static bool TryParseTimeParts(ReadOnlySpan<byte> text, ref int index, ref int bytesConsumed,
-            out int hour, out int minute, out int second, out int millisecond)
+        private static bool TryParseTimeParts(ReadOnlySpan<byte> text, ref int index,
+            out int hour, out int minute, out int second, out int millisecond, out int milliLength)
         {
             hour = 0;
             minute = 0;
             second = 0;
             millisecond = 0;
+            milliLength = 0;
 
             //Time (hh:mm)
             if (text.Length < 16 || text[index] != (byte)'T')  //0001-01-01T01:01
                 return true;
             index++;
 
-            if (!TryParseNumericPart(text, ref index, out hour, ref bytesConsumed, 2) ||
+            if (!TryParseNumericPart(text, ref index, out hour, out var ignored, 2) ||
                 text[index++] != (byte)':' ||
-                !TryParseNumericPart(text, ref index, out minute, ref bytesConsumed, 2))
-            {
-                bytesConsumed = 0;
+                !TryParseNumericPart(text, ref index, out minute, out ignored, 2))
                 return false;
-            }
 
             //Time (hh:mm:ss)
             if (text.Length < 19 || text[index] != (byte)':')  //0001-01-01T01:01:01
                 return true;
             index++;
 
-            if (!TryParseNumericPart(text, ref index, out second, ref bytesConsumed, 2))
-            {
-                bytesConsumed = 0;
+            if (!TryParseNumericPart(text, ref index, out second, out ignored, 2))
                 return false;
-            }
 
             //Time (hh:mm:ss.sss)
             if (text.Length < 21 || text[index] != (byte)'.')  //0001-01-01T01:01:01.1
                 return true;
             index++;
-
-            if (!TryParseNumericPart(text, ref index, out millisecond, ref bytesConsumed, 3))
-            {
-                bytesConsumed = 0;
+            
+            if (!TryParseNumericPart(text, ref index, out millisecond, out milliLength, 6))
                 return false;
-            }
 
             return true;
         }
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static bool TryParseTimezoneParts(ReadOnlySpan<byte> text, ref int index, ref int bytesConsumed,
+        private static bool TryParseTimezoneParts(ReadOnlySpan<byte> text, ref int index,
             out TimeSpan offset)
         {
             offset = default;
@@ -222,13 +218,10 @@ namespace Discord.Serialization
                     return false;
                 index++;
                 
-                if (!TryParseNumericPart(text, ref index, out int hours, ref bytesConsumed, 2) ||
+                if (!TryParseNumericPart(text, ref index, out int hours, out var ignored, 2) ||
                     text[index++] != (byte)':' ||
-                    !TryParseNumericPart(text, ref index, out int minutes, ref bytesConsumed, 2))
-                {
-                    bytesConsumed = 0;
+                    !TryParseNumericPart(text, ref index, out int minutes, out ignored, 2))
                     return false;
-                }
                 offset = new TimeSpan(hours, minutes, 0);
                 if (isNegative) offset = -offset;
                 return true;
@@ -239,16 +232,17 @@ namespace Discord.Serialization
 
         //From https://github.com/dotnet/corefxlab/blob/master/src/System.Text.Primitives/System/Text/Parsing/Unsigned.cs
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static bool TryParseNumericPart(ReadOnlySpan<byte> text, ref int index, out int value, ref int bytesConsumed, int maxLength)
+        private static bool TryParseNumericPart(ReadOnlySpan<byte> text, ref int index, out int value, out int valueLength, int maxLength)
         {
             // Parse the first digit separately. If invalid here, we need to return false.
             uint firstDigit = text[index++] - 48u; // '0'
             if (firstDigit > 9)
             {
-                bytesConsumed = 0;
+                valueLength = 0;
                 value = default;
                 return false;
             }
+            valueLength = 1;
             uint parsedValue = firstDigit;
             
             for (int i = 1; i < maxLength && index < text.Length; i++, index++)
@@ -256,14 +250,13 @@ namespace Discord.Serialization
                 uint nextDigit = text[index] - 48u; // '0'
                 if (nextDigit > 9)
                 {
-                    bytesConsumed = index;
                     value = (int)(parsedValue);
                     return true;
                 }
+                valueLength++;
                 parsedValue = parsedValue * 10 + nextDigit;
             }
-
-            bytesConsumed = text.Length;
+            
             value = (int)(parsedValue);
             return true;
         }

src/Discord.Net.Serialization/Json/JsonSerializer.cs

@@ -3,6 +3,7 @@ using System.Reflection;
 using System.Text;
 using System.Text.Formatting;
 using System.Text.Json;
+using System.Text.Utf8;
 
 namespace Discord.Serialization.Json
 {
@@ -22,15 +23,18 @@ namespace Discord.Serialization.Json
             var converter = (JsonPropertyConverter<TValue>)GetConverter(typeof(TValue), propInfo);
             return new JsonPropertyMap<TModel, TValue>(this, propInfo, converter);
         }
-
-        public override TModel Read<TModel>(ReadOnlyBuffer<byte> data)
+        
+        public TModel Read<TModel>(Utf8String str)
+            => Read<TModel>(str.Bytes);
+        public override TModel Read<TModel>(ReadOnlySpan<byte> data)
         {
-            var reader = new JsonReader(data.Span, SymbolTable.InvariantUtf8);
+            var reader = new JsonReader(data, SymbolTable.InvariantUtf8);
             if (!reader.Read())
                 return default;
             var converter = GetConverter(typeof(TModel)) as JsonPropertyConverter<TModel>;
             return converter.Read(null, null, ref reader, false);
         }
+
         public override void Write<TModel>(ArrayFormatter stream, TModel model)
         {
             var writer = new JsonWriter(stream);

src/Discord.Net.Serialization/Serializer.cs

@@ -77,7 +77,9 @@ namespace Discord.Serialization
             => _createPropertyMapMethod.MakeGenericMethod(typeof(TModel), propInfo.PropertyType).Invoke(this, new object[] { propInfo }) as PropertyMap;
         protected abstract PropertyMap CreatePropertyMap<TModel, TValue>(PropertyInfo propInfo);
 
-        public abstract TModel Read<TModel>(ReadOnlyBuffer<byte> data);
+        public TModel Read<TModel>(ReadOnlyBuffer<byte> data)
+            => Read<TModel>(data.Span);
+        public abstract TModel Read<TModel>(ReadOnlySpan<byte> data);
         public abstract void Write<TModel>(ArrayFormatter stream, TModel model);
     }
 }

src/Discord.Net.Serialization/_corefxlab/System.Text.Json/System/Text/Json/JsonConstants.cs

@@ -25,6 +25,7 @@ namespace System.Text.Json
         public const byte ListSeperator = (byte)',';
         public const byte KeyValueSeperator = (byte)':';
         public const byte Quote = (byte)'"';
+        public const byte Backslash = (byte)'\\';
 
         #endregion Control characters
 

src/Discord.Net.Serialization/_corefxlab/System.Text.Json/System/Text/Json/JsonReader.cs

@@ -1,7 +1,10 @@
 // Copyright (c) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using System.Buffers;
+using System.Collections.Sequences;
 using System.Runtime.CompilerServices;
+using System.Text.Formatting;
 
 namespace System.Text.Json
 {
@@ -14,6 +17,7 @@ namespace System.Text.Json
         private readonly SymbolTable _symbolTable;
 
         private ReadOnlySpan<byte> _buffer;
+        private ResizableArray<byte> _working;
 
         // Depth tracks the recursive depth of the nested objects / arrays within the JSON data.
         internal int _depth;
@@ -71,6 +75,7 @@ namespace System.Text.Json
             _symbolTable = symbolTable;
             _depth = 0;
             _containerMask = 0;
+            _working = default;
 
             if (_symbolTable == SymbolTable.InvariantUtf8)
                 _encoderState = JsonEncoderState.UseFastUtf8;
@@ -660,17 +665,89 @@ namespace System.Text.Json
             // If we are in this method, the first char is already known to be a JSON quote character.
             // Skip through the bytes until we find the closing JSON quote character.
             int idx = 1;
-            while (idx < length && Unsafe.Add(ref src, idx++) != JsonConstants.Quote) ;
+            int start = idx;
+            bool hasEscapes = false;
 
-            // If we hit the end of the source and never saw an ending quote, then fail.
-            if (idx == length && Unsafe.Add(ref src, idx - 1) != JsonConstants.Quote)
-                throw new JsonReaderException();
+            while (idx < length)
+            {
+                byte c = Unsafe.Add(ref src, idx++);
+                if (c == JsonConstants.Quote)
+                    break;
+                else if (c == JsonConstants.Backslash)
+                {
+                    hasEscapes = true;
+                    break;
+                }
+            }
 
-            // Calculate the real start of the property name based on our current buffer location.
-            // Also, skip the opening JSON quote character.
-            int startIndex = (int)Unsafe.ByteOffset(ref _buffer.DangerousGetPinnableReference(), ref src) + 1;
+            if (!hasEscapes) //Fast route
+            {
+                // If we hit the end of the source and never saw an ending quote, then fail.
+                if (idx == length && Unsafe.Add(ref src, idx - 1) != JsonConstants.Quote)
+                    throw new JsonReaderException();
+
+                // Calculate the real start of the property name based on our current buffer location.
+                // Also, skip the opening JSON quote character.
+                int startIndex = (int)Unsafe.ByteOffset(ref _buffer.DangerousGetPinnableReference(), ref src) + 1;
 
-            Value = _buffer.Slice(startIndex, idx - 2); // -2 to exclude the quote characters.
+                Value = _buffer.Slice(startIndex, idx - 2); // -2 to exclude the quote characters.
+            }
+            else //Slow route
+            {
+                if (_working.Items == null)
+                    _working = new ResizableArray<byte>(ArrayPool<byte>.Shared.Rent(128));
+                _working.Clear();
+
+                int arrLength = idx - start;
+                idx = start;
+                
+                bool isEscaping = false;
+                bool success = false;
+                while (idx < length)
+                {
+                    byte c = Unsafe.Add(ref src, idx);
+                    if (isEscaping)
+                        isEscaping = false;
+                    else if (c == JsonConstants.Backslash || c == JsonConstants.Quote)
+                    {
+                        int segmentLength = idx - start;
+                        if (segmentLength != 0)
+                        {
+                            //Ensure we have enough space in the buffer
+                            int remaining = _working.Capacity - _working.Count;
+                            if (segmentLength > remaining)
+                            {
+                                int doubleSize = _working.Free.Count * 2;
+                                int minNewSize = _working.Capacity + segmentLength;
+                                int newSize = minNewSize > doubleSize ? minNewSize : doubleSize;
+                                var newArray = ArrayPool<byte>.Shared.Rent(minNewSize + _working.Count);
+                                var oldArray = _working.Resize(newArray);
+                                ArrayPool<byte>.Shared.Return(oldArray);
+                            }
+
+                            //Copy all data before the backslash
+                            var span = _working.Free.AsSpan();
+                            Unsafe.CopyBlock(ref span.DangerousGetPinnableReference(), ref Unsafe.Add(ref src, start), (uint)segmentLength);
+                            _working.Count += segmentLength;
+                        }
+                        start = idx + 1;
+                        isEscaping = true;
+
+                        if (c == JsonConstants.Quote)
+                        {
+                            idx++;
+                            success = true;
+                            break;
+                        }
+                    }
+                    idx++;
+                }
+
+                if (!success)
+                    throw new JsonReaderException();
+
+                Value = _working.Full;
+            }
             ValueType = JsonValueType.String;
             return idx;
         }