diff --git a/src/Discord.Net.Core/API/DiscordRestApiClient.cs b/src/Discord.Net.Core/API/DiscordRestApiClient.cs index 7268d7b34..8535fcf33 100644 --- a/src/Discord.Net.Core/API/DiscordRestApiClient.cs +++ b/src/Discord.Net.Core/API/DiscordRestApiClient.cs @@ -165,30 +165,30 @@ namespace Discord.API //Core internal Task SendAsync(string method, Expression> endpointExpr, BucketIds ids, - string clientBucketId = null, RequestOptions options = null, [CallerMemberName] string funcName = null) - => SendAsync(method, GetEndpoint(endpointExpr), GetBucketId(ids, endpointExpr, AuthTokenType, funcName), clientBucketId, options); + ClientBucketType clientBucket = ClientBucketType.Unbucketed, RequestOptions options = null, [CallerMemberName] string funcName = null) + => SendAsync(method, GetEndpoint(endpointExpr), GetBucketId(ids, endpointExpr, AuthTokenType, funcName), clientBucket, options); public async Task SendAsync(string method, string endpoint, - string bucketId = null, string clientBucketId = null, RequestOptions options = null) + string bucketId = null, ClientBucketType clientBucket = ClientBucketType.Unbucketed, RequestOptions options = null) { options = options ?? new RequestOptions(); options.HeaderOnly = true; - options.BucketId = bucketId; - options.ClientBucketId = AuthTokenType == TokenType.User ? clientBucketId : null; + options.BucketId = AuthTokenType == TokenType.User ? ClientBucket.Get(clientBucket).Id : bucketId; + options.IsClientBucket = AuthTokenType == TokenType.User; var request = new RestRequest(_restClient, method, endpoint, options); await SendInternalAsync(method, endpoint, request).ConfigureAwait(false); } internal Task SendJsonAsync(string method, Expression> endpointExpr, object payload, BucketIds ids, - string clientBucketId = null, RequestOptions options = null, [CallerMemberName] string funcName = null) - => SendJsonAsync(method, GetEndpoint(endpointExpr), payload, GetBucketId(ids, endpointExpr, AuthTokenType, funcName), clientBucketId, options); + ClientBucketType clientBucket = ClientBucketType.Unbucketed, RequestOptions options = null, [CallerMemberName] string funcName = null) + => SendJsonAsync(method, GetEndpoint(endpointExpr), payload, GetBucketId(ids, endpointExpr, AuthTokenType, funcName), clientBucket, options); public async Task SendJsonAsync(string method, string endpoint, object payload, - string bucketId = null, string clientBucketId = null, RequestOptions options = null) + string bucketId = null, ClientBucketType clientBucket = ClientBucketType.Unbucketed, RequestOptions options = null) { options = options ?? new RequestOptions(); options.HeaderOnly = true; - options.BucketId = bucketId; - options.ClientBucketId = AuthTokenType == TokenType.User ? clientBucketId : null; + options.BucketId = AuthTokenType == TokenType.User ? ClientBucket.Get(clientBucket).Id : bucketId; + options.IsClientBucket = AuthTokenType == TokenType.User; var json = payload != null ? SerializeJson(payload) : null; var request = new JsonRestRequest(_restClient, method, endpoint, json, options); @@ -196,43 +196,43 @@ namespace Discord.API } internal Task SendMultipartAsync(string method, Expression> endpointExpr, IReadOnlyDictionary multipartArgs, BucketIds ids, - string clientBucketId = null, RequestOptions options = null, [CallerMemberName] string funcName = null) - => SendMultipartAsync(method, GetEndpoint(endpointExpr), multipartArgs, GetBucketId(ids, endpointExpr, AuthTokenType, funcName), clientBucketId, options); + ClientBucketType clientBucket = ClientBucketType.Unbucketed, RequestOptions options = null, [CallerMemberName] string funcName = null) + => SendMultipartAsync(method, GetEndpoint(endpointExpr), multipartArgs, GetBucketId(ids, endpointExpr, AuthTokenType, funcName), clientBucket, options); public async Task SendMultipartAsync(string method, string endpoint, IReadOnlyDictionary multipartArgs, - string bucketId = null, string clientBucketId = null, RequestOptions options = null) + string bucketId = null, ClientBucketType clientBucket = ClientBucketType.Unbucketed, RequestOptions options = null) { options = options ?? new RequestOptions(); options.HeaderOnly = true; - options.BucketId = bucketId; - options.ClientBucketId = AuthTokenType == TokenType.User ? clientBucketId : null; + options.BucketId = AuthTokenType == TokenType.User ? ClientBucket.Get(clientBucket).Id : bucketId; + options.IsClientBucket = AuthTokenType == TokenType.User; var request = new MultipartRestRequest(_restClient, method, endpoint, multipartArgs, options); await SendInternalAsync(method, endpoint, request).ConfigureAwait(false); } internal Task SendAsync(string method, Expression> endpointExpr, BucketIds ids, - string clientBucketId = null, RequestOptions options = null, [CallerMemberName] string funcName = null) where TResponse : class - => SendAsync(method, GetEndpoint(endpointExpr), GetBucketId(ids, endpointExpr, AuthTokenType, funcName), clientBucketId, options); + ClientBucketType clientBucket = ClientBucketType.Unbucketed, RequestOptions options = null, [CallerMemberName] string funcName = null) where TResponse : class + => SendAsync(method, GetEndpoint(endpointExpr), GetBucketId(ids, endpointExpr, AuthTokenType, funcName), clientBucket, options); public async Task SendAsync(string method, string endpoint, - string bucketId = null, string clientBucketId = null, RequestOptions options = null) where TResponse : class + string bucketId = null, ClientBucketType clientBucket = ClientBucketType.Unbucketed, RequestOptions options = null) where TResponse : class { options = options ?? new RequestOptions(); - options.BucketId = bucketId; - options.ClientBucketId = AuthTokenType == TokenType.User ? clientBucketId : null; + options.BucketId = AuthTokenType == TokenType.User ? ClientBucket.Get(clientBucket).Id : bucketId; + options.IsClientBucket = AuthTokenType == TokenType.User; var request = new RestRequest(_restClient, method, endpoint, options); return DeserializeJson(await SendInternalAsync(method, endpoint, request).ConfigureAwait(false)); } internal Task SendJsonAsync(string method, Expression> endpointExpr, object payload, BucketIds ids, - string clientBucketId = null, RequestOptions options = null, [CallerMemberName] string funcName = null) where TResponse : class - => SendJsonAsync(method, GetEndpoint(endpointExpr), payload, GetBucketId(ids, endpointExpr, AuthTokenType, funcName), clientBucketId, options); + ClientBucketType clientBucket = ClientBucketType.Unbucketed, RequestOptions options = null, [CallerMemberName] string funcName = null) where TResponse : class + => SendJsonAsync(method, GetEndpoint(endpointExpr), payload, GetBucketId(ids, endpointExpr, AuthTokenType, funcName), clientBucket, options); public async Task SendJsonAsync(string method, string endpoint, object payload, - string bucketId = null, string clientBucketId = null, RequestOptions options = null) where TResponse : class + string bucketId = null, ClientBucketType clientBucket = ClientBucketType.Unbucketed, RequestOptions options = null) where TResponse : class { options = options ?? new RequestOptions(); - options.BucketId = bucketId; - options.ClientBucketId = AuthTokenType == TokenType.User ? clientBucketId : null; + options.BucketId = AuthTokenType == TokenType.User ? ClientBucket.Get(clientBucket).Id : bucketId; + options.IsClientBucket = AuthTokenType == TokenType.User; var json = payload != null ? SerializeJson(payload) : null; var request = new JsonRestRequest(_restClient, method, endpoint, json, options); @@ -240,14 +240,14 @@ namespace Discord.API } internal Task SendMultipartAsync(string method, Expression> endpointExpr, IReadOnlyDictionary multipartArgs, BucketIds ids, - string clientBucketId = null, RequestOptions options = null, [CallerMemberName] string funcName = null) - => SendMultipartAsync(method, GetEndpoint(endpointExpr), multipartArgs, GetBucketId(ids, endpointExpr, AuthTokenType, funcName), clientBucketId, options); + ClientBucketType clientBucket = ClientBucketType.Unbucketed, RequestOptions options = null, [CallerMemberName] string funcName = null) + => SendMultipartAsync(method, GetEndpoint(endpointExpr), multipartArgs, GetBucketId(ids, endpointExpr, AuthTokenType, funcName), clientBucket, options); public async Task SendMultipartAsync(string method, string endpoint, IReadOnlyDictionary multipartArgs, - string bucketId = null, string clientBucketId = null, RequestOptions options = null) + string bucketId = null, ClientBucketType clientBucket = ClientBucketType.Unbucketed, RequestOptions options = null) { options = options ?? new RequestOptions(); - options.BucketId = bucketId; - options.ClientBucketId = AuthTokenType == TokenType.User ? clientBucketId : null; + options.BucketId = AuthTokenType == TokenType.User ? ClientBucket.Get(clientBucket).Id : bucketId; + options.IsClientBucket = AuthTokenType == TokenType.User; var request = new MultipartRestRequest(_restClient, method, endpoint, multipartArgs, options); return DeserializeJson(await SendInternalAsync(method, endpoint, request).ConfigureAwait(false)); @@ -447,7 +447,7 @@ namespace Discord.API options = RequestOptions.CreateOrClone(options); var ids = new BucketIds(channelId: channelId); - return await SendJsonAsync("POST", () => $"channels/{channelId}/messages", args, ids, clientBucketId: ClientBucket.SendEditId, options: options).ConfigureAwait(false); + return await SendJsonAsync("POST", () => $"channels/{channelId}/messages", args, ids, clientBucket: ClientBucketType.SendEdit, options: options).ConfigureAwait(false); } public async Task UploadFileAsync(ulong channelId, UploadFileParams args, RequestOptions options = null) { @@ -466,7 +466,7 @@ namespace Discord.API } var ids = new BucketIds(channelId: channelId); - return await SendMultipartAsync("POST", () => $"channels/{channelId}/messages", args.ToDictionary(), ids, clientBucketId: ClientBucket.SendEditId, options: options).ConfigureAwait(false); + return await SendMultipartAsync("POST", () => $"channels/{channelId}/messages", args.ToDictionary(), ids, clientBucket: ClientBucketType.SendEdit, options: options).ConfigureAwait(false); } public async Task DeleteMessageAsync(ulong channelId, ulong messageId, RequestOptions options = null) { @@ -512,7 +512,7 @@ namespace Discord.API options = RequestOptions.CreateOrClone(options); var ids = new BucketIds(channelId: channelId); - return await SendJsonAsync("PATCH", () => $"channels/{channelId}/messages/{messageId}", args, ids, clientBucketId: ClientBucket.SendEditId, options: options).ConfigureAwait(false); + return await SendJsonAsync("PATCH", () => $"channels/{channelId}/messages/{messageId}", args, ids, clientBucket: ClientBucketType.SendEdit, options: options).ConfigureAwait(false); } public async Task AckMessageAsync(ulong channelId, ulong messageId, RequestOptions options = null) { @@ -1061,19 +1061,6 @@ namespace Discord.API using (JsonReader reader = new JsonTextReader(text)) return _serializer.Deserialize(reader); } - internal string GetBucketId(ulong guildId = 0, ulong channelId = 0, [CallerMemberName] string methodName = "") - { - if (guildId != 0) - { - if (channelId != 0) - return $"{methodName}({guildId}/{channelId})"; - else - return $"{methodName}({guildId})"; - } - else if (channelId != 0) - return $"{methodName}({channelId})"; - return $"{methodName}()"; - } internal class BucketIds { diff --git a/src/Discord.Net.Core/DiscordConfig.cs b/src/Discord.Net.Core/DiscordConfig.cs index 737cf0050..0cb190726 100644 --- a/src/Discord.Net.Core/DiscordConfig.cs +++ b/src/Discord.Net.Core/DiscordConfig.cs @@ -14,6 +14,7 @@ namespace Discord public const string CDNUrl = "https://discordcdn.com/"; public const string InviteUrl = "https://discord.gg/"; + public const int DefaultRequestTimeout = 15000; public const int MaxMessageSize = 2000; public const int MaxMessagesPerBatch = 100; public const int MaxUsersPerBatch = 1000; diff --git a/src/Discord.Net.Core/Net/Queue/ClientBucket.cs b/src/Discord.Net.Core/Net/Queue/ClientBucket.cs index 14d3c3207..f32df1bcf 100644 --- a/src/Discord.Net.Core/Net/Queue/ClientBucket.cs +++ b/src/Discord.Net.Core/Net/Queue/ClientBucket.cs @@ -2,25 +2,47 @@ namespace Discord.Net.Queue { - public struct ClientBucket + public enum ClientBucketType { - public const string SendEditId = ""; + Unbucketed = 0, + SendEdit = 1 + } + internal struct ClientBucket + { + private static readonly ImmutableDictionary _defsByType; + private static readonly ImmutableDictionary _defsById; - private static readonly ImmutableDictionary _defs; static ClientBucket() { - var builder = ImmutableDictionary.CreateBuilder(); - builder.Add(SendEditId, new ClientBucket(10, 10)); - _defs = builder.ToImmutable(); - } + var buckets = new[] + { + new ClientBucket(ClientBucketType.Unbucketed, "", 10, 10), + new ClientBucket(ClientBucketType.SendEdit, "", 10, 10) + }; - public static ClientBucket Get(string id) =>_defs[id]; + var builder = ImmutableDictionary.CreateBuilder(); + foreach (var bucket in buckets) + builder.Add(bucket.Type, bucket); + _defsByType = builder.ToImmutable(); + + var builder2 = ImmutableDictionary.CreateBuilder(); + foreach (var bucket in buckets) + builder2.Add(bucket.Id, bucket); + _defsById = builder2.ToImmutable(); + } + public static ClientBucket Get(ClientBucketType type) => _defsByType[type]; + public static ClientBucket Get(string id) => _defsById[id]; + + public ClientBucketType Type { get; } + public string Id { get; } public int WindowCount { get; } public int WindowSeconds { get; } - public ClientBucket(int count, int seconds) + public ClientBucket(ClientBucketType type, string id, int count, int seconds) { + Type = type; + Id = id; WindowCount = count; WindowSeconds = seconds; } diff --git a/src/Discord.Net.Core/Net/Queue/RequestQueue.cs b/src/Discord.Net.Core/Net/Queue/RequestQueue.cs index ab20d7c18..1ea586481 100644 --- a/src/Discord.Net.Core/Net/Queue/RequestQueue.cs +++ b/src/Discord.Net.Core/Net/Queue/RequestQueue.cs @@ -79,7 +79,9 @@ namespace Discord.Net.Queue int millis = (int)Math.Ceiling((_waitUntil - DateTimeOffset.UtcNow).TotalMilliseconds); if (millis > 0) { +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] Sleeping {millis} ms (Pre-emptive) [Global]"); +#endif await Task.Delay(millis).ConfigureAwait(false); } } diff --git a/src/Discord.Net.Core/Net/Queue/RequestQueueBucket.cs b/src/Discord.Net.Core/Net/Queue/RequestQueueBucket.cs index b750afbf6..19976e998 100644 --- a/src/Discord.Net.Core/Net/Queue/RequestQueueBucket.cs +++ b/src/Discord.Net.Core/Net/Queue/RequestQueueBucket.cs @@ -1,7 +1,9 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using System; +#if DEBUG_LIMITS using System.Diagnostics; +#endif using System.IO; using System.Net; using System.Threading; @@ -27,8 +29,8 @@ namespace Discord.Net.Queue _lock = new object(); - if (request.Options.ClientBucketId != null) - WindowCount = ClientBucket.Get(request.Options.ClientBucketId).WindowCount; + if (request.Options.IsClientBucket) + WindowCount = ClientBucket.Get(request.Options.BucketId).WindowCount; else WindowCount = 1; //Only allow one request until we get a header back _semaphore = WindowCount; @@ -40,14 +42,18 @@ namespace Discord.Net.Queue public async Task SendAsync(RestRequest request) { int id = Interlocked.Increment(ref nextId); +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] Start"); +#endif LastAttemptAt = DateTimeOffset.UtcNow; while (true) { await _queue.EnterGlobalAsync(id, request).ConfigureAwait(false); await EnterAsync(id, request).ConfigureAwait(false); +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] Sending..."); +#endif var response = await request.SendAsync().ConfigureAwait(false); TimeSpan lag = DateTimeOffset.UtcNow - DateTimeOffset.Parse(response.Headers["Date"]); var info = new RateLimitInfo(response.Headers); @@ -59,18 +65,24 @@ namespace Discord.Net.Queue case (HttpStatusCode)429: if (info.IsGlobal) { +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] (!) 429 [Global]"); +#endif _queue.PauseGlobal(info, lag); } else { +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] (!) 429"); +#endif UpdateRateLimit(id, request, info, lag, true); } await _queue.RaiseRateLimitTriggered(Id, info).ConfigureAwait(false); continue; //Retry case HttpStatusCode.BadGateway: //502 +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] (!) 502"); +#endif continue; //Continue default: string reason = null; @@ -92,9 +104,13 @@ namespace Discord.Net.Queue } else { +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] Success"); +#endif UpdateRateLimit(id, request, info, lag, false); +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] Stop"); +#endif return response.Stream; } } @@ -135,7 +151,9 @@ namespace Discord.Net.Queue if (resetAt > timeoutAt) throw new RateLimitedException(); int millis = (int)Math.Ceiling((resetAt.Value - DateTimeOffset.UtcNow).TotalMilliseconds); +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] Sleeping {millis} ms (Pre-emptive)"); +#endif if (millis > 0) await Task.Delay(millis, request.CancelToken).ConfigureAwait(false); } @@ -143,13 +161,17 @@ namespace Discord.Net.Queue { if ((timeoutAt.Value - DateTimeOffset.UtcNow).TotalMilliseconds < 500.0) throw new RateLimitedException(); +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] Sleeping 500* ms (Pre-emptive)"); +#endif await Task.Delay(500, request.CancelToken).ConfigureAwait(false); } continue; } +#if DEBUG_LIMITS else Debug.WriteLine($"[{id}] Entered Semaphore ({_semaphore}/{WindowCount} remaining)"); +#endif break; } } @@ -166,7 +188,9 @@ namespace Discord.Net.Queue { WindowCount = info.Limit.Value; _semaphore = info.Remaining.Value; +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] Upgraded Semaphore to {info.Remaining.Value}/{WindowCount}"); +#endif } var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); @@ -182,24 +206,32 @@ namespace Discord.Net.Queue { //RetryAfter is more accurate than Reset, where available resetTick = DateTimeOffset.UtcNow.AddMilliseconds(info.RetryAfter.Value); +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] Retry-After: {info.RetryAfter.Value} ({info.RetryAfter.Value} ms)"); +#endif } else if (info.Reset.HasValue) { resetTick = info.Reset.Value.AddSeconds(/*1.0 +*/ lag.TotalSeconds); int diff = (int)(resetTick.Value - DateTimeOffset.UtcNow).TotalMilliseconds; +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] X-RateLimit-Reset: {info.Reset.Value.ToUnixTimeSeconds()} ({diff} ms, {lag.TotalMilliseconds} ms lag)"); +#endif } - else if (request.Options.ClientBucketId != null) + else if (request.Options.IsClientBucket && request.Options.BucketId != null) { - resetTick = DateTimeOffset.UtcNow.AddSeconds(ClientBucket.Get(request.Options.ClientBucketId).WindowSeconds); - Debug.WriteLine($"[{id}] Client Bucket ({ClientBucket.Get(request.Options.ClientBucketId).WindowSeconds * 1000} ms)"); + resetTick = DateTimeOffset.UtcNow.AddSeconds(ClientBucket.Get(request.Options.BucketId).WindowSeconds); +#if DEBUG_LIMITS + Debug.WriteLine($"[{id}] Client Bucket ({ClientBucket.Get(request.Options.BucketId).WindowSeconds * 1000} ms)"); +#endif } if (resetTick == null) { WindowCount = 0; //No rate limit info, disable limits on this bucket (should only ever happen with a user token) +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] Disabled Semaphore"); +#endif return; } @@ -207,7 +239,9 @@ namespace Discord.Net.Queue { _resetTick = resetTick; LastAttemptAt = resetTick.Value; //Make sure we dont destroy this until after its been reset +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] Reset in {(int)Math.Ceiling((resetTick - DateTimeOffset.UtcNow).Value.TotalMilliseconds)} ms"); +#endif if (!hasQueuedReset) { @@ -227,7 +261,9 @@ namespace Discord.Net.Queue millis = (int)Math.Ceiling((_resetTick.Value - DateTimeOffset.UtcNow).TotalMilliseconds); if (millis <= 0) //Make sure we havent gotten a more accurate reset time { +#if DEBUG_LIMITS Debug.WriteLine($"[{id}] * Reset *"); +#endif _semaphore = WindowCount; _resetTick = null; return; @@ -236,4 +272,4 @@ namespace Discord.Net.Queue } } } -} \ No newline at end of file +} diff --git a/src/Discord.Net.Core/Net/Rest/DefaultRestClient.cs b/src/Discord.Net.Core/Net/Rest/DefaultRestClient.cs index 5ec30c750..588785230 100644 --- a/src/Discord.Net.Core/Net/Rest/DefaultRestClient.cs +++ b/src/Discord.Net.Core/Net/Rest/DefaultRestClient.cs @@ -120,7 +120,7 @@ namespace Discord.Net.Rest cancelToken = CancellationTokenSource.CreateLinkedTokenSource(_cancelToken, cancelToken).Token; HttpResponseMessage response = await _client.SendAsync(request, cancelToken).ConfigureAwait(false); - var headers = response.Headers.ToDictionary(x => x.Key, x => x.Value.FirstOrDefault()); + var headers = response.Headers.ToDictionary(x => x.Key, x => x.Value.FirstOrDefault(), StringComparer.OrdinalIgnoreCase); var stream = !headerOnly ? await response.Content.ReadAsStreamAsync().ConfigureAwait(false) : null; return new RestResponse(response.StatusCode, headers, stream); diff --git a/src/Discord.Net.Core/RequestOptions.cs b/src/Discord.Net.Core/RequestOptions.cs index 3af6c929d..b82ec29c8 100644 --- a/src/Discord.Net.Core/RequestOptions.cs +++ b/src/Discord.Net.Core/RequestOptions.cs @@ -10,7 +10,7 @@ internal bool IgnoreState { get; set; } internal string BucketId { get; set; } - internal string ClientBucketId { get; set; } + internal bool IsClientBucket { get; set; } internal static RequestOptions CreateOrClone(RequestOptions options) { @@ -22,7 +22,7 @@ public RequestOptions() { - Timeout = 30000; + Timeout = DiscordConfig.DefaultRequestTimeout; } public RequestOptions Clone() => MemberwiseClone() as RequestOptions; diff --git a/src/Discord.Net.Rest/DiscordRestConfig.cs b/src/Discord.Net.Rest/DiscordRestConfig.cs index 8dee72231..33a3cb4e8 100644 --- a/src/Discord.Net.Rest/DiscordRestConfig.cs +++ b/src/Discord.Net.Rest/DiscordRestConfig.cs @@ -6,7 +6,6 @@ namespace Discord.Rest { public static string UserAgent { get; } = $"DiscordBot (https://github.com/RogueException/Discord.Net, v{Version})"; - internal const int RestTimeout = 10000; internal const int MessageQueueInterval = 100; internal const int WebSocketQueueInterval = 100;