diff --git a/src/Discord.Net/DiscordClient.cs b/src/Discord.Net/DiscordClient.cs
index 50a5604a9..15ae3b287 100644
--- a/src/Discord.Net/DiscordClient.cs
+++ b/src/Discord.Net/DiscordClient.cs
@@ -409,7 +409,11 @@ namespace Discord
         private Server RemoveServer(ulong id)
         {
             Server server;
-            _servers.TryRemove(id, out server);
+            if (_servers.TryRemove(id, out server))
+            {
+                foreach (var channel in server.AllChannels)
+                    RemoveChannel(channel.Id);
+            }
             return server;
         }