diff --git a/src/Discord.Net/Models/Channel.cs b/src/Discord.Net/Models/Channel.cs
index 610100431..fc27cea51 100644
--- a/src/Discord.Net/Models/Channel.cs
+++ b/src/Discord.Net/Models/Channel.cs
@@ -168,10 +168,26 @@ namespace Discord
 		}
 		private void UpdateMembersCache()
 		{
-			if (_server.Id != null)
-				_members = Server.Members.Where(x => x.GetPermissions(this)?.ReadMessages ?? false).ToDictionary(x => x.Id, x => x);
-			else
-				_members = new Dictionary<string, User>();
+			if (IsPrivate)
+			{
+				_members = new Dictionary<string, User>()
+				{
+					{ _client.CurrentUserId, _client.CurrentUser },
+					{ _recipient.Id, _recipient.Value }
+				};
+			}
+			else if (Type == ChannelType.Text)
+			{
+				_members = Server.Members
+					.Where(x => x.GetPermissions(this)?.ReadMessages ?? false)
+					.ToDictionary(x => x.Id, x => x);
+			}
+			else if (Type == ChannelType.Voice)
+			{
+				_members = Server.Members
+					.Where(x => x.VoiceChannel == this)
+					.ToDictionary(x => x.Id, x => x);
+			}
 			_areMembersStale = false;
 		}
 
@@ -181,12 +197,12 @@ namespace Discord
 			foreach (var member in _members)
 				member.Value.UpdateChannelPermissions(this);
 		}
-		internal void InvalidatePermissionsCache(Role role)
+		/*internal void InvalidatePermissionsCache(Role role)
 		{
 			_areMembersStale = true;
 			foreach (var member in role.Members)
 				member.UpdateChannelPermissions(this);
-		}
+		}*/
 		internal void InvalidatePermissionsCache(User user)
 		{
 			_areMembersStale = true;
diff --git a/src/Discord.Net/Models/Server.cs b/src/Discord.Net/Models/Server.cs
index 602f01db0..c38327c20 100644
--- a/src/Discord.Net/Models/Server.cs
+++ b/src/Discord.Net/Models/Server.cs
@@ -214,7 +214,7 @@ namespace Discord
 				if (user.Id == _ownerId)
 					Owner = user;
 
-				foreach (var channel in Channels)
+				foreach (var channel in TextChannels)
 				{
 					user.AddChannel(channel);
 					channel.InvalidatePermissionsCache(user);
diff --git a/src/Discord.Net/Models/User.cs b/src/Discord.Net/Models/User.cs
index 1c28c5531..4f53cdc07 100644
--- a/src/Discord.Net/Models/User.cs
+++ b/src/Discord.Net/Models/User.cs
@@ -221,7 +221,18 @@ namespace Discord
 			if (model.IsServerSuppressed != null)
 				IsServerSuppressed = model.IsServerSuppressed.Value;
 
-			_voiceChannel.Id = model.ChannelId; //Can be null
+			if (_voiceChannel.Id != model.ChannelId)
+			{
+				var oldChannel = _voiceChannel.Value;
+				if (oldChannel != null)
+					oldChannel.InvalidateMembersCache();
+
+				_voiceChannel.Id = model.ChannelId; //Can be null
+
+				var newChannel = _voiceChannel.Value;
+				if (newChannel != null)
+					newChannel.InvalidateMembersCache();
+			}
         }
 		private void UpdateRoles(IEnumerable<Role> roles)
 		{