diff --git a/src/Discord.Net.WebSocket/Audio/Streams/SodiumDecryptStream.cs b/src/Discord.Net.WebSocket/Audio/Streams/SodiumDecryptStream.cs
index 2b1a97f04..99b7679f9 100644
--- a/src/Discord.Net.WebSocket/Audio/Streams/SodiumDecryptStream.cs
+++ b/src/Discord.Net.WebSocket/Audio/Streams/SodiumDecryptStream.cs
@@ -12,28 +12,36 @@ namespace Discord.Audio.Streams
         private readonly AudioClient _client;
         private readonly AudioStream _next;
         private readonly byte[] _nonce;
+        private readonly byte[] _buffer;
 
         public override bool CanRead => true;
         public override bool CanSeek => false;
         public override bool CanWrite => true;
 
-        public SodiumDecryptStream(AudioStream next, IAudioClient client)
+        public SodiumDecryptStream(AudioStream next, IAudioClient client, int bufferSize = 4000)
         {
             _next = next;
             _client = (AudioClient)client;
             _nonce = new byte[24];
+            _buffer = new byte[bufferSize];
         }
 
+        /// <exception cref="InvalidOperationException">Received payload without an RTP header.</exception>
+        /// <exception cref="OperationCanceledException">The token has had cancellation requested.</exception>
+        /// <exception cref="ObjectDisposedException">The associated <see cref="T:System.Threading.CancellationTokenSource" /> has been disposed.</exception>
         public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancelToken)
         {
             cancelToken.ThrowIfCancellationRequested();
+            if (count < 12)
+                throw new InvalidOperationException("Received payload without an RTP header.");
 
             if (_client.SecretKey == null)
                 return;
 
-            Buffer.BlockCopy(buffer, 0, _nonce, 0, 12); //Copy RTP header to nonce
-            count = SecretBox.Decrypt(buffer, offset + 12, count - 12, buffer, offset + 12, _nonce, _client.SecretKey);
-            await _next.WriteAsync(buffer, 0, count + 12, cancelToken).ConfigureAwait(false);
+            Buffer.BlockCopy(buffer, offset, _nonce, 0, 12); //Copy RTP header to nonce
+            Buffer.BlockCopy(buffer, offset, _buffer, 0, 12);
+            count = SecretBox.Decrypt(buffer, offset + 12, count - 12, _buffer, 12, _nonce, _client.SecretKey);
+            await _next.WriteAsync(_buffer, 0, count + 12, cancelToken).ConfigureAwait(false);
         }
 
         public override async Task FlushAsync(CancellationToken cancelToken)
diff --git a/src/Discord.Net.WebSocket/Audio/Streams/SodiumEncryptStream.cs b/src/Discord.Net.WebSocket/Audio/Streams/SodiumEncryptStream.cs
index 8b3f0e302..d3e07c0c2 100644
--- a/src/Discord.Net.WebSocket/Audio/Streams/SodiumEncryptStream.cs
+++ b/src/Discord.Net.WebSocket/Audio/Streams/SodiumEncryptStream.cs
@@ -12,44 +12,32 @@ namespace Discord.Audio.Streams
         private readonly AudioClient _client;
         private readonly AudioStream _next;
         private readonly byte[] _nonce;
-        private bool _hasHeader;
-        private ushort _nextSeq;
-        private uint _nextTimestamp;
+        private readonly byte[] _buffer;
 
-        public SodiumEncryptStream(AudioStream next, IAudioClient client)
+        public SodiumEncryptStream(AudioStream next, IAudioClient client, int bufferSize = 4000)
         {
             _next = next;
             _client = (AudioClient)client;
             _nonce = new byte[24];
+            _buffer = new byte[bufferSize];
         }
 
-        /// <exception cref="InvalidOperationException">Header received with no payload.</exception>
-        public override void WriteHeader(ushort seq, uint timestamp, bool missed)
-        {
-            if (_hasHeader)
-                throw new InvalidOperationException("Header received with no payload.");
-
-            _nextSeq = seq;
-            _nextTimestamp = timestamp;
-            _hasHeader = true;
-        }
         /// <exception cref="InvalidOperationException">Received payload without an RTP header.</exception>
         /// <exception cref="OperationCanceledException">The token has had cancellation requested.</exception>
         /// <exception cref="ObjectDisposedException">The associated <see cref="T:System.Threading.CancellationTokenSource" /> has been disposed.</exception>
         public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancelToken)
         {
             cancelToken.ThrowIfCancellationRequested();
-            if (!_hasHeader)
+            if (count < 12)
                 throw new InvalidOperationException("Received payload without an RTP header.");
-            _hasHeader = false;
 
             if (_client.SecretKey == null)
                 return;
-                
+
             Buffer.BlockCopy(buffer, offset, _nonce, 0, 12); //Copy nonce from RTP header
-            count = SecretBox.Encrypt(buffer, offset + 12, count - 12, buffer, 12, _nonce, _client.SecretKey);
-            _next.WriteHeader(_nextSeq, _nextTimestamp, false);
-            await _next.WriteAsync(buffer, 0, count + 12, cancelToken).ConfigureAwait(false);
+            Buffer.BlockCopy(buffer, offset, _buffer, 0, 12);
+            count = SecretBox.Encrypt(buffer, offset + 12, count - 12, _buffer, 12, _nonce, _client.SecretKey);
+            await _next.WriteAsync(_buffer, 0, count + 12, cancelToken).ConfigureAwait(false);
         }
 
         public override async Task FlushAsync(CancellationToken cancelToken)