diff --git a/src/Discord.Net.WebSocket/Audio/AudioClient.cs b/src/Discord.Net.WebSocket/Audio/AudioClient.cs
index 0d9f82ab4..911df97ef 100644
--- a/src/Discord.Net.WebSocket/Audio/AudioClient.cs
+++ b/src/Discord.Net.WebSocket/Audio/AudioClient.cs
@@ -103,7 +103,7 @@ namespace Discord.Audio
             {
                 StopAsync().GetAwaiter().GetResult();
                 ApiClient.Dispose();
-                if (_secretKeyHandle.IsAllocated)
+                if (_secretKeyHandle.Target != null)
                     _secretKeyHandle.Free();
                 _isDisposed = true;
             }
@@ -260,9 +260,9 @@ namespace Discord.Audio
                             if (data.Mode != DiscordVoiceAPIClient.Mode)
                                 throw new InvalidOperationException($"Discord selected an unexpected mode: {data.Mode}");
 
-                            _secretKey = data.SecretKey;
-                            if (_secretKeyHandle != null)
+                            if (_secretKeyHandle.Target != null)
                                 _secretKeyHandle.Free();
+                            _secretKey = data.SecretKey;
                             _secretKeyHandle = GCHandle.Alloc(data.SecretKey, GCHandleType.Pinned);
                             SecretKeyPtr = _secretKeyHandle.AddrOfPinnedObject();