diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index fe8c287cbaa1..ae4784734547 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -1645,10 +1645,10 @@ int tls_sw_recvmsg(struct sock *sk,
 
 	do {
 		bool retain_skb = false;
-		bool async = false;
 		bool zc = false;
 		int to_decrypt;
 		int chunk = 0;
+		bool async;
 
 		skb = tls_wait_data(sk, psock, flags, timeo, &err);
 		if (!skb) {
@@ -1674,18 +1674,21 @@ int tls_sw_recvmsg(struct sock *sk,
 		    tls_ctx->crypto_recv.info.version != TLS_1_3_VERSION)
 			zc = true;
 
+		/* Do not use async mode if record is non-data */
+		if (ctx->control == TLS_RECORD_TYPE_DATA)
+			async = ctx->async_capable;
+		else
+			async = false;
+
 		err = decrypt_skb_update(sk, skb, &msg->msg_iter,
-					 &chunk, &zc, ctx->async_capable);
+					 &chunk, &zc, async);
 		if (err < 0 && err != -EINPROGRESS) {
 			tls_err_abort(sk, EBADMSG);
 			goto recv_end;
 		}
 
-		if (err == -EINPROGRESS) {
-			async = true;
+		if (err == -EINPROGRESS)
 			num_async++;
-			goto pick_next_record;
-		}
 
 		if (!cmsg) {
 			int cerr;
@@ -1704,6 +1707,9 @@ int tls_sw_recvmsg(struct sock *sk,
 			goto recv_end;
 		}
 
+		if (async)
+			goto pick_next_record;
+
 		if (!zc) {
 			if (rxm->full_len > len) {
 				retain_skb = true;