diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c
index f98a19e61a3d..3ad9b48212de 100644
--- a/drivers/infiniband/sw/rxe/rxe_qp.c
+++ b/drivers/infiniband/sw/rxe/rxe_qp.c
@@ -273,10 +273,11 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,
 	rxe_init_task(rxe, &qp->comp.task, qp,
 		      rxe_completer, "comp");
 
-	setup_timer(&qp->rnr_nak_timer, rnr_nak_timer, (unsigned long)qp);
-	setup_timer(&qp->retrans_timer, retransmit_timer, (unsigned long)qp);
 	qp->qp_timeout_jiffies = 0; /* Can't be set for UD/UC in modify_qp */
-
+	if (init->qp_type == IB_QPT_RC) {
+		setup_timer(&qp->rnr_nak_timer, rnr_nak_timer, (unsigned long)qp);
+		setup_timer(&qp->retrans_timer, retransmit_timer, (unsigned long)qp);
+	}
 	return 0;
 }
 
@@ -804,8 +805,10 @@ void rxe_qp_destroy(struct rxe_qp *qp)
 	qp->qp_timeout_jiffies = 0;
 	rxe_cleanup_task(&qp->resp.task);
 
-	del_timer_sync(&qp->retrans_timer);
-	del_timer_sync(&qp->rnr_nak_timer);
+	if (qp_type(qp) == IB_QPT_RC) {
+		del_timer_sync(&qp->retrans_timer);
+		del_timer_sync(&qp->rnr_nak_timer);
+	}
 
 	rxe_cleanup_task(&qp->req.task);
 	rxe_cleanup_task(&qp->comp.task);