mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-30 08:02:30 +00:00
IB/mlx5: Generate completions in software
The GSI QP emulation requires also emulating completions for transmitted MADs. The CQ on which these completions are generated can also be used by the hardware, and the MAD layer is free to use any CQ of the device for the GSI QP. Add a method for generating software completions to each mlx5 CQ. Software completions are polled first, and generate calls to the completion handler callback if necessary. Reviewed-by: Leon Romanovsky <leonro@mellanox.com> Signed-off-by: Haggai Eran <haggaie@mellanox.com> Signed-off-by: Doug Ledford <dledford@redhat.com>
This commit is contained in:
parent
7722f47e71
commit
25361e02c4
2 changed files with 88 additions and 5 deletions
|
@ -568,18 +568,44 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq,
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int poll_soft_wc(struct mlx5_ib_cq *cq, int num_entries,
|
||||
struct ib_wc *wc)
|
||||
{
|
||||
struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
|
||||
struct mlx5_ib_wc *soft_wc, *next;
|
||||
int npolled = 0;
|
||||
|
||||
list_for_each_entry_safe(soft_wc, next, &cq->wc_list, list) {
|
||||
if (npolled >= num_entries)
|
||||
break;
|
||||
|
||||
mlx5_ib_dbg(dev, "polled software generated completion on CQ 0x%x\n",
|
||||
cq->mcq.cqn);
|
||||
|
||||
wc[npolled++] = soft_wc->wc;
|
||||
list_del(&soft_wc->list);
|
||||
kfree(soft_wc);
|
||||
}
|
||||
|
||||
return npolled;
|
||||
}
|
||||
|
||||
int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
|
||||
{
|
||||
struct mlx5_ib_cq *cq = to_mcq(ibcq);
|
||||
struct mlx5_ib_qp *cur_qp = NULL;
|
||||
unsigned long flags;
|
||||
int soft_polled = 0;
|
||||
int npolled;
|
||||
int err = 0;
|
||||
|
||||
spin_lock_irqsave(&cq->lock, flags);
|
||||
|
||||
for (npolled = 0; npolled < num_entries; npolled++) {
|
||||
err = mlx5_poll_one(cq, &cur_qp, wc + npolled);
|
||||
if (unlikely(!list_empty(&cq->wc_list)))
|
||||
soft_polled = poll_soft_wc(cq, num_entries, wc);
|
||||
|
||||
for (npolled = 0; npolled < num_entries - soft_polled; npolled++) {
|
||||
err = mlx5_poll_one(cq, &cur_qp, wc + soft_polled + npolled);
|
||||
if (err)
|
||||
break;
|
||||
}
|
||||
|
@ -590,7 +616,7 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
|
|||
spin_unlock_irqrestore(&cq->lock, flags);
|
||||
|
||||
if (err == 0 || err == -EAGAIN)
|
||||
return npolled;
|
||||
return soft_polled + npolled;
|
||||
else
|
||||
return err;
|
||||
}
|
||||
|
@ -598,16 +624,27 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
|
|||
int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
|
||||
{
|
||||
struct mlx5_core_dev *mdev = to_mdev(ibcq->device)->mdev;
|
||||
struct mlx5_ib_cq *cq = to_mcq(ibcq);
|
||||
void __iomem *uar_page = mdev->priv.uuari.uars[0].map;
|
||||
unsigned long irq_flags;
|
||||
int ret = 0;
|
||||
|
||||
mlx5_cq_arm(&to_mcq(ibcq)->mcq,
|
||||
spin_lock_irqsave(&cq->lock, irq_flags);
|
||||
if (cq->notify_flags != IB_CQ_NEXT_COMP)
|
||||
cq->notify_flags = flags & IB_CQ_SOLICITED_MASK;
|
||||
|
||||
if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !list_empty(&cq->wc_list))
|
||||
ret = 1;
|
||||
spin_unlock_irqrestore(&cq->lock, irq_flags);
|
||||
|
||||
mlx5_cq_arm(&cq->mcq,
|
||||
(flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
|
||||
MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT,
|
||||
uar_page,
|
||||
MLX5_GET_DOORBELL_LOCK(&mdev->priv.cq_uar_lock),
|
||||
to_mcq(ibcq)->mcq.cons_index);
|
||||
|
||||
return 0;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int alloc_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf,
|
||||
|
@ -760,6 +797,14 @@ static void destroy_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq)
|
|||
mlx5_db_free(dev->mdev, &cq->db);
|
||||
}
|
||||
|
||||
static void notify_soft_wc_handler(struct work_struct *work)
|
||||
{
|
||||
struct mlx5_ib_cq *cq = container_of(work, struct mlx5_ib_cq,
|
||||
notify_work);
|
||||
|
||||
cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
|
||||
}
|
||||
|
||||
struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
|
||||
const struct ib_cq_init_attr *attr,
|
||||
struct ib_ucontext *context,
|
||||
|
@ -810,6 +855,8 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
|
|||
&index, &inlen);
|
||||
if (err)
|
||||
goto err_create;
|
||||
|
||||
INIT_WORK(&cq->notify_work, notify_soft_wc_handler);
|
||||
}
|
||||
|
||||
cq->cqe_size = cqe_size;
|
||||
|
@ -835,6 +882,8 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
|
|||
cq->mcq.comp = mlx5_ib_cq_comp;
|
||||
cq->mcq.event = mlx5_ib_cq_event;
|
||||
|
||||
INIT_LIST_HEAD(&cq->wc_list);
|
||||
|
||||
if (context)
|
||||
if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
|
||||
err = -EFAULT;
|
||||
|
@ -1222,3 +1271,27 @@ int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq)
|
|||
cq = to_mcq(ibcq);
|
||||
return cq->cqe_size;
|
||||
}
|
||||
|
||||
/* Called from atomic context */
|
||||
int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc)
|
||||
{
|
||||
struct mlx5_ib_wc *soft_wc;
|
||||
struct mlx5_ib_cq *cq = to_mcq(ibcq);
|
||||
unsigned long flags;
|
||||
|
||||
soft_wc = kmalloc(sizeof(*soft_wc), GFP_ATOMIC);
|
||||
if (!soft_wc)
|
||||
return -ENOMEM;
|
||||
|
||||
soft_wc->wc = *wc;
|
||||
spin_lock_irqsave(&cq->lock, flags);
|
||||
list_add_tail(&soft_wc->list, &cq->wc_list);
|
||||
if (cq->notify_flags == IB_CQ_NEXT_COMP ||
|
||||
wc->status != IB_WC_SUCCESS) {
|
||||
cq->notify_flags = 0;
|
||||
schedule_work(&cq->notify_work);
|
||||
}
|
||||
spin_unlock_irqrestore(&cq->lock, flags);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
@ -393,6 +393,14 @@ struct mlx5_ib_cq {
|
|||
struct ib_umem *resize_umem;
|
||||
int cqe_size;
|
||||
u32 create_flags;
|
||||
struct list_head wc_list;
|
||||
enum ib_cq_notify_flags notify_flags;
|
||||
struct work_struct notify_work;
|
||||
};
|
||||
|
||||
struct mlx5_ib_wc {
|
||||
struct ib_wc wc;
|
||||
struct list_head list;
|
||||
};
|
||||
|
||||
struct mlx5_ib_srq {
|
||||
|
@ -785,6 +793,8 @@ int mlx5_ib_gsi_post_recv(struct ib_qp *qp, struct ib_recv_wr *wr,
|
|||
struct ib_recv_wr **bad_wr);
|
||||
void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi);
|
||||
|
||||
int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc);
|
||||
|
||||
static inline void init_query_mad(struct ib_smp *mad)
|
||||
{
|
||||
mad->base_version = 1;
|
||||
|
|
Loading…
Reference in a new issue