From 30e07416cf48801f127019c1dfece8039f1da8e2 Mon Sep 17 00:00:00 2001 From: Don Hiatt Date: Fri, 4 Aug 2017 13:54:04 -0700 Subject: [PATCH] IB/hfi1: Add support to send 16B bypass packets We introduce struct hfi1_opa_header as a union of ib (9B) and 16B headers. Reviewed-by: Dennis Dalessandro Signed-off-by: Dasaratharaman Chandramouli Signed-off-by: Don Hiatt Signed-off-by: Dennis Dalessandro Signed-off-by: Doug Ledford --- drivers/infiniband/hw/hfi1/rc.c | 33 +++++++++++++---------- drivers/infiniband/hw/hfi1/ruc.c | 17 ++++++------ drivers/infiniband/hw/hfi1/trace_ibhdrs.h | 19 ++++++------- drivers/infiniband/hw/hfi1/uc.c | 4 +-- drivers/infiniband/hw/hfi1/ud.c | 25 ++++++++--------- drivers/infiniband/hw/hfi1/verbs.c | 24 ++++++++++++++--- drivers/infiniband/hw/hfi1/verbs.h | 22 +++++++-------- 7 files changed, 84 insertions(+), 60 deletions(-) diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c index cf74a56e20e5..e3dbf6d45afe 100644 --- a/drivers/infiniband/hw/hfi1/rc.c +++ b/drivers/infiniband/hw/hfi1/rc.c @@ -273,9 +273,9 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) if (IS_ERR(ps->s_txreq)) goto bail_no_tx; - ohdr = &ps->s_txreq->phdr.hdr.u.oth; + ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth; if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) - ohdr = &ps->s_txreq->phdr.hdr.u.l.oth; + ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth; /* Sending responses has higher priority over sending requests. */ if ((qp->s_flags & RVT_S_RESP_PENDING) && @@ -724,7 +724,8 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp, u32 vl, plen; struct send_context *sc; struct pio_buf *pbuf; - struct ib_header hdr; + struct hfi1_opa_header opah; + struct ib_header *hdr; struct ib_other_headers *ohdr; unsigned long flags; @@ -741,16 +742,19 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp, goto queue_ack; /* Construct the header */ + opah.hdr_type = 0; + hdr = &opah.ibh; + /* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4 */ hwords = 6; if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) { - hwords += hfi1_make_grh(ibp, &hdr.u.l.grh, + hwords += hfi1_make_grh(ibp, &hdr->u.l.grh, rdma_ah_read_grh(&qp->remote_ah_attr), hwords, 0); - ohdr = &hdr.u.l.oth; + ohdr = &hdr->u.l.oth; lrh0 = HFI1_LRH_GRH; } else { - ohdr = &hdr.u.oth; + ohdr = &hdr->u.oth; lrh0 = HFI1_LRH_BTH; } /* read pkey_index w/o lock (its atomic) */ @@ -768,11 +772,11 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp, pbc_flags |= (ib_is_sc5(sc5) << PBC_DC_INFO_SHIFT); lrh0 |= (sc5 & 0xf) << 12 | (rdma_ah_get_sl(&qp->remote_ah_attr) & 0xf) << 4; - hdr.lrh[0] = cpu_to_be16(lrh0); - hdr.lrh[1] = cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr)); - hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC); - hdr.lrh[3] = cpu_to_be16(ppd->lid | - rdma_ah_get_path_bits(&qp->remote_ah_attr)); + hdr->lrh[0] = cpu_to_be16(lrh0); + hdr->lrh[1] = cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr)); + hdr->lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC); + hdr->lrh[3] = cpu_to_be16(ppd->lid | + rdma_ah_get_path_bits(&qp->remote_ah_attr)); ohdr->bth[0] = cpu_to_be32(bth0); ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); ohdr->bth[1] |= cpu_to_be32((!!is_fecn) << IB_BECN_SHIFT); @@ -799,10 +803,10 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp, } trace_ack_output_ibhdr(dd_from_ibdev(qp->ibqp.device), - &hdr, ib_is_sc5(sc5)); + &opah, ib_is_sc5(sc5)); /* write the pbc and data */ - ppd->dd->pio_inline_send(ppd->dd, pbuf, pbc, &hdr, hwords); + ppd->dd->pio_inline_send(ppd->dd, pbuf, pbc, hdr, hwords); return; @@ -985,9 +989,10 @@ static void reset_sending_psn(struct rvt_qp *qp, u32 psn) /* * This should be called with the QP s_lock held and interrupts disabled. */ -void hfi1_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr) +void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah) { struct ib_other_headers *ohdr; + struct ib_header *hdr = &opah->ibh; struct rvt_swqe *wqe; u32 opcode; u32 psn; diff --git a/drivers/infiniband/hw/hfi1/ruc.c b/drivers/infiniband/hw/hfi1/ruc.c index 4afa00f921f2..e30c64fcce67 100644 --- a/drivers/infiniband/hw/hfi1/ruc.c +++ b/drivers/infiniband/hw/hfi1/ruc.c @@ -668,7 +668,8 @@ u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr, return sizeof(struct ib_grh) / sizeof(u32); } -#define BTH2_OFFSET (offsetof(struct hfi1_sdma_header, hdr.u.oth.bth[2]) / 4) +#define BTH2_OFFSET (offsetof(struct hfi1_sdma_header, \ + hdr.ibh.u.oth.bth[2]) / 4) /** * build_ahg - create ahg in s_ahg @@ -743,8 +744,8 @@ void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr, if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) { qp->s_hdrwords += hfi1_make_grh(ibp, - &ps->s_txreq->phdr.hdr.u.l.grh, - rdma_ah_read_grh(&qp->remote_ah_attr), + &ps->s_txreq->phdr.hdr.ibh.u.l.grh, + &qp->remote_ah_attr.grh, qp->s_hdrwords, nwords); lrh0 = HFI1_LRH_GRH; middle = 0; @@ -773,14 +774,14 @@ void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr, build_ahg(qp, bth2); else qp->s_flags &= ~RVT_S_AHG_VALID; - ps->s_txreq->phdr.hdr.lrh[0] = cpu_to_be16(lrh0); - ps->s_txreq->phdr.hdr.lrh[1] = + ps->s_txreq->phdr.hdr.ibh.lrh[0] = cpu_to_be16(lrh0); + ps->s_txreq->phdr.hdr.ibh.lrh[1] = cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr)); - ps->s_txreq->phdr.hdr.lrh[2] = + ps->s_txreq->phdr.hdr.ibh.lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC); - ps->s_txreq->phdr.hdr.lrh[3] = + ps->s_txreq->phdr.hdr.ibh.lrh[3] = cpu_to_be16(ppd_from_ibp(ibp)->lid | - rdma_ah_get_path_bits(&qp->remote_ah_attr)); + rdma_ah_get_path_bits(&qp->remote_ah_attr)); bth0 |= hfi1_get_pkey(ibp, qp->s_pkey_index); bth0 |= extra_bytes << 20; ohdr->bth[0] = cpu_to_be32(bth0); diff --git a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h index 0f2d2da057ec..73240255ffc5 100644 --- a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h +++ b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h @@ -213,9 +213,9 @@ DEFINE_EVENT(hfi1_input_ibhdr_template, input_ibhdr, DECLARE_EVENT_CLASS(hfi1_output_ibhdr_template, TP_PROTO(struct hfi1_devdata *dd, - struct ib_header *hdr, + struct hfi1_opa_header *opah, bool sc5), - TP_ARGS(dd, hdr, sc5), + TP_ARGS(dd, opah, sc5), TP_STRUCT__entry( DD_DEV_ENTRY(dd) __field(u8, lnh) @@ -238,10 +238,11 @@ DECLARE_EVENT_CLASS(hfi1_output_ibhdr_template, __field(u32, psn) /* extended headers */ __dynamic_array(u8, ehdrs, - hfi1_trace_ib_hdr_len(hdr)) + hfi1_trace_ib_hdr_len(&opah->ibh)) ), TP_fast_assign( struct ib_other_headers *ohdr; + struct ib_header *hdr = &opah->ibh; DD_DEV_ASSIGN(dd); @@ -294,18 +295,18 @@ DECLARE_EVENT_CLASS(hfi1_output_ibhdr_template, DEFINE_EVENT(hfi1_output_ibhdr_template, pio_output_ibhdr, TP_PROTO(struct hfi1_devdata *dd, - struct ib_header *hdr, bool sc5), - TP_ARGS(dd, hdr, sc5)); + struct hfi1_opa_header *opah, bool sc5), + TP_ARGS(dd, opah, sc5)); DEFINE_EVENT(hfi1_output_ibhdr_template, ack_output_ibhdr, TP_PROTO(struct hfi1_devdata *dd, - struct ib_header *hdr, bool sc5), - TP_ARGS(dd, hdr, sc5)); + struct hfi1_opa_header *opah, bool sc5), + TP_ARGS(dd, opah, sc5)); DEFINE_EVENT(hfi1_output_ibhdr_template, sdma_output_ibhdr, TP_PROTO(struct hfi1_devdata *dd, - struct ib_header *hdr, bool sc5), - TP_ARGS(dd, hdr, sc5)); + struct hfi1_opa_header *opah, bool sc5), + TP_ARGS(dd, opah, sc5)); #endif /* __HFI1_TRACE_IBHDRS_H */ diff --git a/drivers/infiniband/hw/hfi1/uc.c b/drivers/infiniband/hw/hfi1/uc.c index 366f7b9517fe..e0bb766ae36c 100644 --- a/drivers/infiniband/hw/hfi1/uc.c +++ b/drivers/infiniband/hw/hfi1/uc.c @@ -93,9 +93,9 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) goto done_free_tx; } - ohdr = &ps->s_txreq->phdr.hdr.u.oth; + ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth; if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) - ohdr = &ps->s_txreq->phdr.hdr.u.l.oth; + ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth; /* Get the next send request. */ wqe = rvt_get_swqe_ptr(qp, qp->s_cur); diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c index dcf8c14c6d0e..2af993cbe5af 100644 --- a/drivers/infiniband/hw/hfi1/ud.c +++ b/drivers/infiniband/hw/hfi1/ud.c @@ -357,12 +357,13 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) { /* Header size in 32-bit words. */ - qp->s_hdrwords += hfi1_make_grh(ibp, - &ps->s_txreq->phdr.hdr.u.l.grh, - rdma_ah_read_grh(ah_attr), - qp->s_hdrwords, nwords); + qp->s_hdrwords += + hfi1_make_grh(ibp, + &ps->s_txreq->phdr.hdr.ibh.u.l.grh, + rdma_ah_read_grh(ah_attr), + qp->s_hdrwords, nwords); lrh0 = HFI1_LRH_GRH; - ohdr = &ps->s_txreq->phdr.hdr.u.l.oth; + ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth; /* * Don't worry about sending to locally attached multicast * QPs. It is unspecified by the spec. what happens. @@ -370,7 +371,7 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) } else { /* Header size in 32-bit words. */ lrh0 = HFI1_LRH_BTH; - ohdr = &ps->s_txreq->phdr.hdr.u.oth; + ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth; } if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) { qp->s_hdrwords++; @@ -392,21 +393,21 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) ps->s_txreq->sde = priv->s_sde; priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc); ps->s_txreq->psc = priv->s_sendcontext; - ps->s_txreq->phdr.hdr.lrh[0] = cpu_to_be16(lrh0); - ps->s_txreq->phdr.hdr.lrh[1] = + ps->s_txreq->phdr.hdr.ibh.lrh[0] = cpu_to_be16(lrh0); + ps->s_txreq->phdr.hdr.ibh.lrh[1] = cpu_to_be16(rdma_ah_get_dlid(ah_attr)); - ps->s_txreq->phdr.hdr.lrh[2] = + ps->s_txreq->phdr.hdr.ibh.lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC); if (rdma_ah_get_dlid(ah_attr) == be16_to_cpu(IB_LID_PERMISSIVE)) { - ps->s_txreq->phdr.hdr.lrh[3] = IB_LID_PERMISSIVE; + ps->s_txreq->phdr.hdr.ibh.lrh[3] = IB_LID_PERMISSIVE; } else { lid = ppd->lid; if (lid) { lid |= rdma_ah_get_path_bits(ah_attr) & ((1 << ppd->lmc) - 1); - ps->s_txreq->phdr.hdr.lrh[3] = cpu_to_be16(lid); + ps->s_txreq->phdr.hdr.ibh.lrh[3] = cpu_to_be16(lid); } else { - ps->s_txreq->phdr.hdr.lrh[3] = IB_LID_PERMISSIVE; + ps->s_txreq->phdr.hdr.ibh.lrh[3] = IB_LID_PERMISSIVE; } } if (wqe->wr.send_flags & IB_SEND_SOLICITED) diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c index ebddab1a06f4..4aec805c645d 100644 --- a/drivers/infiniband/hw/hfi1/verbs.c +++ b/drivers/infiniband/hw/hfi1/verbs.c @@ -506,6 +506,24 @@ void hfi1_copy_sge( } } +static u8 get_opcode(struct hfi1_opa_header *hdr) +{ + struct ib_other_headers *ohdr; + + if (hdr->hdr_type) { + if (hfi1_16B_get_l4(&hdr->opah) == OPA_16B_L4_IB_LOCAL) + ohdr = &hdr->opah.u.oth; + else + ohdr = &hdr->opah.u.l.oth; + } else { + if (ib_get_lnh(&hdr->ibh) == HFI1_LRH_BTH) + ohdr = &hdr->ibh.u.oth; + else + ohdr = &hdr->ibh.u.l.oth; + } + return ib_bth_get_opcode(ohdr); +} + /* * Make sure the QP is ready and able to accept the given opcode. */ @@ -686,7 +704,7 @@ static void verbs_sdma_complete( if (tx->wqe) { hfi1_send_complete(qp, tx->wqe, IB_WC_SUCCESS); } else if (qp->ibqp.qp_type == IB_QPT_RC) { - struct ib_header *hdr; + struct hfi1_opa_header *hdr; hdr = &tx->phdr.hdr; hfi1_rc_send_complete(qp, hdr); @@ -1175,7 +1193,7 @@ static inline send_routine get_send_routine(struct rvt_qp *qp, { struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); struct hfi1_qp_priv *priv = qp->priv; - struct ib_header *h = &tx->phdr.hdr; + struct hfi1_opa_header *h = &tx->phdr.hdr; if (unlikely(!(dd->flags & HFI1_HAS_SEND_DMA))) return dd->process_pio_send; @@ -1221,7 +1239,7 @@ int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps) int ret; u8 lnh; - hdr = &ps->s_txreq->phdr.hdr; + hdr = &ps->s_txreq->phdr.hdr.ibh; /* locate the pkey within the headers */ lnh = ib_get_lnh(hdr); if (lnh == HFI1_LRH_GRH) diff --git a/drivers/infiniband/hw/hfi1/verbs.h b/drivers/infiniband/hw/hfi1/verbs.h index 590aab270f98..20224100cbc5 100644 --- a/drivers/infiniband/hw/hfi1/verbs.h +++ b/drivers/infiniband/hw/hfi1/verbs.h @@ -115,6 +115,14 @@ struct hfi1_16b_header { } u; } __packed; +struct hfi1_opa_header { + union { + struct ib_header ibh; /* 9B header */ + struct hfi1_16b_header opah; /* 16B header */ + }; + u8 hdr_type; /* 9B or 16B */ +} __packed; + struct hfi1_ahg_info { u32 ahgdesc[2]; u16 tx_flags; @@ -124,7 +132,7 @@ struct hfi1_ahg_info { struct hfi1_sdma_header { __le64 pbc; - struct ib_header hdr; + struct hfi1_opa_header hdr; } __packed; /* @@ -326,7 +334,7 @@ u8 ah_to_sc(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr); struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid); -void hfi1_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr); +void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah); void hfi1_ud_rcv(struct hfi1_packet *packet); @@ -347,16 +355,6 @@ int hfi1_check_send_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe); extern const u32 rc_only_opcode; extern const u32 uc_only_opcode; -static inline u8 get_opcode(struct ib_header *h) -{ - u16 lnh = be16_to_cpu(h->lrh[0]) & 3; - - if (lnh == IB_LNH_IBA_LOCAL) - return be32_to_cpu(h->u.oth.bth[0]) >> 24; - else - return be32_to_cpu(h->u.l.oth.bth[0]) >> 24; -} - int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct hfi1_packet *packet); u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr,