From 6dad4e8ab3ec65c3b948ad79e83751cf0f04cbdf Mon Sep 17 00:00:00 2001 From: Atul Gupta Date: Thu, 16 Nov 2017 16:57:08 +0530 Subject: [PATCH] chcr: Add support for Inline IPSec register xfrmdev_ops callbacks, Send IPsec tunneled data to HW for inline processing. The driver use hardware crypto accelerator to encrypt and generate ICV for the transmitted packet in Inline mode. Signed-off-by: Atul Gupta Signed-off-by: Harsh Jain Signed-off-by: Ganesh Goudar Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/Kconfig | 10 + drivers/crypto/chelsio/Makefile | 1 + drivers/crypto/chelsio/chcr_algo.c | 221 ++++----- drivers/crypto/chelsio/chcr_algo.h | 15 - drivers/crypto/chelsio/chcr_core.c | 14 + drivers/crypto/chelsio/chcr_core.h | 38 ++ drivers/crypto/chelsio/chcr_crypto.h | 69 +-- drivers/crypto/chelsio/chcr_ipsec.c | 660 +++++++++++++++++++++++++++ 8 files changed, 860 insertions(+), 168 deletions(-) create mode 100644 drivers/crypto/chelsio/chcr_ipsec.c diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig index 3e104f5aa0c2..51932c7bde5d 100644 --- a/drivers/crypto/chelsio/Kconfig +++ b/drivers/crypto/chelsio/Kconfig @@ -18,3 +18,13 @@ config CRYPTO_DEV_CHELSIO To compile this driver as a module, choose M here: the module will be called chcr. + +config CHELSIO_IPSEC_INLINE + bool "Chelsio IPSec XFRM Tx crypto offload" + depends on CHELSIO_T4 + depends on CRYPTO_DEV_CHELSIO + depends on XFRM_OFFLOAD + depends on INET_ESP_OFFLOAD || INET6_ESP_OFFLOAD + default n + ---help--- + Enable support for IPSec Tx Inline. diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile index bebdf06687ad..eaecaf1ebcf3 100644 --- a/drivers/crypto/chelsio/Makefile +++ b/drivers/crypto/chelsio/Makefile @@ -2,3 +2,4 @@ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o chcr-objs := chcr_core.o chcr_algo.o +chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index af08dd264ca7..e61ec8a46340 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -73,6 +73,25 @@ #define IV AES_BLOCK_SIZE +unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88, + 96, 112, 120, 136, 144, 160, 168, 184, + 192, 208, 216, 232, 240, 256, 264, 280, + 288, 304, 312, 328, 336, 352, 360, 376}; + +unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80, + 112, 112, 128, 128, 144, 144, 160, 160, + 192, 192, 208, 208, 224, 224, 240, 240, + 272, 272, 288, 288, 304, 304, 320, 320}; + +static u32 round_constant[11] = { + 0x01000000, 0x02000000, 0x04000000, 0x08000000, + 0x10000000, 0x20000000, 0x40000000, 0x80000000, + 0x1B000000, 0x36000000, 0x6C000000 +}; + +static int chcr_handle_cipher_resp(struct ablkcipher_request *req, + unsigned char *input, int err); + static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) { return ctx->crypto_ctx->aeadctx; @@ -108,18 +127,6 @@ static inline int is_ofld_imm(const struct sk_buff *skb) return (skb->len <= SGE_MAX_WR_LEN); } -/* - * sgl_len - calculates the size of an SGL of the given capacity - * @n: the number of SGL entries - * Calculates the number of flits needed for a scatter/gather list that - * can hold the given number of entries. - */ -static inline unsigned int sgl_len(unsigned int n) -{ - n--; - return (3 * n) / 2 + (n & 1) + 2; -} - static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen, unsigned int entlen, unsigned int skip) @@ -182,30 +189,17 @@ static inline void chcr_handle_ahash_resp(struct ahash_request *req, } out: req->base.complete(&req->base, err); +} - } - -static inline void chcr_handle_aead_resp(struct aead_request *req, - unsigned char *input, - int err) +static inline int get_aead_subtype(struct crypto_aead *aead) { - struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm)); - - - chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op); - if (reqctx->b0_dma) - dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma, - reqctx->b0_len, DMA_BIDIRECTIONAL); - if (reqctx->verify == VERIFY_SW) { - chcr_verify_tag(req, input, &err); - reqctx->verify = VERIFY_HW; + struct aead_alg *alg = crypto_aead_alg(aead); + struct chcr_alg_template *chcr_crypto_alg = + container_of(alg, struct chcr_alg_template, alg.aead); + return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; } - req->base.complete(&req->base, err); -} -static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) +void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) { u8 temp[SHA512_DIGEST_SIZE]; struct crypto_aead *tfm = crypto_aead_reqtfm(req); @@ -230,6 +224,25 @@ static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) *err = 0; } +static inline void chcr_handle_aead_resp(struct aead_request *req, + unsigned char *input, + int err) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm)); + + chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op); + if (reqctx->b0_dma) + dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma, + reqctx->b0_len, DMA_BIDIRECTIONAL); + if (reqctx->verify == VERIFY_SW) { + chcr_verify_tag(req, input, &err); + reqctx->verify = VERIFY_HW; + } + req->base.complete(&req->base, err); +} + /* * chcr_handle_resp - Unmap the DMA buffers associated with the request * @req: crypto request @@ -594,14 +607,6 @@ static void ulptx_walk_add_sg(struct ulptx_walk *walk, } } -static inline int get_aead_subtype(struct crypto_aead *aead) -{ - struct aead_alg *alg = crypto_aead_alg(aead); - struct chcr_alg_template *chcr_crypto_alg = - container_of(alg, struct chcr_alg_template, alg.aead); - return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; -} - static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm) { struct crypto_alg *alg = tfm->__crt_alg; @@ -1100,7 +1105,6 @@ static int chcr_final_cipher_iv(struct ablkcipher_request *req, } - static int chcr_handle_cipher_resp(struct ablkcipher_request *req, unsigned char *input, int err) { @@ -2198,9 +2202,9 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, return ERR_PTR(error); } -static int chcr_aead_dma_map(struct device *dev, - struct aead_request *req, - unsigned short op_type) +int chcr_aead_dma_map(struct device *dev, + struct aead_request *req, + unsigned short op_type) { int error; struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); @@ -2242,9 +2246,9 @@ static int chcr_aead_dma_map(struct device *dev, return -ENOMEM; } -static void chcr_aead_dma_unmap(struct device *dev, - struct aead_request *req, - unsigned short op_type) +void chcr_aead_dma_unmap(struct device *dev, + struct aead_request *req, + unsigned short op_type) { struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct crypto_aead *tfm = crypto_aead_reqtfm(req); @@ -2269,10 +2273,10 @@ static void chcr_aead_dma_unmap(struct device *dev, } } -static inline void chcr_add_aead_src_ent(struct aead_request *req, - struct ulptx_sgl *ulptx, - unsigned int assoclen, - unsigned short op_type) +void chcr_add_aead_src_ent(struct aead_request *req, + struct ulptx_sgl *ulptx, + unsigned int assoclen, + unsigned short op_type) { struct ulptx_walk ulp_walk; struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); @@ -2304,11 +2308,11 @@ static inline void chcr_add_aead_src_ent(struct aead_request *req, } } -static inline void chcr_add_aead_dst_ent(struct aead_request *req, - struct cpl_rx_phys_dsgl *phys_cpl, - unsigned int assoclen, - unsigned short op_type, - unsigned short qid) +void chcr_add_aead_dst_ent(struct aead_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + unsigned int assoclen, + unsigned short op_type, + unsigned short qid) { struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct crypto_aead *tfm = crypto_aead_reqtfm(req); @@ -2326,9 +2330,9 @@ static inline void chcr_add_aead_dst_ent(struct aead_request *req, dsgl_walk_end(&dsgl_walk, qid); } -static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, - struct ulptx_sgl *ulptx, - struct cipher_wr_param *wrparam) +void chcr_add_cipher_src_ent(struct ablkcipher_request *req, + struct ulptx_sgl *ulptx, + struct cipher_wr_param *wrparam) { struct ulptx_walk ulp_walk; struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); @@ -2351,10 +2355,10 @@ static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, } } -static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, - struct cpl_rx_phys_dsgl *phys_cpl, - struct cipher_wr_param *wrparam, - unsigned short qid) +void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + struct cipher_wr_param *wrparam, + unsigned short qid) { struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); struct dsgl_walk dsgl_walk; @@ -2369,9 +2373,9 @@ static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, dsgl_walk_end(&dsgl_walk, qid); } -static inline void chcr_add_hash_src_ent(struct ahash_request *req, - struct ulptx_sgl *ulptx, - struct hash_wr_param *param) +void chcr_add_hash_src_ent(struct ahash_request *req, + struct ulptx_sgl *ulptx, + struct hash_wr_param *param) { struct ulptx_walk ulp_walk; struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); @@ -2398,9 +2402,8 @@ static inline void chcr_add_hash_src_ent(struct ahash_request *req, } } - -static inline int chcr_hash_dma_map(struct device *dev, - struct ahash_request *req) +int chcr_hash_dma_map(struct device *dev, + struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); int error = 0; @@ -2415,8 +2418,8 @@ static inline int chcr_hash_dma_map(struct device *dev, return 0; } -static inline void chcr_hash_dma_unmap(struct device *dev, - struct ahash_request *req) +void chcr_hash_dma_unmap(struct device *dev, + struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); @@ -2429,9 +2432,8 @@ static inline void chcr_hash_dma_unmap(struct device *dev, } - -static int chcr_cipher_dma_map(struct device *dev, - struct ablkcipher_request *req) +int chcr_cipher_dma_map(struct device *dev, + struct ablkcipher_request *req) { int error; struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); @@ -2465,8 +2467,9 @@ static int chcr_cipher_dma_map(struct device *dev, dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); return -ENOMEM; } -static void chcr_cipher_dma_unmap(struct device *dev, - struct ablkcipher_request *req) + +void chcr_cipher_dma_unmap(struct device *dev, + struct ablkcipher_request *req) { struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); @@ -3371,6 +3374,40 @@ static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, aeadctx->enckey_len = 0; return -EINVAL; } + +static int chcr_aead_op(struct aead_request *req, + unsigned short op_type, + int size, + create_wr_t create_wr_fn) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct uld_ctx *u_ctx; + struct sk_buff *skb; + + if (!a_ctx(tfm)->dev) { + pr_err("chcr : %s : No crypto device.\n", __func__); + return -ENXIO; + } + u_ctx = ULD_CTX(a_ctx(tfm)); + if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], + a_ctx(tfm)->tx_qidx)) { + if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return -EBUSY; + } + + /* Form a WR from req */ + skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size, + op_type); + + if (IS_ERR(skb) || !skb) + return PTR_ERR(skb); + + skb->dev = u_ctx->lldi.ports[0]; + set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx); + chcr_send_wr(skb); + return -EINPROGRESS; +} + static int chcr_aead_encrypt(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); @@ -3423,38 +3460,6 @@ static int chcr_aead_decrypt(struct aead_request *req) } } -static int chcr_aead_op(struct aead_request *req, - unsigned short op_type, - int size, - create_wr_t create_wr_fn) -{ - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct uld_ctx *u_ctx; - struct sk_buff *skb; - - if (!a_ctx(tfm)->dev) { - pr_err("chcr : %s : No crypto device.\n", __func__); - return -ENXIO; - } - u_ctx = ULD_CTX(a_ctx(tfm)); - if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - a_ctx(tfm)->tx_qidx)) { - if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) - return -EBUSY; - } - - /* Form a WR from req */ - skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size, - op_type); - - if (IS_ERR(skb) || !skb) - return PTR_ERR(skb); - - skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx); - chcr_send_wr(skb); - return -EINPROGRESS; -} static struct chcr_alg_template driver_algs[] = { /* AES-CBC */ { diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index 96c9335ee728..d1673a5d4bf5 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -226,15 +226,6 @@ #define SPACE_LEFT(len) \ ((SGE_MAX_WR_LEN - WR_MIN_LEN - (len))) -unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88, - 96, 112, 120, 136, 144, 160, 168, 184, - 192, 208, 216, 232, 240, 256, 264, 280, - 288, 304, 312, 328, 336, 352, 360, 376}; -unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80, - 112, 112, 128, 128, 144, 144, 160, 160, - 192, 192, 208, 208, 224, 224, 240, 240, - 272, 272, 288, 288, 304, 304, 320, 320}; - struct algo_param { unsigned int auth_mode; unsigned int mk_size; @@ -404,10 +395,4 @@ static inline u32 aes_ks_subword(const u32 w) return *(u32 *)(&bytes[0]); } -static u32 round_constant[11] = { - 0x01000000, 0x02000000, 0x04000000, 0x08000000, - 0x10000000, 0x20000000, 0x40000000, 0x80000000, - 0x1B000000, 0x36000000, 0x6C000000 -}; - #endif /* __CHCR_ALGO_H__ */ diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index f5a2624081dc..04f277cade7c 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -48,6 +48,9 @@ static struct cxgb4_uld_info chcr_uld_info = { .add = chcr_uld_add, .state_change = chcr_uld_state_change, .rx_handler = chcr_uld_rx_handler, +#ifdef CONFIG_CHELSIO_IPSEC_INLINE + .tx_handler = chcr_uld_tx_handler, +#endif /* CONFIG_CHELSIO_IPSEC_INLINE */ }; struct uld_ctx *assign_chcr_device(void) @@ -164,6 +167,10 @@ static void *chcr_uld_add(const struct cxgb4_lld_info *lld) goto out; } u_ctx->lldi = *lld; +#ifdef CONFIG_CHELSIO_IPSEC_INLINE + if (lld->crypto & ULP_CRYPTO_IPSEC_INLINE) + chcr_add_xfrmops(lld); +#endif /* CONFIG_CHELSIO_IPSEC_INLINE */ out: return u_ctx; } @@ -187,6 +194,13 @@ int chcr_uld_rx_handler(void *handle, const __be64 *rsp, return 0; } +#ifdef CONFIG_CHELSIO_IPSEC_INLINE +int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev) +{ + return chcr_ipsec_xmit(skb, dev); +} +#endif /* CONFIG_CHELSIO_IPSEC_INLINE */ + static int chcr_uld_state_change(void *handle, enum cxgb4_state state) { struct uld_ctx *u_ctx = handle; diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h index 94e7412f6164..3c29ee09b8b5 100644 --- a/drivers/crypto/chelsio/chcr_core.h +++ b/drivers/crypto/chelsio/chcr_core.h @@ -39,6 +39,7 @@ #include #include "t4_hw.h" #include "cxgb4.h" +#include "t4_msg.h" #include "cxgb4_uld.h" #define DRV_MODULE_NAME "chcr" @@ -89,12 +90,49 @@ struct uld_ctx { struct chcr_dev *dev; }; +struct chcr_ipsec_req { + struct ulp_txpkt ulptx; + struct ulptx_idata sc_imm; + struct cpl_tx_sec_pdu sec_cpl; + struct _key_ctx key_ctx; +}; + +struct chcr_ipsec_wr { + struct fw_ulptx_wr wreq; + struct chcr_ipsec_req req; +}; + +struct ipsec_sa_entry { + int hmac_ctrl; + unsigned int enckey_len; + unsigned int kctx_len; + unsigned int authsize; + __be32 key_ctx_hdr; + char salt[MAX_SALT]; + char key[2 * AES_MAX_KEY_SIZE]; +}; + +/* + * sgl_len - calculates the size of an SGL of the given capacity + * @n: the number of SGL entries + * Calculates the number of flits needed for a scatter/gather list that + * can hold the given number of entries. + */ +static inline unsigned int sgl_len(unsigned int n) +{ + n--; + return (3 * n) / 2 + (n & 1) + 2; +} + struct uld_ctx *assign_chcr_device(void); int chcr_send_wr(struct sk_buff *skb); int start_crypto(void); int stop_crypto(void); int chcr_uld_rx_handler(void *handle, const __be64 *rsp, const struct pkt_gl *pgl); +int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev); int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, int err); +int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev); +void chcr_add_xfrmops(const struct cxgb4_lld_info *lld); #endif /* __CHCR_CORE_H__ */ diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 94a87e3ad9bc..ea2c578805e6 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -210,8 +210,6 @@ struct dsgl_walk { struct phys_sge_pairs *to; }; - - struct chcr_gcm_ctx { u8 ghash_h[AEAD_H_SIZE]; }; @@ -227,8 +225,6 @@ struct __aead_ctx { struct chcr_authenc_ctx authenc[0]; }; - - struct chcr_aead_ctx { __be32 key_ctx_hdr; unsigned int enckey_len; @@ -240,8 +236,6 @@ struct chcr_aead_ctx { struct __aead_ctx ctx[0]; }; - - struct hmac_ctx { struct crypto_shash *base_hash; u8 ipad[CHCR_HASH_MAX_BLOCK_SIZE_128]; @@ -307,44 +301,29 @@ typedef struct sk_buff *(*create_wr_t)(struct aead_request *req, int size, unsigned short op_type); -static int chcr_aead_op(struct aead_request *req_base, - unsigned short op_type, - int size, - create_wr_t create_wr_fn); -static inline int get_aead_subtype(struct crypto_aead *aead); -static int chcr_handle_cipher_resp(struct ablkcipher_request *req, - unsigned char *input, int err); -static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err); -static int chcr_aead_dma_map(struct device *dev, struct aead_request *req, - unsigned short op_type); -static void chcr_aead_dma_unmap(struct device *dev, struct aead_request - *req, unsigned short op_type); -static inline void chcr_add_aead_dst_ent(struct aead_request *req, - struct cpl_rx_phys_dsgl *phys_cpl, - unsigned int assoclen, - unsigned short op_type, - unsigned short qid); -static inline void chcr_add_aead_src_ent(struct aead_request *req, - struct ulptx_sgl *ulptx, - unsigned int assoclen, - unsigned short op_type); -static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, - struct ulptx_sgl *ulptx, - struct cipher_wr_param *wrparam); -static int chcr_cipher_dma_map(struct device *dev, - struct ablkcipher_request *req); -static void chcr_cipher_dma_unmap(struct device *dev, - struct ablkcipher_request *req); -static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, - struct cpl_rx_phys_dsgl *phys_cpl, - struct cipher_wr_param *wrparam, - unsigned short qid); +void chcr_verify_tag(struct aead_request *req, u8 *input, int *err); +int chcr_aead_dma_map(struct device *dev, struct aead_request *req, + unsigned short op_type); +void chcr_aead_dma_unmap(struct device *dev, struct aead_request *req, + unsigned short op_type); +void chcr_add_aead_dst_ent(struct aead_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + unsigned int assoclen, unsigned short op_type, + unsigned short qid); +void chcr_add_aead_src_ent(struct aead_request *req, struct ulptx_sgl *ulptx, + unsigned int assoclen, unsigned short op_type); +void chcr_add_cipher_src_ent(struct ablkcipher_request *req, + struct ulptx_sgl *ulptx, + struct cipher_wr_param *wrparam); +int chcr_cipher_dma_map(struct device *dev, struct ablkcipher_request *req); +void chcr_cipher_dma_unmap(struct device *dev, struct ablkcipher_request *req); +void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + struct cipher_wr_param *wrparam, + unsigned short qid); int sg_nents_len_skip(struct scatterlist *sg, u64 len, u64 skip); -static inline void chcr_add_hash_src_ent(struct ahash_request *req, - struct ulptx_sgl *ulptx, - struct hash_wr_param *param); -static inline int chcr_hash_dma_map(struct device *dev, - struct ahash_request *req); -static inline void chcr_hash_dma_unmap(struct device *dev, - struct ahash_request *req); +void chcr_add_hash_src_ent(struct ahash_request *req, struct ulptx_sgl *ulptx, + struct hash_wr_param *param); +int chcr_hash_dma_map(struct device *dev, struct ahash_request *req); +void chcr_hash_dma_unmap(struct device *dev, struct ahash_request *req); #endif /* __CHCR_CRYPTO_H__ */ diff --git a/drivers/crypto/chelsio/chcr_ipsec.c b/drivers/crypto/chelsio/chcr_ipsec.c new file mode 100644 index 000000000000..f90f991c133f --- /dev/null +++ b/drivers/crypto/chelsio/chcr_ipsec.c @@ -0,0 +1,660 @@ +/* + * This file is part of the Chelsio T6 Crypto driver for Linux. + * + * Copyright (c) 2003-2017 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Written and Maintained by: + * Atul Gupta (atul.gupta@chelsio.com) + */ + +#define pr_fmt(fmt) "chcr:" fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "chcr_core.h" +#include "chcr_algo.h" +#include "chcr_crypto.h" + +/* + * Max Tx descriptor space we allow for an Ethernet packet to be inlined + * into a WR. + */ +#define MAX_IMM_TX_PKT_LEN 256 +#define GCM_ESP_IV_SIZE 8 + +static int chcr_xfrm_add_state(struct xfrm_state *x); +static void chcr_xfrm_del_state(struct xfrm_state *x); +static void chcr_xfrm_free_state(struct xfrm_state *x); +static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x); + +static const struct xfrmdev_ops chcr_xfrmdev_ops = { + .xdo_dev_state_add = chcr_xfrm_add_state, + .xdo_dev_state_delete = chcr_xfrm_del_state, + .xdo_dev_state_free = chcr_xfrm_free_state, + .xdo_dev_offload_ok = chcr_ipsec_offload_ok, +}; + +/* Add offload xfrms to Chelsio Interface */ +void chcr_add_xfrmops(const struct cxgb4_lld_info *lld) +{ + struct net_device *netdev = NULL; + int i; + + for (i = 0; i < lld->nports; i++) { + netdev = lld->ports[i]; + if (!netdev) + continue; + netdev->xfrmdev_ops = &chcr_xfrmdev_ops; + netdev->hw_enc_features |= NETIF_F_HW_ESP; + netdev->features |= NETIF_F_HW_ESP; + rtnl_lock(); + netdev_change_features(netdev); + rtnl_unlock(); + } +} + +static inline int chcr_ipsec_setauthsize(struct xfrm_state *x, + struct ipsec_sa_entry *sa_entry) +{ + int hmac_ctrl; + int authsize = x->aead->alg_icv_len / 8; + + sa_entry->authsize = authsize; + + switch (authsize) { + case ICV_8: + hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; + break; + case ICV_12: + hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; + break; + case ICV_16: + hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; + break; + default: + return -EINVAL; + } + return hmac_ctrl; +} + +static inline int chcr_ipsec_setkey(struct xfrm_state *x, + struct ipsec_sa_entry *sa_entry) +{ + struct crypto_cipher *cipher; + int keylen = (x->aead->alg_key_len + 7) / 8; + unsigned char *key = x->aead->alg_key; + int ck_size, key_ctx_size = 0; + unsigned char ghash_h[AEAD_H_SIZE]; + int ret = 0; + + if (keylen > 3) { + keylen -= 4; /* nonce/salt is present in the last 4 bytes */ + memcpy(sa_entry->salt, key + keylen, 4); + } + + if (keylen == AES_KEYSIZE_128) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; + } else if (keylen == AES_KEYSIZE_192) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; + } else if (keylen == AES_KEYSIZE_256) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; + } else { + pr_err("GCM: Invalid key length %d\n", keylen); + ret = -EINVAL; + goto out; + } + + memcpy(sa_entry->key, key, keylen); + sa_entry->enckey_len = keylen; + key_ctx_size = sizeof(struct _key_ctx) + + ((DIV_ROUND_UP(keylen, 16)) << 4) + + AEAD_H_SIZE; + + sa_entry->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, + CHCR_KEYCTX_MAC_KEY_SIZE_128, + 0, 0, + key_ctx_size >> 4); + + /* Calculate the H = CIPH(K, 0 repeated 16 times). + * It will go in key context + */ + cipher = crypto_alloc_cipher("aes-generic", 0, 0); + if (IS_ERR(cipher)) { + sa_entry->enckey_len = 0; + ret = -ENOMEM; + goto out; + } + + ret = crypto_cipher_setkey(cipher, key, keylen); + if (ret) { + sa_entry->enckey_len = 0; + goto out1; + } + memset(ghash_h, 0, AEAD_H_SIZE); + crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h); + memcpy(sa_entry->key + (DIV_ROUND_UP(sa_entry->enckey_len, 16) * + 16), ghash_h, AEAD_H_SIZE); + sa_entry->kctx_len = ((DIV_ROUND_UP(sa_entry->enckey_len, 16)) << 4) + + AEAD_H_SIZE; +out1: + crypto_free_cipher(cipher); +out: + return ret; +} + +/* + * chcr_xfrm_add_state + * returns 0 on success, negative error if failed to send message to FPGA + * positive error if FPGA returned a bad response + */ +static int chcr_xfrm_add_state(struct xfrm_state *x) +{ + struct net_device *netdev = x->xso.dev; + struct port_info *pi = netdev_priv(netdev); + struct ipsec_sa_entry *sa_entry; + struct adapter *adap; + int res = 0; + + adap = pi->adapter; + + if (x->props.aalgo != SADB_AALG_NONE) { + pr_debug("CHCR: Cannot offload authenticated xfrm states\n"); + return -EINVAL; + } + if (x->props.calgo != SADB_X_CALG_NONE) { + pr_debug("CHCR: Cannot offload compressed xfrm states\n"); + return -EINVAL; + } + if (x->props.flags & XFRM_STATE_ESN) { + pr_debug("CHCR: Cannot offload ESN xfrm states\n"); + return -EINVAL; + } + if (x->props.family != AF_INET && + x->props.family != AF_INET6) { + pr_debug("CHCR: Only IPv4/6 xfrm state offloaded\n"); + return -EINVAL; + } + if (x->props.mode != XFRM_MODE_TRANSPORT && + x->props.mode != XFRM_MODE_TUNNEL) { + pr_debug("CHCR: Only transport and tunnel xfrm offload\n"); + return -EINVAL; + } + if (x->id.proto != IPPROTO_ESP) { + pr_debug("CHCR: Only ESP xfrm state offloaded\n"); + return -EINVAL; + } + if (x->encap) { + pr_debug("CHCR: Encapsulated xfrm state not offloaded\n"); + return -EINVAL; + } + if (!x->aead) { + pr_debug("CHCR: Cannot offload xfrm states without aead\n"); + return -EINVAL; + } + if (x->aead->alg_icv_len != 128 && + x->aead->alg_icv_len != 96) { + pr_debug("CHCR: Cannot offload xfrm states with AEAD ICV length other than 96b & 128b\n"); + return -EINVAL; + } + if ((x->aead->alg_key_len != 128 + 32) && + (x->aead->alg_key_len != 256 + 32)) { + pr_debug("CHCR: Cannot offload xfrm states with AEAD key length other than 128/256 bit\n"); + return -EINVAL; + } + if (x->tfcpad) { + pr_debug("CHCR: Cannot offload xfrm states with tfc padding\n"); + return -EINVAL; + } + if (!x->geniv) { + pr_debug("CHCR: Cannot offload xfrm states without geniv\n"); + return -EINVAL; + } + if (strcmp(x->geniv, "seqiv")) { + pr_debug("CHCR: Cannot offload xfrm states with geniv other than seqiv\n"); + return -EINVAL; + } + + sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL); + if (!sa_entry) { + res = -ENOMEM; + goto out; + } + + sa_entry->hmac_ctrl = chcr_ipsec_setauthsize(x, sa_entry); + chcr_ipsec_setkey(x, sa_entry); + x->xso.offload_handle = (unsigned long)sa_entry; + try_module_get(THIS_MODULE); +out: + return res; +} + +static void chcr_xfrm_del_state(struct xfrm_state *x) +{ + /* do nothing */ + if (!x->xso.offload_handle) + return; +} + +static void chcr_xfrm_free_state(struct xfrm_state *x) +{ + struct ipsec_sa_entry *sa_entry; + + if (!x->xso.offload_handle) + return; + + sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle; + kfree(sa_entry); + module_put(THIS_MODULE); +} + +static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) +{ + /* Offload with IP options is not supported yet */ + if (ip_hdr(skb)->ihl > 5) + return false; + + return true; +} + +static inline int is_eth_imm(const struct sk_buff *skb, unsigned int kctx_len) +{ + int hdrlen = sizeof(struct chcr_ipsec_req) + kctx_len; + + hdrlen += sizeof(struct cpl_tx_pkt); + if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen) + return hdrlen; + return 0; +} + +static inline unsigned int calc_tx_sec_flits(const struct sk_buff *skb, + unsigned int kctx_len) +{ + unsigned int flits; + int hdrlen = is_eth_imm(skb, kctx_len); + + /* If the skb is small enough, we can pump it out as a work request + * with only immediate data. In that case we just have to have the + * TX Packet header plus the skb data in the Work Request. + */ + + if (hdrlen) + return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64)); + + flits = sgl_len(skb_shinfo(skb)->nr_frags + 1); + + /* Otherwise, we're going to have to construct a Scatter gather list + * of the skb body and fragments. We also include the flits necessary + * for the TX Packet Work Request and CPL. We always have a firmware + * Write Header (incorporated as part of the cpl_tx_pkt_lso and + * cpl_tx_pkt structures), followed by either a TX Packet Write CPL + * message or, if we're doing a Large Send Offload, an LSO CPL message + * with an embedded TX Packet Write CPL message. + */ + flits += (sizeof(struct fw_ulptx_wr) + + sizeof(struct chcr_ipsec_req) + + kctx_len + + sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64); + return flits; +} + +inline void *copy_cpltx_pktxt(struct sk_buff *skb, + struct net_device *dev, + void *pos) +{ + struct adapter *adap; + struct port_info *pi; + struct sge_eth_txq *q; + struct cpl_tx_pkt_core *cpl; + u64 cntrl; + u32 ctrl0, qidx; + + pi = netdev_priv(dev); + adap = pi->adapter; + qidx = skb->queue_mapping; + q = &adap->sge.ethtxq[qidx + pi->first_qset]; + + cpl = (struct cpl_tx_pkt_core *)pos; + + if (skb->ip_summed == CHECKSUM_PARTIAL) + cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F; + ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) | + TXPKT_PF_V(adap->pf); + if (skb_vlan_tag_present(skb)) { + q->vlan_ins++; + cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb)); + } + + cpl->ctrl0 = htonl(ctrl0); + cpl->pack = htons(0); + cpl->len = htons(skb->len); + cpl->ctrl1 = cpu_to_be64(cntrl); + + pos += sizeof(struct cpl_tx_pkt_core); + return pos; +} + +inline void *copy_key_cpltx_pktxt(struct sk_buff *skb, + struct net_device *dev, + void *pos, + struct ipsec_sa_entry *sa_entry) +{ + struct adapter *adap; + struct port_info *pi; + struct sge_eth_txq *q; + unsigned int len, qidx; + struct _key_ctx *key_ctx; + int left, eoq, key_len; + + pi = netdev_priv(dev); + adap = pi->adapter; + qidx = skb->queue_mapping; + q = &adap->sge.ethtxq[qidx + pi->first_qset]; + len = sa_entry->enckey_len + sizeof(struct cpl_tx_pkt_core); + key_len = sa_entry->kctx_len; + + /* end of queue, reset pos to start of queue */ + eoq = (void *)q->q.stat - pos; + left = eoq; + if (!eoq) { + pos = q->q.desc; + left = 64 * q->q.size; + } + + /* Copy the Key context header */ + key_ctx = (struct _key_ctx *)pos; + key_ctx->ctx_hdr = sa_entry->key_ctx_hdr; + memcpy(key_ctx->salt, sa_entry->salt, MAX_SALT); + pos += sizeof(struct _key_ctx); + left -= sizeof(struct _key_ctx); + + if (likely(len <= left)) { + memcpy(key_ctx->key, sa_entry->key, key_len); + pos += key_len; + } else { + if (key_len <= left) { + memcpy(pos, sa_entry->key, key_len); + pos += key_len; + } else { + memcpy(pos, sa_entry->key, left); + memcpy(q->q.desc, sa_entry->key + left, + key_len - left); + pos = q->q.desc + (key_len - left); + } + } + /* Copy CPL TX PKT XT */ + pos = copy_cpltx_pktxt(skb, dev, pos); + + return pos; +} + +inline void *chcr_crypto_wreq(struct sk_buff *skb, + struct net_device *dev, + void *pos, + int credits, + struct ipsec_sa_entry *sa_entry) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + unsigned int immdatalen = 0; + unsigned int ivsize = GCM_ESP_IV_SIZE; + struct chcr_ipsec_wr *wr; + unsigned int flits; + u32 wr_mid; + int qidx = skb_get_queue_mapping(skb); + struct sge_eth_txq *q = &adap->sge.ethtxq[qidx + pi->first_qset]; + unsigned int kctx_len = sa_entry->kctx_len; + int qid = q->q.cntxt_id; + + atomic_inc(&adap->chcr_stats.ipsec_cnt); + + flits = calc_tx_sec_flits(skb, kctx_len); + + if (is_eth_imm(skb, kctx_len)) + immdatalen = skb->len; + + /* WR Header */ + wr = (struct chcr_ipsec_wr *)pos; + wr->wreq.op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); + wr_mid = FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(flits, 2)); + + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + netif_tx_stop_queue(q->txq); + q->q.stops++; + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + wr_mid |= FW_ULPTX_WR_DATA_F; + wr->wreq.flowid_len16 = htonl(wr_mid); + + /* ULPTX */ + wr->req.ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(pi->port_id, qid); + wr->req.ulptx.len = htonl(DIV_ROUND_UP(flits, 2) - 1); + + /* Sub-command */ + wr->req.sc_imm.cmd_more = FILL_CMD_MORE(immdatalen); + wr->req.sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + + sizeof(wr->req.key_ctx) + + kctx_len + + sizeof(struct cpl_tx_pkt_core) + + immdatalen); + + /* CPL_SEC_PDU */ + wr->req.sec_cpl.op_ivinsrtofst = htonl( + CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | + CPL_TX_SEC_PDU_CPLLEN_V(2) | + CPL_TX_SEC_PDU_PLACEHOLDER_V(1) | + CPL_TX_SEC_PDU_IVINSRTOFST_V( + (skb_transport_offset(skb) + + sizeof(struct ip_esp_hdr) + 1))); + + wr->req.sec_cpl.pldlen = htonl(skb->len); + + wr->req.sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( + (skb_transport_offset(skb) + 1), + (skb_transport_offset(skb) + + sizeof(struct ip_esp_hdr)), + (skb_transport_offset(skb) + + sizeof(struct ip_esp_hdr) + + GCM_ESP_IV_SIZE + 1), 0); + + wr->req.sec_cpl.cipherstop_lo_authinsert = + FILL_SEC_CPL_AUTHINSERT(0, skb_transport_offset(skb) + + sizeof(struct ip_esp_hdr) + + GCM_ESP_IV_SIZE + 1, + sa_entry->authsize, + sa_entry->authsize); + wr->req.sec_cpl.seqno_numivs = + FILL_SEC_CPL_SCMD0_SEQNO(CHCR_ENCRYPT_OP, 1, + CHCR_SCMD_CIPHER_MODE_AES_GCM, + CHCR_SCMD_AUTH_MODE_GHASH, + sa_entry->hmac_ctrl, + ivsize >> 1); + wr->req.sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, + 0, 0, 0); + + pos += sizeof(struct fw_ulptx_wr) + + sizeof(struct ulp_txpkt) + + sizeof(struct ulptx_idata) + + sizeof(struct cpl_tx_sec_pdu); + + pos = copy_key_cpltx_pktxt(skb, dev, pos, sa_entry); + + return pos; +} + +/** + * flits_to_desc - returns the num of Tx descriptors for the given flits + * @n: the number of flits + * + * Returns the number of Tx descriptors needed for the supplied number + * of flits. + */ +static inline unsigned int flits_to_desc(unsigned int n) +{ + WARN_ON(n > SGE_MAX_WR_LEN / 8); + return DIV_ROUND_UP(n, 8); +} + +static inline unsigned int txq_avail(const struct sge_txq *q) +{ + return q->size - 1 - q->in_use; +} + +static void eth_txq_stop(struct sge_eth_txq *q) +{ + netif_tx_stop_queue(q->txq); + q->q.stops++; +} + +static inline void txq_advance(struct sge_txq *q, unsigned int n) +{ + q->in_use += n; + q->pidx += n; + if (q->pidx >= q->size) + q->pidx -= q->size; +} + +/* + * chcr_ipsec_xmit called from ULD Tx handler + */ +int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct xfrm_state *x = xfrm_input_state(skb); + struct ipsec_sa_entry *sa_entry; + u64 *pos, *end, *before, cntrl, *sgl; + int qidx, left, credits; + unsigned int flits = 0, ndesc, kctx_len; + struct adapter *adap; + struct sge_eth_txq *q; + struct port_info *pi; + dma_addr_t addr[MAX_SKB_FRAGS + 1]; + bool immediate = false; + + if (!x->xso.offload_handle) + return NETDEV_TX_BUSY; + + sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle; + kctx_len = sa_entry->kctx_len; + + if (skb->sp->len != 1) { +out_free: dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + + pi = netdev_priv(dev); + adap = pi->adapter; + qidx = skb->queue_mapping; + q = &adap->sge.ethtxq[qidx + pi->first_qset]; + + cxgb4_reclaim_completed_tx(adap, &q->q, true); + cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F; + + flits = calc_tx_sec_flits(skb, sa_entry->kctx_len); + ndesc = flits_to_desc(flits); + credits = txq_avail(&q->q) - ndesc; + + if (unlikely(credits < 0)) { + eth_txq_stop(q); + dev_err(adap->pdev_dev, + "%s: Tx ring %u full while queue awake! cred:%d %d %d flits:%d\n", + dev->name, qidx, credits, ndesc, txq_avail(&q->q), + flits); + return NETDEV_TX_BUSY; + } + + if (is_eth_imm(skb, kctx_len)) + immediate = true; + + if (!immediate && + unlikely(cxgb4_map_skb(adap->pdev_dev, skb, addr) < 0)) { + q->mapping_err++; + goto out_free; + } + + pos = (u64 *)&q->q.desc[q->q.pidx]; + before = (u64 *)pos; + end = (u64 *)pos + flits; + /* Setup IPSec CPL */ + pos = (void *)chcr_crypto_wreq(skb, dev, (void *)pos, + credits, sa_entry); + if (before > (u64 *)pos) { + left = (u8 *)end - (u8 *)q->q.stat; + end = (void *)q->q.desc + left; + } + if (pos == (u64 *)q->q.stat) { + left = (u8 *)end - (u8 *)q->q.stat; + end = (void *)q->q.desc + left; + pos = (void *)q->q.desc; + } + + sgl = (void *)pos; + if (immediate) { + cxgb4_inline_tx_skb(skb, &q->q, sgl); + dev_consume_skb_any(skb); + } else { + int last_desc; + + cxgb4_write_sgl(skb, &q->q, (void *)sgl, end, + 0, addr); + skb_orphan(skb); + + last_desc = q->q.pidx + ndesc - 1; + if (last_desc >= q->q.size) + last_desc -= q->q.size; + q->q.sdesc[last_desc].skb = skb; + q->q.sdesc[last_desc].sgl = (struct ulptx_sgl *)sgl; + } + txq_advance(&q->q, ndesc); + + cxgb4_ring_tx_db(adap, &q->q, ndesc); + return NETDEV_TX_OK; +}