mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-26 19:00:25 +00:00
![Romain Perier](/assets/img/avatar_default.png)
Currently, inner IV/DIGEST data are only copied once into the hash
engines and not set explicitly before launching a request that is not a
first frag. This is an issue especially when multiple ahash reqs are
computed in parallel or chained with cipher request, as the state of the
request being computed is not updated into the hash engine. It leads to
non-deterministic corrupted digest results.
Fixes: commit 2786cee8e5
("crypto: marvell - Move SRAM I/O operations to step functions")
Signed-off-by: Romain Perier <romain.perier@free-electrons.com>
Acked-by: Boris Brezillon <boris.brezillon@free-electrons.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
356 lines
9 KiB
C
356 lines
9 KiB
C
/*
|
|
* Provide TDMA helper functions used by cipher and hash algorithm
|
|
* implementations.
|
|
*
|
|
* Author: Boris Brezillon <boris.brezillon@free-electrons.com>
|
|
* Author: Arnaud Ebalard <arno@natisbad.org>
|
|
*
|
|
* This work is based on an initial version written by
|
|
* Sebastian Andrzej Siewior < sebastian at breakpoint dot cc >
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
* under the terms of the GNU General Public License version 2 as published
|
|
* by the Free Software Foundation.
|
|
*/
|
|
|
|
#include "cesa.h"
|
|
|
|
bool mv_cesa_req_dma_iter_next_transfer(struct mv_cesa_dma_iter *iter,
|
|
struct mv_cesa_sg_dma_iter *sgiter,
|
|
unsigned int len)
|
|
{
|
|
if (!sgiter->sg)
|
|
return false;
|
|
|
|
sgiter->op_offset += len;
|
|
sgiter->offset += len;
|
|
if (sgiter->offset == sg_dma_len(sgiter->sg)) {
|
|
if (sg_is_last(sgiter->sg))
|
|
return false;
|
|
sgiter->offset = 0;
|
|
sgiter->sg = sg_next(sgiter->sg);
|
|
}
|
|
|
|
if (sgiter->op_offset == iter->op_len)
|
|
return false;
|
|
|
|
return true;
|
|
}
|
|
|
|
void mv_cesa_dma_step(struct mv_cesa_req *dreq)
|
|
{
|
|
struct mv_cesa_engine *engine = dreq->engine;
|
|
|
|
writel_relaxed(0, engine->regs + CESA_SA_CFG);
|
|
|
|
mv_cesa_set_int_mask(engine, CESA_SA_INT_ACC0_IDMA_DONE);
|
|
writel_relaxed(CESA_TDMA_DST_BURST_128B | CESA_TDMA_SRC_BURST_128B |
|
|
CESA_TDMA_NO_BYTE_SWAP | CESA_TDMA_EN,
|
|
engine->regs + CESA_TDMA_CONTROL);
|
|
|
|
writel_relaxed(CESA_SA_CFG_ACT_CH0_IDMA | CESA_SA_CFG_MULTI_PKT |
|
|
CESA_SA_CFG_CH0_W_IDMA | CESA_SA_CFG_PARA_DIS,
|
|
engine->regs + CESA_SA_CFG);
|
|
writel_relaxed(dreq->chain.first->cur_dma,
|
|
engine->regs + CESA_TDMA_NEXT_ADDR);
|
|
BUG_ON(readl(engine->regs + CESA_SA_CMD) &
|
|
CESA_SA_CMD_EN_CESA_SA_ACCL0);
|
|
writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD);
|
|
}
|
|
|
|
void mv_cesa_dma_cleanup(struct mv_cesa_req *dreq)
|
|
{
|
|
struct mv_cesa_tdma_desc *tdma;
|
|
|
|
for (tdma = dreq->chain.first; tdma;) {
|
|
struct mv_cesa_tdma_desc *old_tdma = tdma;
|
|
u32 type = tdma->flags & CESA_TDMA_TYPE_MSK;
|
|
|
|
if (type == CESA_TDMA_OP)
|
|
dma_pool_free(cesa_dev->dma->op_pool, tdma->op,
|
|
le32_to_cpu(tdma->src));
|
|
|
|
tdma = tdma->next;
|
|
dma_pool_free(cesa_dev->dma->tdma_desc_pool, old_tdma,
|
|
old_tdma->cur_dma);
|
|
}
|
|
|
|
dreq->chain.first = NULL;
|
|
dreq->chain.last = NULL;
|
|
}
|
|
|
|
void mv_cesa_dma_prepare(struct mv_cesa_req *dreq,
|
|
struct mv_cesa_engine *engine)
|
|
{
|
|
struct mv_cesa_tdma_desc *tdma;
|
|
|
|
for (tdma = dreq->chain.first; tdma; tdma = tdma->next) {
|
|
if (tdma->flags & CESA_TDMA_DST_IN_SRAM)
|
|
tdma->dst = cpu_to_le32(tdma->dst + engine->sram_dma);
|
|
|
|
if (tdma->flags & CESA_TDMA_SRC_IN_SRAM)
|
|
tdma->src = cpu_to_le32(tdma->src + engine->sram_dma);
|
|
|
|
if ((tdma->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_OP)
|
|
mv_cesa_adjust_op(engine, tdma->op);
|
|
}
|
|
}
|
|
|
|
void mv_cesa_tdma_chain(struct mv_cesa_engine *engine,
|
|
struct mv_cesa_req *dreq)
|
|
{
|
|
if (engine->chain.first == NULL && engine->chain.last == NULL) {
|
|
engine->chain.first = dreq->chain.first;
|
|
engine->chain.last = dreq->chain.last;
|
|
} else {
|
|
struct mv_cesa_tdma_desc *last;
|
|
|
|
last = engine->chain.last;
|
|
last->next = dreq->chain.first;
|
|
engine->chain.last = dreq->chain.last;
|
|
|
|
/*
|
|
* Break the DMA chain if the CESA_TDMA_BREAK_CHAIN is set on
|
|
* the last element of the current chain, or if the request
|
|
* being queued needs the IV regs to be set before lauching
|
|
* the request.
|
|
*/
|
|
if (!(last->flags & CESA_TDMA_BREAK_CHAIN) &&
|
|
!(dreq->chain.first->flags & CESA_TDMA_SET_STATE))
|
|
last->next_dma = dreq->chain.first->cur_dma;
|
|
}
|
|
}
|
|
|
|
int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status)
|
|
{
|
|
struct crypto_async_request *req = NULL;
|
|
struct mv_cesa_tdma_desc *tdma = NULL, *next = NULL;
|
|
dma_addr_t tdma_cur;
|
|
int res = 0;
|
|
|
|
tdma_cur = readl(engine->regs + CESA_TDMA_CUR);
|
|
|
|
for (tdma = engine->chain.first; tdma; tdma = next) {
|
|
spin_lock_bh(&engine->lock);
|
|
next = tdma->next;
|
|
spin_unlock_bh(&engine->lock);
|
|
|
|
if (tdma->flags & CESA_TDMA_END_OF_REQ) {
|
|
struct crypto_async_request *backlog = NULL;
|
|
struct mv_cesa_ctx *ctx;
|
|
u32 current_status;
|
|
|
|
spin_lock_bh(&engine->lock);
|
|
/*
|
|
* if req is NULL, this means we're processing the
|
|
* request in engine->req.
|
|
*/
|
|
if (!req)
|
|
req = engine->req;
|
|
else
|
|
req = mv_cesa_dequeue_req_locked(engine,
|
|
&backlog);
|
|
|
|
/* Re-chaining to the next request */
|
|
engine->chain.first = tdma->next;
|
|
tdma->next = NULL;
|
|
|
|
/* If this is the last request, clear the chain */
|
|
if (engine->chain.first == NULL)
|
|
engine->chain.last = NULL;
|
|
spin_unlock_bh(&engine->lock);
|
|
|
|
ctx = crypto_tfm_ctx(req->tfm);
|
|
current_status = (tdma->cur_dma == tdma_cur) ?
|
|
status : CESA_SA_INT_ACC0_IDMA_DONE;
|
|
res = ctx->ops->process(req, current_status);
|
|
ctx->ops->complete(req);
|
|
|
|
if (res == 0)
|
|
mv_cesa_engine_enqueue_complete_request(engine,
|
|
req);
|
|
|
|
if (backlog)
|
|
backlog->complete(backlog, -EINPROGRESS);
|
|
}
|
|
|
|
if (res || tdma->cur_dma == tdma_cur)
|
|
break;
|
|
}
|
|
|
|
/* Save the last request in error to engine->req, so that the core
|
|
* knows which request was fautly */
|
|
if (res) {
|
|
spin_lock_bh(&engine->lock);
|
|
engine->req = req;
|
|
spin_unlock_bh(&engine->lock);
|
|
}
|
|
|
|
return res;
|
|
}
|
|
|
|
static struct mv_cesa_tdma_desc *
|
|
mv_cesa_dma_add_desc(struct mv_cesa_tdma_chain *chain, gfp_t flags)
|
|
{
|
|
struct mv_cesa_tdma_desc *new_tdma = NULL;
|
|
dma_addr_t dma_handle;
|
|
|
|
new_tdma = dma_pool_zalloc(cesa_dev->dma->tdma_desc_pool, flags,
|
|
&dma_handle);
|
|
if (!new_tdma)
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
new_tdma->cur_dma = dma_handle;
|
|
if (chain->last) {
|
|
chain->last->next_dma = cpu_to_le32(dma_handle);
|
|
chain->last->next = new_tdma;
|
|
} else {
|
|
chain->first = new_tdma;
|
|
}
|
|
|
|
chain->last = new_tdma;
|
|
|
|
return new_tdma;
|
|
}
|
|
|
|
int mv_cesa_dma_add_result_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src,
|
|
u32 size, u32 flags, gfp_t gfp_flags)
|
|
{
|
|
struct mv_cesa_tdma_desc *tdma, *op_desc;
|
|
|
|
tdma = mv_cesa_dma_add_desc(chain, gfp_flags);
|
|
if (IS_ERR(tdma))
|
|
return PTR_ERR(tdma);
|
|
|
|
/* We re-use an existing op_desc object to retrieve the context
|
|
* and result instead of allocating a new one.
|
|
* There is at least one object of this type in a CESA crypto
|
|
* req, just pick the first one in the chain.
|
|
*/
|
|
for (op_desc = chain->first; op_desc; op_desc = op_desc->next) {
|
|
u32 type = op_desc->flags & CESA_TDMA_TYPE_MSK;
|
|
|
|
if (type == CESA_TDMA_OP)
|
|
break;
|
|
}
|
|
|
|
if (!op_desc)
|
|
return -EIO;
|
|
|
|
tdma->byte_cnt = cpu_to_le32(size | BIT(31));
|
|
tdma->src = src;
|
|
tdma->dst = op_desc->src;
|
|
tdma->op = op_desc->op;
|
|
|
|
flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM);
|
|
tdma->flags = flags | CESA_TDMA_RESULT;
|
|
return 0;
|
|
}
|
|
|
|
struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain,
|
|
const struct mv_cesa_op_ctx *op_templ,
|
|
bool skip_ctx,
|
|
gfp_t flags)
|
|
{
|
|
struct mv_cesa_tdma_desc *tdma;
|
|
struct mv_cesa_op_ctx *op;
|
|
dma_addr_t dma_handle;
|
|
unsigned int size;
|
|
|
|
tdma = mv_cesa_dma_add_desc(chain, flags);
|
|
if (IS_ERR(tdma))
|
|
return ERR_CAST(tdma);
|
|
|
|
op = dma_pool_alloc(cesa_dev->dma->op_pool, flags, &dma_handle);
|
|
if (!op)
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
*op = *op_templ;
|
|
|
|
size = skip_ctx ? sizeof(op->desc) : sizeof(*op);
|
|
|
|
tdma = chain->last;
|
|
tdma->op = op;
|
|
tdma->byte_cnt = cpu_to_le32(size | BIT(31));
|
|
tdma->src = cpu_to_le32(dma_handle);
|
|
tdma->dst = CESA_SA_CFG_SRAM_OFFSET;
|
|
tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP;
|
|
|
|
return op;
|
|
}
|
|
|
|
int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain,
|
|
dma_addr_t dst, dma_addr_t src, u32 size,
|
|
u32 flags, gfp_t gfp_flags)
|
|
{
|
|
struct mv_cesa_tdma_desc *tdma;
|
|
|
|
tdma = mv_cesa_dma_add_desc(chain, gfp_flags);
|
|
if (IS_ERR(tdma))
|
|
return PTR_ERR(tdma);
|
|
|
|
tdma->byte_cnt = cpu_to_le32(size | BIT(31));
|
|
tdma->src = src;
|
|
tdma->dst = dst;
|
|
|
|
flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM);
|
|
tdma->flags = flags | CESA_TDMA_DATA;
|
|
|
|
return 0;
|
|
}
|
|
|
|
int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, gfp_t flags)
|
|
{
|
|
struct mv_cesa_tdma_desc *tdma;
|
|
|
|
tdma = mv_cesa_dma_add_desc(chain, flags);
|
|
if (IS_ERR(tdma))
|
|
return PTR_ERR(tdma);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags)
|
|
{
|
|
struct mv_cesa_tdma_desc *tdma;
|
|
|
|
tdma = mv_cesa_dma_add_desc(chain, flags);
|
|
if (IS_ERR(tdma))
|
|
return PTR_ERR(tdma);
|
|
|
|
tdma->byte_cnt = cpu_to_le32(BIT(31));
|
|
|
|
return 0;
|
|
}
|
|
|
|
int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain,
|
|
struct mv_cesa_dma_iter *dma_iter,
|
|
struct mv_cesa_sg_dma_iter *sgiter,
|
|
gfp_t gfp_flags)
|
|
{
|
|
u32 flags = sgiter->dir == DMA_TO_DEVICE ?
|
|
CESA_TDMA_DST_IN_SRAM : CESA_TDMA_SRC_IN_SRAM;
|
|
unsigned int len;
|
|
|
|
do {
|
|
dma_addr_t dst, src;
|
|
int ret;
|
|
|
|
len = mv_cesa_req_dma_iter_transfer_len(dma_iter, sgiter);
|
|
if (sgiter->dir == DMA_TO_DEVICE) {
|
|
dst = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset;
|
|
src = sg_dma_address(sgiter->sg) + sgiter->offset;
|
|
} else {
|
|
dst = sg_dma_address(sgiter->sg) + sgiter->offset;
|
|
src = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset;
|
|
}
|
|
|
|
ret = mv_cesa_dma_add_data_transfer(chain, dst, src, len,
|
|
flags, gfp_flags);
|
|
if (ret)
|
|
return ret;
|
|
|
|
} while (mv_cesa_req_dma_iter_next_transfer(dma_iter, sgiter, len));
|
|
|
|
return 0;
|
|
}
|