mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-29 04:09:39 +00:00
![Tero Kristo](/assets/img/avatar_default.png)
Currently omap-aes-gcm algorithms are using local implementation for crypto request queuing logic. Instead, implement this via usage of crypto engine which is used already for rest of the omap aes algorithms. This avoids some random conflicts / crashes also which can happen if both aes and aes-gcm are attempted to be used simultaneously. Signed-off-by: Tero Kristo <t-kristo@ti.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
390 lines
9.4 KiB
C
390 lines
9.4 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/*
|
|
* Cryptographic API.
|
|
*
|
|
* Support for OMAP AES GCM HW acceleration.
|
|
*
|
|
* Copyright (c) 2016 Texas Instruments Incorporated
|
|
*/
|
|
|
|
#include <linux/errno.h>
|
|
#include <linux/scatterlist.h>
|
|
#include <linux/dma-mapping.h>
|
|
#include <linux/dmaengine.h>
|
|
#include <linux/omap-dma.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/pm_runtime.h>
|
|
#include <crypto/aes.h>
|
|
#include <crypto/gcm.h>
|
|
#include <crypto/scatterwalk.h>
|
|
#include <crypto/skcipher.h>
|
|
#include <crypto/internal/aead.h>
|
|
|
|
#include "omap-crypto.h"
|
|
#include "omap-aes.h"
|
|
|
|
static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
|
|
struct aead_request *req);
|
|
|
|
static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)
|
|
{
|
|
struct aead_request *req = dd->aead_req;
|
|
|
|
dd->in_sg = NULL;
|
|
dd->out_sg = NULL;
|
|
|
|
crypto_finalize_aead_request(dd->engine, req, ret);
|
|
|
|
pm_runtime_mark_last_busy(dd->dev);
|
|
pm_runtime_put_autosuspend(dd->dev);
|
|
}
|
|
|
|
static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
|
|
{
|
|
u8 *tag;
|
|
int alen, clen, i, ret = 0, nsg;
|
|
struct omap_aes_reqctx *rctx;
|
|
|
|
alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);
|
|
clen = ALIGN(dd->total, AES_BLOCK_SIZE);
|
|
rctx = aead_request_ctx(dd->aead_req);
|
|
|
|
nsg = !!(dd->assoc_len && dd->total);
|
|
|
|
dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
|
|
DMA_FROM_DEVICE);
|
|
dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
|
|
dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
|
|
omap_aes_crypt_dma_stop(dd);
|
|
|
|
omap_crypto_cleanup(dd->out_sg, dd->orig_out,
|
|
dd->aead_req->assoclen, dd->total,
|
|
FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
|
|
|
|
if (dd->flags & FLAGS_ENCRYPT)
|
|
scatterwalk_map_and_copy(rctx->auth_tag,
|
|
dd->aead_req->dst,
|
|
dd->total + dd->aead_req->assoclen,
|
|
dd->authsize, 1);
|
|
|
|
omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,
|
|
FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);
|
|
|
|
omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,
|
|
FLAGS_IN_DATA_ST_SHIFT, dd->flags);
|
|
|
|
if (!(dd->flags & FLAGS_ENCRYPT)) {
|
|
tag = (u8 *)rctx->auth_tag;
|
|
for (i = 0; i < dd->authsize; i++) {
|
|
if (tag[i]) {
|
|
dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n");
|
|
ret = -EBADMSG;
|
|
}
|
|
}
|
|
}
|
|
|
|
omap_aes_gcm_finish_req(dd, ret);
|
|
}
|
|
|
|
static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
|
|
struct aead_request *req)
|
|
{
|
|
int alen, clen, cryptlen, assoclen, ret;
|
|
struct crypto_aead *aead = crypto_aead_reqtfm(req);
|
|
unsigned int authlen = crypto_aead_authsize(aead);
|
|
struct scatterlist *tmp, sg_arr[2];
|
|
int nsg;
|
|
u16 flags;
|
|
|
|
assoclen = req->assoclen;
|
|
cryptlen = req->cryptlen;
|
|
|
|
if (dd->flags & FLAGS_RFC4106_GCM)
|
|
assoclen -= 8;
|
|
|
|
if (!(dd->flags & FLAGS_ENCRYPT))
|
|
cryptlen -= authlen;
|
|
|
|
alen = ALIGN(assoclen, AES_BLOCK_SIZE);
|
|
clen = ALIGN(cryptlen, AES_BLOCK_SIZE);
|
|
|
|
nsg = !!(assoclen && cryptlen);
|
|
|
|
omap_aes_clear_copy_flags(dd);
|
|
|
|
sg_init_table(dd->in_sgl, nsg + 1);
|
|
if (assoclen) {
|
|
tmp = req->src;
|
|
ret = omap_crypto_align_sg(&tmp, assoclen,
|
|
AES_BLOCK_SIZE, dd->in_sgl,
|
|
OMAP_CRYPTO_COPY_DATA |
|
|
OMAP_CRYPTO_ZERO_BUF |
|
|
OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
|
|
FLAGS_ASSOC_DATA_ST_SHIFT,
|
|
&dd->flags);
|
|
if (ret)
|
|
return ret;
|
|
}
|
|
|
|
if (cryptlen) {
|
|
tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);
|
|
|
|
if (nsg)
|
|
sg_unmark_end(dd->in_sgl);
|
|
|
|
ret = omap_crypto_align_sg(&tmp, cryptlen,
|
|
AES_BLOCK_SIZE, &dd->in_sgl[nsg],
|
|
OMAP_CRYPTO_COPY_DATA |
|
|
OMAP_CRYPTO_ZERO_BUF |
|
|
OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
|
|
FLAGS_IN_DATA_ST_SHIFT,
|
|
&dd->flags);
|
|
if (ret)
|
|
return ret;
|
|
}
|
|
|
|
dd->in_sg = dd->in_sgl;
|
|
dd->total = cryptlen;
|
|
dd->assoc_len = assoclen;
|
|
dd->authsize = authlen;
|
|
|
|
dd->out_sg = req->dst;
|
|
dd->orig_out = req->dst;
|
|
|
|
dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen);
|
|
|
|
flags = 0;
|
|
if (req->src == req->dst || dd->out_sg == sg_arr)
|
|
flags |= OMAP_CRYPTO_FORCE_COPY;
|
|
|
|
if (cryptlen) {
|
|
ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
|
|
AES_BLOCK_SIZE, &dd->out_sgl,
|
|
flags,
|
|
FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
|
|
if (ret)
|
|
return ret;
|
|
}
|
|
|
|
dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
|
|
dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv)
|
|
{
|
|
struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
|
|
|
|
aes_encrypt(&ctx->actx, (u8 *)tag, (u8 *)iv);
|
|
return 0;
|
|
}
|
|
|
|
void omap_aes_gcm_dma_out_callback(void *data)
|
|
{
|
|
struct omap_aes_dev *dd = data;
|
|
struct omap_aes_reqctx *rctx;
|
|
int i, val;
|
|
u32 *auth_tag, tag[4];
|
|
|
|
if (!(dd->flags & FLAGS_ENCRYPT))
|
|
scatterwalk_map_and_copy(tag, dd->aead_req->src,
|
|
dd->total + dd->aead_req->assoclen,
|
|
dd->authsize, 0);
|
|
|
|
rctx = aead_request_ctx(dd->aead_req);
|
|
auth_tag = (u32 *)rctx->auth_tag;
|
|
for (i = 0; i < 4; i++) {
|
|
val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));
|
|
auth_tag[i] = val ^ auth_tag[i];
|
|
if (!(dd->flags & FLAGS_ENCRYPT))
|
|
auth_tag[i] = auth_tag[i] ^ tag[i];
|
|
}
|
|
|
|
omap_aes_gcm_done_task(dd);
|
|
}
|
|
|
|
static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
|
|
struct aead_request *req)
|
|
{
|
|
if (req)
|
|
return crypto_transfer_aead_request_to_engine(dd->engine, req);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int omap_aes_gcm_prepare_req(struct crypto_engine *engine, void *areq)
|
|
{
|
|
struct aead_request *req = container_of(areq, struct aead_request,
|
|
base);
|
|
struct omap_aes_reqctx *rctx = aead_request_ctx(req);
|
|
struct omap_aes_dev *dd = rctx->dd;
|
|
struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
|
|
int err;
|
|
|
|
dd->aead_req = req;
|
|
|
|
rctx->mode &= FLAGS_MODE_MASK;
|
|
dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
|
|
|
|
err = omap_aes_gcm_copy_buffers(dd, req);
|
|
if (err)
|
|
return err;
|
|
|
|
dd->ctx = &ctx->octx;
|
|
|
|
return omap_aes_write_ctrl(dd);
|
|
}
|
|
|
|
static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
|
|
{
|
|
struct omap_aes_reqctx *rctx = aead_request_ctx(req);
|
|
struct crypto_aead *aead = crypto_aead_reqtfm(req);
|
|
unsigned int authlen = crypto_aead_authsize(aead);
|
|
struct omap_aes_dev *dd;
|
|
__be32 counter = cpu_to_be32(1);
|
|
int err, assoclen;
|
|
|
|
memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
|
|
memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4);
|
|
|
|
err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv);
|
|
if (err)
|
|
return err;
|
|
|
|
if (mode & FLAGS_RFC4106_GCM)
|
|
assoclen = req->assoclen - 8;
|
|
else
|
|
assoclen = req->assoclen;
|
|
if (assoclen + req->cryptlen == 0) {
|
|
scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
|
|
1);
|
|
return 0;
|
|
}
|
|
|
|
dd = omap_aes_find_dev(rctx);
|
|
if (!dd)
|
|
return -ENODEV;
|
|
rctx->mode = mode;
|
|
|
|
return omap_aes_gcm_handle_queue(dd, req);
|
|
}
|
|
|
|
int omap_aes_gcm_encrypt(struct aead_request *req)
|
|
{
|
|
struct omap_aes_reqctx *rctx = aead_request_ctx(req);
|
|
|
|
memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
|
|
return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM);
|
|
}
|
|
|
|
int omap_aes_gcm_decrypt(struct aead_request *req)
|
|
{
|
|
struct omap_aes_reqctx *rctx = aead_request_ctx(req);
|
|
|
|
memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
|
|
return omap_aes_gcm_crypt(req, FLAGS_GCM);
|
|
}
|
|
|
|
int omap_aes_4106gcm_encrypt(struct aead_request *req)
|
|
{
|
|
struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
|
|
struct omap_aes_reqctx *rctx = aead_request_ctx(req);
|
|
|
|
memcpy(rctx->iv, ctx->octx.nonce, 4);
|
|
memcpy(rctx->iv + 4, req->iv, 8);
|
|
return crypto_ipsec_check_assoclen(req->assoclen) ?:
|
|
omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM |
|
|
FLAGS_RFC4106_GCM);
|
|
}
|
|
|
|
int omap_aes_4106gcm_decrypt(struct aead_request *req)
|
|
{
|
|
struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
|
|
struct omap_aes_reqctx *rctx = aead_request_ctx(req);
|
|
|
|
memcpy(rctx->iv, ctx->octx.nonce, 4);
|
|
memcpy(rctx->iv + 4, req->iv, 8);
|
|
return crypto_ipsec_check_assoclen(req->assoclen) ?:
|
|
omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM);
|
|
}
|
|
|
|
int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
|
|
unsigned int keylen)
|
|
{
|
|
struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
|
|
int ret;
|
|
|
|
ret = aes_expandkey(&ctx->actx, key, keylen);
|
|
if (ret)
|
|
return ret;
|
|
|
|
memcpy(ctx->octx.key, key, keylen);
|
|
ctx->octx.keylen = keylen;
|
|
|
|
return 0;
|
|
}
|
|
|
|
int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key,
|
|
unsigned int keylen)
|
|
{
|
|
struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
|
|
int ret;
|
|
|
|
if (keylen < 4)
|
|
return -EINVAL;
|
|
keylen -= 4;
|
|
|
|
ret = aes_expandkey(&ctx->actx, key, keylen);
|
|
if (ret)
|
|
return ret;
|
|
|
|
memcpy(ctx->octx.key, key, keylen);
|
|
memcpy(ctx->octx.nonce, key + keylen, 4);
|
|
ctx->octx.keylen = keylen;
|
|
|
|
return 0;
|
|
}
|
|
|
|
int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
|
|
{
|
|
return crypto_gcm_check_authsize(authsize);
|
|
}
|
|
|
|
int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent,
|
|
unsigned int authsize)
|
|
{
|
|
return crypto_rfc4106_check_authsize(authsize);
|
|
}
|
|
|
|
static int omap_aes_gcm_crypt_req(struct crypto_engine *engine, void *areq)
|
|
{
|
|
struct aead_request *req = container_of(areq, struct aead_request,
|
|
base);
|
|
struct omap_aes_reqctx *rctx = aead_request_ctx(req);
|
|
struct omap_aes_dev *dd = rctx->dd;
|
|
int ret = 0;
|
|
|
|
if (!dd)
|
|
return -ENODEV;
|
|
|
|
if (dd->in_sg_len)
|
|
ret = omap_aes_crypt_dma_start(dd);
|
|
else
|
|
omap_aes_gcm_dma_out_callback(dd);
|
|
|
|
return ret;
|
|
}
|
|
|
|
int omap_aes_gcm_cra_init(struct crypto_aead *tfm)
|
|
{
|
|
struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
|
|
|
|
ctx->enginectx.op.prepare_request = omap_aes_gcm_prepare_req;
|
|
ctx->enginectx.op.unprepare_request = NULL;
|
|
ctx->enginectx.op.do_one_request = omap_aes_gcm_crypt_req;
|
|
|
|
crypto_aead_set_reqsize(tfm, sizeof(struct omap_aes_reqctx));
|
|
|
|
return 0;
|
|
}
|