2019-11-09 17:09:48 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
|
|
|
/*
|
|
|
|
* Crypto acceleration support for Rockchip RK3288
|
|
|
|
*
|
|
|
|
* Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd
|
|
|
|
*
|
|
|
|
* Author: Zain Wang <zain.wang@rock-chips.com>
|
|
|
|
*
|
|
|
|
* Some ideas are from marvell-cesa.c and s5p-sss.c driver.
|
|
|
|
*/
|
2023-08-13 06:55:10 +00:00
|
|
|
|
|
|
|
#include <crypto/engine.h>
|
|
|
|
#include <crypto/internal/skcipher.h>
|
2022-09-27 07:54:48 +00:00
|
|
|
#include <crypto/scatterwalk.h>
|
2023-08-13 06:55:10 +00:00
|
|
|
#include <linux/device.h>
|
|
|
|
#include <linux/err.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/string.h>
|
2019-11-09 17:09:48 +00:00
|
|
|
#include "rk3288_crypto.h"
|
|
|
|
|
|
|
|
#define RK_CRYPTO_DEC BIT(0)
|
|
|
|
|
2022-09-27 07:54:44 +00:00
|
|
|
static int rk_cipher_need_fallback(struct skcipher_request *req)
|
|
|
|
{
|
|
|
|
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
|
2022-09-27 07:54:50 +00:00
|
|
|
struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
|
2023-08-13 06:55:10 +00:00
|
|
|
struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base);
|
2022-09-27 07:54:44 +00:00
|
|
|
struct scatterlist *sgs, *sgd;
|
|
|
|
unsigned int stodo, dtodo, len;
|
2022-09-27 07:55:11 +00:00
|
|
|
unsigned int bs = crypto_skcipher_blocksize(tfm);
|
2022-09-27 07:54:44 +00:00
|
|
|
|
|
|
|
if (!req->cryptlen)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
len = req->cryptlen;
|
|
|
|
sgs = req->src;
|
|
|
|
sgd = req->dst;
|
|
|
|
while (sgs && sgd) {
|
|
|
|
if (!IS_ALIGNED(sgs->offset, sizeof(u32))) {
|
2022-09-27 07:54:50 +00:00
|
|
|
algt->stat_fb_align++;
|
2022-09-27 07:54:44 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
if (!IS_ALIGNED(sgd->offset, sizeof(u32))) {
|
2022-09-27 07:54:50 +00:00
|
|
|
algt->stat_fb_align++;
|
2022-09-27 07:54:44 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
stodo = min(len, sgs->length);
|
|
|
|
if (stodo % bs) {
|
2022-09-27 07:54:50 +00:00
|
|
|
algt->stat_fb_len++;
|
2022-09-27 07:54:44 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
dtodo = min(len, sgd->length);
|
|
|
|
if (dtodo % bs) {
|
2022-09-27 07:54:50 +00:00
|
|
|
algt->stat_fb_len++;
|
2022-09-27 07:54:44 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
if (stodo != dtodo) {
|
2022-09-27 07:54:50 +00:00
|
|
|
algt->stat_fb_sgdiff++;
|
2022-09-27 07:54:44 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
len -= stodo;
|
|
|
|
sgs = sg_next(sgs);
|
|
|
|
sgd = sg_next(sgd);
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_cipher_fallback(struct skcipher_request *areq)
|
|
|
|
{
|
|
|
|
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
|
|
|
|
struct rk_cipher_ctx *op = crypto_skcipher_ctx(tfm);
|
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(areq);
|
2022-09-27 07:54:50 +00:00
|
|
|
struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
|
2023-08-13 06:55:10 +00:00
|
|
|
struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base);
|
2022-09-27 07:54:44 +00:00
|
|
|
int err;
|
|
|
|
|
2022-09-27 07:54:50 +00:00
|
|
|
algt->stat_fb++;
|
|
|
|
|
2022-09-27 07:54:44 +00:00
|
|
|
skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);
|
|
|
|
skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,
|
|
|
|
areq->base.complete, areq->base.data);
|
|
|
|
skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,
|
|
|
|
areq->cryptlen, areq->iv);
|
|
|
|
if (rctx->mode & RK_CRYPTO_DEC)
|
|
|
|
err = crypto_skcipher_decrypt(&rctx->fallback_req);
|
|
|
|
else
|
|
|
|
err = crypto_skcipher_encrypt(&rctx->fallback_req);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2022-09-27 07:54:59 +00:00
|
|
|
static int rk_cipher_handle_req(struct skcipher_request *req)
|
2019-11-09 17:09:48 +00:00
|
|
|
{
|
2022-09-27 07:55:07 +00:00
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
|
2022-09-27 07:55:11 +00:00
|
|
|
struct rk_crypto_info *rkc;
|
|
|
|
struct crypto_engine *engine;
|
2022-09-27 07:54:48 +00:00
|
|
|
|
2022-09-27 07:54:44 +00:00
|
|
|
if (rk_cipher_need_fallback(req))
|
|
|
|
return rk_cipher_fallback(req);
|
|
|
|
|
2022-09-27 07:55:11 +00:00
|
|
|
rkc = get_rk_crypto();
|
|
|
|
|
|
|
|
engine = rkc->engine;
|
2022-09-27 07:55:07 +00:00
|
|
|
rctx->dev = rkc;
|
|
|
|
|
2022-09-27 07:54:48 +00:00
|
|
|
return crypto_transfer_skcipher_request_to_engine(engine, req);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_aes_setkey(struct crypto_skcipher *cipher,
|
|
|
|
const u8 *key, unsigned int keylen)
|
|
|
|
{
|
|
|
|
struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
|
|
|
|
struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
|
|
|
|
|
|
|
|
if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
|
crypto: remove CRYPTO_TFM_RES_BAD_KEY_LEN
The CRYPTO_TFM_RES_BAD_KEY_LEN flag was apparently meant as a way to
make the ->setkey() functions provide more information about errors.
However, no one actually checks for this flag, which makes it pointless.
Also, many algorithms fail to set this flag when given a bad length key.
Reviewing just the generic implementations, this is the case for
aes-fixed-time, cbcmac, echainiv, nhpoly1305, pcrypt, rfc3686, rfc4309,
rfc7539, rfc7539esp, salsa20, seqiv, and xcbc. But there are probably
many more in arch/*/crypto/ and drivers/crypto/.
Some algorithms can even set this flag when the key is the correct
length. For example, authenc and authencesn set it when the key payload
is malformed in any way (not just a bad length), the atmel-sha and ccree
drivers can set it if a memory allocation fails, and the chelsio driver
sets it for bad auth tag lengths, not just bad key lengths.
So even if someone actually wanted to start checking this flag (which
seems unlikely, since it's been unused for a long time), there would be
a lot of work needed to get it working correctly. But it would probably
be much better to go back to the drawing board and just define different
return values, like -EINVAL if the key is invalid for the algorithm vs.
-EKEYREJECTED if the key was rejected by a policy like "no weak keys".
That would be much simpler, less error-prone, and easier to test.
So just remove this flag.
Signed-off-by: Eric Biggers <ebiggers@google.com>
Reviewed-by: Horia Geantă <horia.geanta@nxp.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2019-12-31 03:19:36 +00:00
|
|
|
keylen != AES_KEYSIZE_256)
|
2019-11-09 17:09:48 +00:00
|
|
|
return -EINVAL;
|
|
|
|
ctx->keylen = keylen;
|
2022-09-27 07:54:46 +00:00
|
|
|
memcpy(ctx->key, key, keylen);
|
2022-09-27 07:54:44 +00:00
|
|
|
|
|
|
|
return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_des_setkey(struct crypto_skcipher *cipher,
|
|
|
|
const u8 *key, unsigned int keylen)
|
|
|
|
{
|
|
|
|
struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = verify_skcipher_des_key(cipher, key);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
ctx->keylen = keylen;
|
2022-09-27 07:54:46 +00:00
|
|
|
memcpy(ctx->key, key, keylen);
|
2022-09-27 07:54:44 +00:00
|
|
|
|
|
|
|
return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_tdes_setkey(struct crypto_skcipher *cipher,
|
|
|
|
const u8 *key, unsigned int keylen)
|
|
|
|
{
|
|
|
|
struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = verify_skcipher_des3_key(cipher, key);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
ctx->keylen = keylen;
|
2022-09-27 07:54:46 +00:00
|
|
|
memcpy(ctx->key, key, keylen);
|
|
|
|
|
2022-09-27 07:54:44 +00:00
|
|
|
return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_aes_ecb_encrypt(struct skcipher_request *req)
|
|
|
|
{
|
2022-09-27 07:54:43 +00:00
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode = RK_CRYPTO_AES_ECB_MODE;
|
2022-09-27 07:54:59 +00:00
|
|
|
return rk_cipher_handle_req(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_aes_ecb_decrypt(struct skcipher_request *req)
|
|
|
|
{
|
2022-09-27 07:54:43 +00:00
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC;
|
2022-09-27 07:54:59 +00:00
|
|
|
return rk_cipher_handle_req(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_aes_cbc_encrypt(struct skcipher_request *req)
|
|
|
|
{
|
2022-09-27 07:54:43 +00:00
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode = RK_CRYPTO_AES_CBC_MODE;
|
2022-09-27 07:54:59 +00:00
|
|
|
return rk_cipher_handle_req(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_aes_cbc_decrypt(struct skcipher_request *req)
|
|
|
|
{
|
2022-09-27 07:54:43 +00:00
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC;
|
2022-09-27 07:54:59 +00:00
|
|
|
return rk_cipher_handle_req(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_des_ecb_encrypt(struct skcipher_request *req)
|
|
|
|
{
|
2022-09-27 07:54:43 +00:00
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode = 0;
|
2022-09-27 07:54:59 +00:00
|
|
|
return rk_cipher_handle_req(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_des_ecb_decrypt(struct skcipher_request *req)
|
|
|
|
{
|
2022-09-27 07:54:43 +00:00
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode = RK_CRYPTO_DEC;
|
2022-09-27 07:54:59 +00:00
|
|
|
return rk_cipher_handle_req(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_des_cbc_encrypt(struct skcipher_request *req)
|
|
|
|
{
|
2022-09-27 07:54:43 +00:00
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC;
|
2022-09-27 07:54:59 +00:00
|
|
|
return rk_cipher_handle_req(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_des_cbc_decrypt(struct skcipher_request *req)
|
|
|
|
{
|
2022-09-27 07:54:43 +00:00
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC;
|
2022-09-27 07:54:59 +00:00
|
|
|
return rk_cipher_handle_req(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_des3_ede_ecb_encrypt(struct skcipher_request *req)
|
|
|
|
{
|
2022-09-27 07:54:43 +00:00
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode = RK_CRYPTO_TDES_SELECT;
|
2022-09-27 07:54:59 +00:00
|
|
|
return rk_cipher_handle_req(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_des3_ede_ecb_decrypt(struct skcipher_request *req)
|
|
|
|
{
|
2022-09-27 07:54:43 +00:00
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC;
|
2022-09-27 07:54:59 +00:00
|
|
|
return rk_cipher_handle_req(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_des3_ede_cbc_encrypt(struct skcipher_request *req)
|
|
|
|
{
|
2022-09-27 07:54:43 +00:00
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC;
|
2022-09-27 07:54:59 +00:00
|
|
|
return rk_cipher_handle_req(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int rk_des3_ede_cbc_decrypt(struct skcipher_request *req)
|
|
|
|
{
|
2022-09-27 07:54:43 +00:00
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC |
|
2019-11-09 17:09:48 +00:00
|
|
|
RK_CRYPTO_DEC;
|
2022-09-27 07:54:59 +00:00
|
|
|
return rk_cipher_handle_req(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
2022-09-27 07:54:58 +00:00
|
|
|
static void rk_cipher_hw_init(struct rk_crypto_info *dev, struct skcipher_request *req)
|
2019-11-09 17:09:48 +00:00
|
|
|
{
|
|
|
|
struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
|
|
|
|
struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
|
2022-09-27 07:54:43 +00:00
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);
|
2019-11-09 17:09:48 +00:00
|
|
|
struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
|
2022-09-27 07:54:48 +00:00
|
|
|
u32 block, conf_reg = 0;
|
2019-11-09 17:09:48 +00:00
|
|
|
|
|
|
|
block = crypto_tfm_alg_blocksize(tfm);
|
|
|
|
|
|
|
|
if (block == DES_BLOCK_SIZE) {
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode |= RK_CRYPTO_TDES_FIFO_MODE |
|
2019-11-09 17:09:48 +00:00
|
|
|
RK_CRYPTO_TDES_BYTESWAP_KEY |
|
|
|
|
RK_CRYPTO_TDES_BYTESWAP_IV;
|
2022-09-27 07:54:43 +00:00
|
|
|
CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, rctx->mode);
|
2022-09-27 07:55:01 +00:00
|
|
|
memcpy_toio(dev->reg + RK_CRYPTO_TDES_KEY1_0, ctx->key, ctx->keylen);
|
2019-11-09 17:09:48 +00:00
|
|
|
conf_reg = RK_CRYPTO_DESSEL;
|
|
|
|
} else {
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode |= RK_CRYPTO_AES_FIFO_MODE |
|
2019-11-09 17:09:48 +00:00
|
|
|
RK_CRYPTO_AES_KEY_CHANGE |
|
|
|
|
RK_CRYPTO_AES_BYTESWAP_KEY |
|
|
|
|
RK_CRYPTO_AES_BYTESWAP_IV;
|
|
|
|
if (ctx->keylen == AES_KEYSIZE_192)
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode |= RK_CRYPTO_AES_192BIT_key;
|
2019-11-09 17:09:48 +00:00
|
|
|
else if (ctx->keylen == AES_KEYSIZE_256)
|
2022-09-27 07:54:43 +00:00
|
|
|
rctx->mode |= RK_CRYPTO_AES_256BIT_key;
|
|
|
|
CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, rctx->mode);
|
2022-09-27 07:55:01 +00:00
|
|
|
memcpy_toio(dev->reg + RK_CRYPTO_AES_KEY_0, ctx->key, ctx->keylen);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
conf_reg |= RK_CRYPTO_BYTESWAP_BTFIFO |
|
|
|
|
RK_CRYPTO_BYTESWAP_BRFIFO;
|
|
|
|
CRYPTO_WRITE(dev, RK_CRYPTO_CONF, conf_reg);
|
|
|
|
CRYPTO_WRITE(dev, RK_CRYPTO_INTENA,
|
|
|
|
RK_CRYPTO_BCDMA_ERR_ENA | RK_CRYPTO_BCDMA_DONE_ENA);
|
|
|
|
}
|
|
|
|
|
2022-09-27 07:54:48 +00:00
|
|
|
static void crypto_dma_start(struct rk_crypto_info *dev,
|
|
|
|
struct scatterlist *sgs,
|
|
|
|
struct scatterlist *sgd, unsigned int todo)
|
2019-11-09 17:09:48 +00:00
|
|
|
{
|
2022-09-27 07:54:48 +00:00
|
|
|
CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAS, sg_dma_address(sgs));
|
|
|
|
CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAL, todo);
|
|
|
|
CRYPTO_WRITE(dev, RK_CRYPTO_BTDMAS, sg_dma_address(sgd));
|
2019-11-09 17:09:48 +00:00
|
|
|
CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, RK_CRYPTO_BLOCK_START |
|
|
|
|
_SBF(RK_CRYPTO_BLOCK_START, 16));
|
|
|
|
}
|
|
|
|
|
2022-09-27 07:54:48 +00:00
|
|
|
static int rk_cipher_run(struct crypto_engine *engine, void *async_req)
|
2019-11-09 17:09:48 +00:00
|
|
|
{
|
2022-09-27 07:54:48 +00:00
|
|
|
struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base);
|
|
|
|
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
|
|
|
|
struct rk_cipher_rctx *rctx = skcipher_request_ctx(areq);
|
|
|
|
struct scatterlist *sgs, *sgd;
|
2019-11-09 17:09:48 +00:00
|
|
|
int err = 0;
|
2022-09-27 07:54:48 +00:00
|
|
|
int ivsize = crypto_skcipher_ivsize(tfm);
|
|
|
|
int offset;
|
|
|
|
u8 iv[AES_BLOCK_SIZE];
|
|
|
|
u8 biv[AES_BLOCK_SIZE];
|
|
|
|
u8 *ivtouse = areq->iv;
|
|
|
|
unsigned int len = areq->cryptlen;
|
|
|
|
unsigned int todo;
|
2022-09-27 07:54:50 +00:00
|
|
|
struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
|
2023-08-13 06:55:10 +00:00
|
|
|
struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base);
|
2022-09-27 07:55:07 +00:00
|
|
|
struct rk_crypto_info *rkc = rctx->dev;
|
2022-09-27 07:54:50 +00:00
|
|
|
|
2022-09-27 07:55:11 +00:00
|
|
|
err = pm_runtime_resume_and_get(rkc->dev);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
2022-09-27 07:54:50 +00:00
|
|
|
algt->stat_req++;
|
2022-09-27 07:55:11 +00:00
|
|
|
rkc->nreq++;
|
2022-09-27 07:54:48 +00:00
|
|
|
|
|
|
|
ivsize = crypto_skcipher_ivsize(tfm);
|
|
|
|
if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) {
|
|
|
|
if (rctx->mode & RK_CRYPTO_DEC) {
|
|
|
|
offset = areq->cryptlen - ivsize;
|
|
|
|
scatterwalk_map_and_copy(rctx->backup_iv, areq->src,
|
|
|
|
offset, ivsize, 0);
|
|
|
|
}
|
|
|
|
}
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:48 +00:00
|
|
|
sgs = areq->src;
|
|
|
|
sgd = areq->dst;
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:48 +00:00
|
|
|
while (sgs && sgd && len) {
|
|
|
|
if (!sgs->length) {
|
|
|
|
sgs = sg_next(sgs);
|
|
|
|
sgd = sg_next(sgd);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (rctx->mode & RK_CRYPTO_DEC) {
|
|
|
|
/* we backup last block of source to be used as IV at next step */
|
|
|
|
offset = sgs->length - ivsize;
|
|
|
|
scatterwalk_map_and_copy(biv, sgs, offset, ivsize, 0);
|
|
|
|
}
|
|
|
|
if (sgs == sgd) {
|
2022-09-27 07:55:00 +00:00
|
|
|
err = dma_map_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL);
|
2022-09-27 07:54:48 +00:00
|
|
|
if (err <= 0) {
|
|
|
|
err = -EINVAL;
|
|
|
|
goto theend_iv;
|
|
|
|
}
|
|
|
|
} else {
|
2022-09-27 07:55:00 +00:00
|
|
|
err = dma_map_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE);
|
2022-09-27 07:54:48 +00:00
|
|
|
if (err <= 0) {
|
|
|
|
err = -EINVAL;
|
|
|
|
goto theend_iv;
|
|
|
|
}
|
2022-09-27 07:55:00 +00:00
|
|
|
err = dma_map_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE);
|
2022-09-27 07:54:48 +00:00
|
|
|
if (err <= 0) {
|
|
|
|
err = -EINVAL;
|
|
|
|
goto theend_sgs;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
err = 0;
|
2022-09-27 07:55:00 +00:00
|
|
|
rk_cipher_hw_init(rkc, areq);
|
2022-09-27 07:54:48 +00:00
|
|
|
if (ivsize) {
|
|
|
|
if (ivsize == DES_BLOCK_SIZE)
|
2022-09-27 07:55:00 +00:00
|
|
|
memcpy_toio(rkc->reg + RK_CRYPTO_TDES_IV_0, ivtouse, ivsize);
|
2022-09-27 07:54:48 +00:00
|
|
|
else
|
2022-09-27 07:55:00 +00:00
|
|
|
memcpy_toio(rkc->reg + RK_CRYPTO_AES_IV_0, ivtouse, ivsize);
|
2022-09-27 07:54:48 +00:00
|
|
|
}
|
2022-09-27 07:55:00 +00:00
|
|
|
reinit_completion(&rkc->complete);
|
|
|
|
rkc->status = 0;
|
2022-09-27 07:54:48 +00:00
|
|
|
|
|
|
|
todo = min(sg_dma_len(sgs), len);
|
|
|
|
len -= todo;
|
2022-09-27 07:55:00 +00:00
|
|
|
crypto_dma_start(rkc, sgs, sgd, todo / 4);
|
|
|
|
wait_for_completion_interruptible_timeout(&rkc->complete,
|
2022-09-27 07:54:48 +00:00
|
|
|
msecs_to_jiffies(2000));
|
2022-09-27 07:55:00 +00:00
|
|
|
if (!rkc->status) {
|
|
|
|
dev_err(rkc->dev, "DMA timeout\n");
|
2022-09-27 07:54:48 +00:00
|
|
|
err = -EFAULT;
|
|
|
|
goto theend;
|
|
|
|
}
|
|
|
|
if (sgs == sgd) {
|
2022-09-27 07:55:00 +00:00
|
|
|
dma_unmap_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL);
|
2022-09-27 07:54:48 +00:00
|
|
|
} else {
|
2022-09-27 07:55:00 +00:00
|
|
|
dma_unmap_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE);
|
|
|
|
dma_unmap_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE);
|
2022-09-27 07:54:48 +00:00
|
|
|
}
|
|
|
|
if (rctx->mode & RK_CRYPTO_DEC) {
|
|
|
|
memcpy(iv, biv, ivsize);
|
|
|
|
ivtouse = iv;
|
|
|
|
} else {
|
|
|
|
offset = sgd->length - ivsize;
|
|
|
|
scatterwalk_map_and_copy(iv, sgd, offset, ivsize, 0);
|
|
|
|
ivtouse = iv;
|
|
|
|
}
|
|
|
|
sgs = sg_next(sgs);
|
|
|
|
sgd = sg_next(sgd);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
2022-09-27 07:54:48 +00:00
|
|
|
if (areq->iv && ivsize > 0) {
|
|
|
|
offset = areq->cryptlen - ivsize;
|
|
|
|
if (rctx->mode & RK_CRYPTO_DEC) {
|
|
|
|
memcpy(areq->iv, rctx->backup_iv, ivsize);
|
|
|
|
memzero_explicit(rctx->backup_iv, ivsize);
|
|
|
|
} else {
|
|
|
|
scatterwalk_map_and_copy(areq->iv, areq->dst, offset,
|
|
|
|
ivsize, 0);
|
|
|
|
}
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
2022-09-27 07:54:48 +00:00
|
|
|
theend:
|
2022-09-27 07:55:11 +00:00
|
|
|
pm_runtime_put_autosuspend(rkc->dev);
|
|
|
|
|
2022-09-27 07:54:48 +00:00
|
|
|
local_bh_disable();
|
|
|
|
crypto_finalize_skcipher_request(engine, areq, err);
|
|
|
|
local_bh_enable();
|
|
|
|
return 0;
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:48 +00:00
|
|
|
theend_sgs:
|
|
|
|
if (sgs == sgd) {
|
2022-09-27 07:55:00 +00:00
|
|
|
dma_unmap_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL);
|
2019-11-09 17:09:48 +00:00
|
|
|
} else {
|
2022-09-27 07:55:00 +00:00
|
|
|
dma_unmap_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE);
|
|
|
|
dma_unmap_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
2022-09-27 07:54:48 +00:00
|
|
|
theend_iv:
|
2019-11-09 17:09:48 +00:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2022-09-27 07:54:58 +00:00
|
|
|
static int rk_cipher_tfm_init(struct crypto_skcipher *tfm)
|
2019-11-09 17:09:48 +00:00
|
|
|
{
|
|
|
|
struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);
|
2022-09-27 07:54:44 +00:00
|
|
|
const char *name = crypto_tfm_alg_name(&tfm->base);
|
2022-09-27 07:55:11 +00:00
|
|
|
struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
|
2023-08-13 06:55:10 +00:00
|
|
|
struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base);
|
2019-11-09 17:09:48 +00:00
|
|
|
|
2022-09-27 07:54:44 +00:00
|
|
|
ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
|
|
|
|
if (IS_ERR(ctx->fallback_tfm)) {
|
2022-09-27 07:55:11 +00:00
|
|
|
dev_err(algt->dev->dev, "ERROR: Cannot allocate fallback for %s %ld\n",
|
2022-09-27 07:54:44 +00:00
|
|
|
name, PTR_ERR(ctx->fallback_tfm));
|
|
|
|
return PTR_ERR(ctx->fallback_tfm);
|
|
|
|
}
|
|
|
|
|
|
|
|
tfm->reqsize = sizeof(struct rk_cipher_rctx) +
|
|
|
|
crypto_skcipher_reqsize(ctx->fallback_tfm);
|
|
|
|
|
2022-09-27 07:54:41 +00:00
|
|
|
return 0;
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
2022-09-27 07:54:58 +00:00
|
|
|
static void rk_cipher_tfm_exit(struct crypto_skcipher *tfm)
|
2019-11-09 17:09:48 +00:00
|
|
|
{
|
|
|
|
struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);
|
|
|
|
|
2022-09-27 07:54:46 +00:00
|
|
|
memzero_explicit(ctx->key, ctx->keylen);
|
2022-09-27 07:54:44 +00:00
|
|
|
crypto_free_skcipher(ctx->fallback_tfm);
|
2019-11-09 17:09:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
struct rk_crypto_tmp rk_ecb_aes_alg = {
|
2022-09-27 07:54:49 +00:00
|
|
|
.type = CRYPTO_ALG_TYPE_SKCIPHER,
|
2023-08-13 06:55:10 +00:00
|
|
|
.alg.skcipher.base = {
|
2019-11-09 17:09:48 +00:00
|
|
|
.base.cra_name = "ecb(aes)",
|
|
|
|
.base.cra_driver_name = "ecb-aes-rk",
|
|
|
|
.base.cra_priority = 300,
|
2022-09-27 07:54:44 +00:00
|
|
|
.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
|
2019-11-09 17:09:48 +00:00
|
|
|
.base.cra_blocksize = AES_BLOCK_SIZE,
|
|
|
|
.base.cra_ctxsize = sizeof(struct rk_cipher_ctx),
|
|
|
|
.base.cra_alignmask = 0x0f,
|
|
|
|
.base.cra_module = THIS_MODULE,
|
|
|
|
|
2022-09-27 07:54:58 +00:00
|
|
|
.init = rk_cipher_tfm_init,
|
|
|
|
.exit = rk_cipher_tfm_exit,
|
2019-11-09 17:09:48 +00:00
|
|
|
.min_keysize = AES_MIN_KEY_SIZE,
|
|
|
|
.max_keysize = AES_MAX_KEY_SIZE,
|
|
|
|
.setkey = rk_aes_setkey,
|
|
|
|
.encrypt = rk_aes_ecb_encrypt,
|
|
|
|
.decrypt = rk_aes_ecb_decrypt,
|
2023-08-13 06:55:10 +00:00
|
|
|
},
|
|
|
|
.alg.skcipher.op = {
|
|
|
|
.do_one_request = rk_cipher_run,
|
|
|
|
},
|
2019-11-09 17:09:48 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
struct rk_crypto_tmp rk_cbc_aes_alg = {
|
2022-09-27 07:54:49 +00:00
|
|
|
.type = CRYPTO_ALG_TYPE_SKCIPHER,
|
2023-08-13 06:55:10 +00:00
|
|
|
.alg.skcipher.base = {
|
2019-11-09 17:09:48 +00:00
|
|
|
.base.cra_name = "cbc(aes)",
|
|
|
|
.base.cra_driver_name = "cbc-aes-rk",
|
|
|
|
.base.cra_priority = 300,
|
2022-09-27 07:54:44 +00:00
|
|
|
.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
|
2019-11-09 17:09:48 +00:00
|
|
|
.base.cra_blocksize = AES_BLOCK_SIZE,
|
|
|
|
.base.cra_ctxsize = sizeof(struct rk_cipher_ctx),
|
|
|
|
.base.cra_alignmask = 0x0f,
|
|
|
|
.base.cra_module = THIS_MODULE,
|
|
|
|
|
2022-09-27 07:54:58 +00:00
|
|
|
.init = rk_cipher_tfm_init,
|
|
|
|
.exit = rk_cipher_tfm_exit,
|
2019-11-09 17:09:48 +00:00
|
|
|
.min_keysize = AES_MIN_KEY_SIZE,
|
|
|
|
.max_keysize = AES_MAX_KEY_SIZE,
|
|
|
|
.ivsize = AES_BLOCK_SIZE,
|
|
|
|
.setkey = rk_aes_setkey,
|
|
|
|
.encrypt = rk_aes_cbc_encrypt,
|
|
|
|
.decrypt = rk_aes_cbc_decrypt,
|
2023-08-13 06:55:10 +00:00
|
|
|
},
|
|
|
|
.alg.skcipher.op = {
|
|
|
|
.do_one_request = rk_cipher_run,
|
|
|
|
},
|
2019-11-09 17:09:48 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
struct rk_crypto_tmp rk_ecb_des_alg = {
|
2022-09-27 07:54:49 +00:00
|
|
|
.type = CRYPTO_ALG_TYPE_SKCIPHER,
|
2023-08-13 06:55:10 +00:00
|
|
|
.alg.skcipher.base = {
|
2019-11-09 17:09:48 +00:00
|
|
|
.base.cra_name = "ecb(des)",
|
|
|
|
.base.cra_driver_name = "ecb-des-rk",
|
|
|
|
.base.cra_priority = 300,
|
2022-09-27 07:54:44 +00:00
|
|
|
.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
|
2019-11-09 17:09:48 +00:00
|
|
|
.base.cra_blocksize = DES_BLOCK_SIZE,
|
|
|
|
.base.cra_ctxsize = sizeof(struct rk_cipher_ctx),
|
|
|
|
.base.cra_alignmask = 0x07,
|
|
|
|
.base.cra_module = THIS_MODULE,
|
|
|
|
|
2022-09-27 07:54:58 +00:00
|
|
|
.init = rk_cipher_tfm_init,
|
|
|
|
.exit = rk_cipher_tfm_exit,
|
2019-11-09 17:09:48 +00:00
|
|
|
.min_keysize = DES_KEY_SIZE,
|
|
|
|
.max_keysize = DES_KEY_SIZE,
|
|
|
|
.setkey = rk_des_setkey,
|
|
|
|
.encrypt = rk_des_ecb_encrypt,
|
|
|
|
.decrypt = rk_des_ecb_decrypt,
|
2023-08-13 06:55:10 +00:00
|
|
|
},
|
|
|
|
.alg.skcipher.op = {
|
|
|
|
.do_one_request = rk_cipher_run,
|
|
|
|
},
|
2019-11-09 17:09:48 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
struct rk_crypto_tmp rk_cbc_des_alg = {
|
2022-09-27 07:54:49 +00:00
|
|
|
.type = CRYPTO_ALG_TYPE_SKCIPHER,
|
2023-08-13 06:55:10 +00:00
|
|
|
.alg.skcipher.base = {
|
2019-11-09 17:09:48 +00:00
|
|
|
.base.cra_name = "cbc(des)",
|
|
|
|
.base.cra_driver_name = "cbc-des-rk",
|
|
|
|
.base.cra_priority = 300,
|
2022-09-27 07:54:44 +00:00
|
|
|
.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
|
2019-11-09 17:09:48 +00:00
|
|
|
.base.cra_blocksize = DES_BLOCK_SIZE,
|
|
|
|
.base.cra_ctxsize = sizeof(struct rk_cipher_ctx),
|
|
|
|
.base.cra_alignmask = 0x07,
|
|
|
|
.base.cra_module = THIS_MODULE,
|
|
|
|
|
2022-09-27 07:54:58 +00:00
|
|
|
.init = rk_cipher_tfm_init,
|
|
|
|
.exit = rk_cipher_tfm_exit,
|
2019-11-09 17:09:48 +00:00
|
|
|
.min_keysize = DES_KEY_SIZE,
|
|
|
|
.max_keysize = DES_KEY_SIZE,
|
|
|
|
.ivsize = DES_BLOCK_SIZE,
|
|
|
|
.setkey = rk_des_setkey,
|
|
|
|
.encrypt = rk_des_cbc_encrypt,
|
|
|
|
.decrypt = rk_des_cbc_decrypt,
|
2023-08-13 06:55:10 +00:00
|
|
|
},
|
|
|
|
.alg.skcipher.op = {
|
|
|
|
.do_one_request = rk_cipher_run,
|
|
|
|
},
|
2019-11-09 17:09:48 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
struct rk_crypto_tmp rk_ecb_des3_ede_alg = {
|
2022-09-27 07:54:49 +00:00
|
|
|
.type = CRYPTO_ALG_TYPE_SKCIPHER,
|
2023-08-13 06:55:10 +00:00
|
|
|
.alg.skcipher.base = {
|
2019-11-09 17:09:48 +00:00
|
|
|
.base.cra_name = "ecb(des3_ede)",
|
|
|
|
.base.cra_driver_name = "ecb-des3-ede-rk",
|
|
|
|
.base.cra_priority = 300,
|
2022-09-27 07:54:44 +00:00
|
|
|
.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
|
2019-11-09 17:09:48 +00:00
|
|
|
.base.cra_blocksize = DES_BLOCK_SIZE,
|
|
|
|
.base.cra_ctxsize = sizeof(struct rk_cipher_ctx),
|
|
|
|
.base.cra_alignmask = 0x07,
|
|
|
|
.base.cra_module = THIS_MODULE,
|
|
|
|
|
2022-09-27 07:54:58 +00:00
|
|
|
.init = rk_cipher_tfm_init,
|
|
|
|
.exit = rk_cipher_tfm_exit,
|
2019-11-09 17:09:48 +00:00
|
|
|
.min_keysize = DES3_EDE_KEY_SIZE,
|
|
|
|
.max_keysize = DES3_EDE_KEY_SIZE,
|
|
|
|
.setkey = rk_tdes_setkey,
|
|
|
|
.encrypt = rk_des3_ede_ecb_encrypt,
|
|
|
|
.decrypt = rk_des3_ede_ecb_decrypt,
|
2023-08-13 06:55:10 +00:00
|
|
|
},
|
|
|
|
.alg.skcipher.op = {
|
|
|
|
.do_one_request = rk_cipher_run,
|
|
|
|
},
|
2019-11-09 17:09:48 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
struct rk_crypto_tmp rk_cbc_des3_ede_alg = {
|
2022-09-27 07:54:49 +00:00
|
|
|
.type = CRYPTO_ALG_TYPE_SKCIPHER,
|
2023-08-13 06:55:10 +00:00
|
|
|
.alg.skcipher.base = {
|
2019-11-09 17:09:48 +00:00
|
|
|
.base.cra_name = "cbc(des3_ede)",
|
|
|
|
.base.cra_driver_name = "cbc-des3-ede-rk",
|
|
|
|
.base.cra_priority = 300,
|
2022-09-27 07:54:44 +00:00
|
|
|
.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
|
2019-11-09 17:09:48 +00:00
|
|
|
.base.cra_blocksize = DES_BLOCK_SIZE,
|
|
|
|
.base.cra_ctxsize = sizeof(struct rk_cipher_ctx),
|
|
|
|
.base.cra_alignmask = 0x07,
|
|
|
|
.base.cra_module = THIS_MODULE,
|
|
|
|
|
2022-09-27 07:54:58 +00:00
|
|
|
.init = rk_cipher_tfm_init,
|
|
|
|
.exit = rk_cipher_tfm_exit,
|
2019-11-09 17:09:48 +00:00
|
|
|
.min_keysize = DES3_EDE_KEY_SIZE,
|
|
|
|
.max_keysize = DES3_EDE_KEY_SIZE,
|
|
|
|
.ivsize = DES_BLOCK_SIZE,
|
|
|
|
.setkey = rk_tdes_setkey,
|
|
|
|
.encrypt = rk_des3_ede_cbc_encrypt,
|
|
|
|
.decrypt = rk_des3_ede_cbc_decrypt,
|
2023-08-13 06:55:10 +00:00
|
|
|
},
|
|
|
|
.alg.skcipher.op = {
|
|
|
|
.do_one_request = rk_cipher_run,
|
|
|
|
},
|
2019-11-09 17:09:48 +00:00
|
|
|
};
|