From 63be5b53b6d15f7706ad21e9801dae5b723e8340 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 14 Feb 2017 13:43:27 -0800 Subject: [PATCH 001/101] crypto: gf128mul - fix some comments Fix incorrect references to GF(128) instead of GF(2^128), as these are two entirely different fields, and fix a few other incorrect comments. Cc: Alex Cope Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/gf128mul.c | 13 +++++++------ include/crypto/gf128mul.h | 26 ++++++++++++++------------ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c index 72015fee533d..d9e3eecc218a 100644 --- a/crypto/gf128mul.c +++ b/crypto/gf128mul.c @@ -44,7 +44,7 @@ --------------------------------------------------------------------------- Issue 31/01/2006 - This file provides fast multiplication in GF(128) as required by several + This file provides fast multiplication in GF(2^128) as required by several cryptographic authentication modes */ @@ -116,9 +116,10 @@ static const u16 gf128mul_table_lle[256] = gf128mul_dat(xda_lle); static const u16 gf128mul_table_bbe[256] = gf128mul_dat(xda_bbe); -/* These functions multiply a field element by x, by x^4 and by x^8 - * in the polynomial field representation. It uses 32-bit word operations - * to gain speed but compensates for machine endianess and hence works +/* + * The following functions multiply a field element by x or by x^8 in + * the polynomial field representation. They use 64-bit word operations + * to gain speed but compensate for machine endianness and hence work * correctly on both styles of machine. */ @@ -251,7 +252,7 @@ EXPORT_SYMBOL(gf128mul_bbe); /* This version uses 64k bytes of table space. A 16 byte buffer has to be multiplied by a 16 byte key - value in GF(128). If we consider a GF(128) value in + value in GF(2^128). If we consider a GF(2^128) value in the buffer's lowest byte, we can construct a table of the 256 16 byte values that result from the 256 values of this byte. This requires 4096 bytes. But we also @@ -330,7 +331,7 @@ EXPORT_SYMBOL(gf128mul_64k_bbe); /* This version uses 4k bytes of table space. A 16 byte buffer has to be multiplied by a 16 byte key - value in GF(128). If we consider a GF(128) value in a + value in GF(2^128). If we consider a GF(2^128) value in a single byte, we can construct a table of the 256 16 byte values that result from the 256 values of this byte. This requires 4096 bytes. If we take the highest byte in diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h index 592d47e565a8..9662c4538873 100644 --- a/include/crypto/gf128mul.h +++ b/include/crypto/gf128mul.h @@ -43,7 +43,7 @@ --------------------------------------------------------------------------- Issue Date: 31/01/2006 - An implementation of field multiplication in Galois Field GF(128) + An implementation of field multiplication in Galois Field GF(2^128) */ #ifndef _CRYPTO_GF128MUL_H @@ -65,7 +65,7 @@ * are left and the lsb's are right. char b[16] is an array and b[0] is * the first octet. * - * 80000000 00000000 00000000 00000000 .... 00000000 00000000 00000000 + * 10000000 00000000 00000000 00000000 .... 00000000 00000000 00000000 * b[0] b[1] b[2] b[3] b[13] b[14] b[15] * * Every bit is a coefficient of some power of X. We can store the bits @@ -85,15 +85,17 @@ * Both of the above formats are easy to implement on big-endian * machines. * - * EME (which is patent encumbered) uses the ble format (bits are stored - * in big endian order and the bytes in little endian). The above buffer - * represents X^7 in this case and the primitive polynomial is b[0] = 0x87. + * XTS and EME (the latter of which is patent encumbered) use the ble + * format (bits are stored in big endian order and the bytes in little + * endian). The above buffer represents X^7 in this case and the + * primitive polynomial is b[0] = 0x87. * * The common machine word-size is smaller than 128 bits, so to make * an efficient implementation we must split into machine word sizes. - * This file uses one 32bit for the moment. Machine endianness comes into - * play. The lle format in relation to machine endianness is discussed - * below by the original author of gf128mul Dr Brian Gladman. + * This implementation uses 64-bit words for the moment. Machine + * endianness comes into play. The lle format in relation to machine + * endianness is discussed below by the original author of gf128mul Dr + * Brian Gladman. * * Let's look at the bbe and ble format on a little endian machine. * @@ -127,10 +129,10 @@ * machines this will automatically aligned to wordsize and on a 64-bit * machine also. */ -/* Multiply a GF128 field element by x. Field elements are held in arrays - of bytes in which field bits 8n..8n + 7 are held in byte[n], with lower - indexed bits placed in the more numerically significant bit positions - within bytes. +/* Multiply a GF(2^128) field element by x. Field elements are + held in arrays of bytes in which field bits 8n..8n + 7 are held in + byte[n], with lower indexed bits placed in the more numerically + significant bit positions within bytes. On little endian machines the bit indexes translate into the bit positions within four 32-bit words in the following way From 2416e4fa98b9763ea06e0f441c23ce7a293a87f4 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 14 Feb 2017 13:43:28 -0800 Subject: [PATCH 002/101] crypto: gf128mul - remove xx() macro The xx() macro serves no purpose and can be removed. Cc: Alex Cope Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/gf128mul.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c index d9e3eecc218a..c050cf6f5aa9 100644 --- a/crypto/gf128mul.c +++ b/crypto/gf128mul.c @@ -97,20 +97,18 @@ the table above */ -#define xx(p, q) 0x##p##q - #define xda_bbe(i) ( \ - (i & 0x80 ? xx(43, 80) : 0) ^ (i & 0x40 ? xx(21, c0) : 0) ^ \ - (i & 0x20 ? xx(10, e0) : 0) ^ (i & 0x10 ? xx(08, 70) : 0) ^ \ - (i & 0x08 ? xx(04, 38) : 0) ^ (i & 0x04 ? xx(02, 1c) : 0) ^ \ - (i & 0x02 ? xx(01, 0e) : 0) ^ (i & 0x01 ? xx(00, 87) : 0) \ + (i & 0x80 ? 0x4380 : 0) ^ (i & 0x40 ? 0x21c0 : 0) ^ \ + (i & 0x20 ? 0x10e0 : 0) ^ (i & 0x10 ? 0x0870 : 0) ^ \ + (i & 0x08 ? 0x0438 : 0) ^ (i & 0x04 ? 0x021c : 0) ^ \ + (i & 0x02 ? 0x010e : 0) ^ (i & 0x01 ? 0x0087 : 0) \ ) #define xda_lle(i) ( \ - (i & 0x80 ? xx(e1, 00) : 0) ^ (i & 0x40 ? xx(70, 80) : 0) ^ \ - (i & 0x20 ? xx(38, 40) : 0) ^ (i & 0x10 ? xx(1c, 20) : 0) ^ \ - (i & 0x08 ? xx(0e, 10) : 0) ^ (i & 0x04 ? xx(07, 08) : 0) ^ \ - (i & 0x02 ? xx(03, 84) : 0) ^ (i & 0x01 ? xx(01, c2) : 0) \ + (i & 0x80 ? 0xe100 : 0) ^ (i & 0x40 ? 0x7080 : 0) ^ \ + (i & 0x20 ? 0x3840 : 0) ^ (i & 0x10 ? 0x1c20 : 0) ^ \ + (i & 0x08 ? 0x0e10 : 0) ^ (i & 0x04 ? 0x0708 : 0) ^ \ + (i & 0x02 ? 0x0384 : 0) ^ (i & 0x01 ? 0x01c2 : 0) \ ) static const u16 gf128mul_table_lle[256] = gf128mul_dat(xda_lle); From f33fd64778f2593b3b377cef47b8604462d56a57 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 14 Feb 2017 13:43:29 -0800 Subject: [PATCH 003/101] crypto: gf128mul - rename the byte overflow tables Though the GF(2^128) byte overflow tables were named the "lle" and "bbe" tables, they are not actually tied to these element formats specifically, but rather to particular a "bit endianness". For example, the bbe table is actually used for both bbe and ble multiplication. Therefore, rename the tables to "le" and "be" and update the comment to explain this. Cc: Alex Cope Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/gf128mul.c | 49 +++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c index c050cf6f5aa9..1fde1c79ffa5 100644 --- a/crypto/gf128mul.c +++ b/crypto/gf128mul.c @@ -88,31 +88,46 @@ q(0xf8), q(0xf9), q(0xfa), q(0xfb), q(0xfc), q(0xfd), q(0xfe), q(0xff) \ } -/* Given the value i in 0..255 as the byte overflow when a field element - in GHASH is multiplied by x^8, this function will return the values that - are generated in the lo 16-bit word of the field value by applying the - modular polynomial. The values lo_byte and hi_byte are returned via the - macro xp_fun(lo_byte, hi_byte) so that the values can be assembled into - memory as required by a suitable definition of this macro operating on - the table above -*/ +/* + * Given a value i in 0..255 as the byte overflow when a field element + * in GF(2^128) is multiplied by x^8, the following macro returns the + * 16-bit value that must be XOR-ed into the low-degree end of the + * product to reduce it modulo the polynomial x^128 + x^7 + x^2 + x + 1. + * + * There are two versions of the macro, and hence two tables: one for + * the "be" convention where the highest-order bit is the coefficient of + * the highest-degree polynomial term, and one for the "le" convention + * where the highest-order bit is the coefficient of the lowest-degree + * polynomial term. In both cases the values are stored in CPU byte + * endianness such that the coefficients are ordered consistently across + * bytes, i.e. in the "be" table bits 15..0 of the stored value + * correspond to the coefficients of x^15..x^0, and in the "le" table + * bits 15..0 correspond to the coefficients of x^0..x^15. + * + * Therefore, provided that the appropriate byte endianness conversions + * are done by the multiplication functions (and these must be in place + * anyway to support both little endian and big endian CPUs), the "be" + * table can be used for multiplications of both "bbe" and "ble" + * elements, and the "le" table can be used for multiplications of both + * "lle" and "lbe" elements. + */ -#define xda_bbe(i) ( \ +#define xda_be(i) ( \ (i & 0x80 ? 0x4380 : 0) ^ (i & 0x40 ? 0x21c0 : 0) ^ \ (i & 0x20 ? 0x10e0 : 0) ^ (i & 0x10 ? 0x0870 : 0) ^ \ (i & 0x08 ? 0x0438 : 0) ^ (i & 0x04 ? 0x021c : 0) ^ \ (i & 0x02 ? 0x010e : 0) ^ (i & 0x01 ? 0x0087 : 0) \ ) -#define xda_lle(i) ( \ +#define xda_le(i) ( \ (i & 0x80 ? 0xe100 : 0) ^ (i & 0x40 ? 0x7080 : 0) ^ \ (i & 0x20 ? 0x3840 : 0) ^ (i & 0x10 ? 0x1c20 : 0) ^ \ (i & 0x08 ? 0x0e10 : 0) ^ (i & 0x04 ? 0x0708 : 0) ^ \ (i & 0x02 ? 0x0384 : 0) ^ (i & 0x01 ? 0x01c2 : 0) \ ) -static const u16 gf128mul_table_lle[256] = gf128mul_dat(xda_lle); -static const u16 gf128mul_table_bbe[256] = gf128mul_dat(xda_bbe); +static const u16 gf128mul_table_le[256] = gf128mul_dat(xda_le); +static const u16 gf128mul_table_be[256] = gf128mul_dat(xda_be); /* * The following functions multiply a field element by x or by x^8 in @@ -125,7 +140,7 @@ static void gf128mul_x_lle(be128 *r, const be128 *x) { u64 a = be64_to_cpu(x->a); u64 b = be64_to_cpu(x->b); - u64 _tt = gf128mul_table_lle[(b << 7) & 0xff]; + u64 _tt = gf128mul_table_le[(b << 7) & 0xff]; r->b = cpu_to_be64((b >> 1) | (a << 63)); r->a = cpu_to_be64((a >> 1) ^ (_tt << 48)); @@ -135,7 +150,7 @@ static void gf128mul_x_bbe(be128 *r, const be128 *x) { u64 a = be64_to_cpu(x->a); u64 b = be64_to_cpu(x->b); - u64 _tt = gf128mul_table_bbe[a >> 63]; + u64 _tt = gf128mul_table_be[a >> 63]; r->a = cpu_to_be64((a << 1) | (b >> 63)); r->b = cpu_to_be64((b << 1) ^ _tt); @@ -145,7 +160,7 @@ void gf128mul_x_ble(be128 *r, const be128 *x) { u64 a = le64_to_cpu(x->a); u64 b = le64_to_cpu(x->b); - u64 _tt = gf128mul_table_bbe[b >> 63]; + u64 _tt = gf128mul_table_be[b >> 63]; r->a = cpu_to_le64((a << 1) ^ _tt); r->b = cpu_to_le64((b << 1) | (a >> 63)); @@ -156,7 +171,7 @@ static void gf128mul_x8_lle(be128 *x) { u64 a = be64_to_cpu(x->a); u64 b = be64_to_cpu(x->b); - u64 _tt = gf128mul_table_lle[b & 0xff]; + u64 _tt = gf128mul_table_le[b & 0xff]; x->b = cpu_to_be64((b >> 8) | (a << 56)); x->a = cpu_to_be64((a >> 8) ^ (_tt << 48)); @@ -166,7 +181,7 @@ static void gf128mul_x8_bbe(be128 *x) { u64 a = be64_to_cpu(x->a); u64 b = be64_to_cpu(x->b); - u64 _tt = gf128mul_table_bbe[a >> 56]; + u64 _tt = gf128mul_table_be[a >> 56]; x->a = cpu_to_be64((a << 8) | (b >> 56)); x->b = cpu_to_be64((b << 8) ^ _tt); From 3ea996ddfb1756658523f371c7ed1137841facaa Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 14 Feb 2017 13:43:30 -0800 Subject: [PATCH 004/101] crypto: gf128mul - constify 4k and 64k multiplication tables Constify the multiplication tables passed to the 4k and 64k multiplication functions, as they are not modified by these functions. Cc: Alex Cope Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/gf128mul.c | 6 +++--- include/crypto/gf128mul.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c index 1fde1c79ffa5..04facc0690aa 100644 --- a/crypto/gf128mul.c +++ b/crypto/gf128mul.c @@ -329,7 +329,7 @@ void gf128mul_free_64k(struct gf128mul_64k *t) } EXPORT_SYMBOL(gf128mul_free_64k); -void gf128mul_64k_bbe(be128 *a, struct gf128mul_64k *t) +void gf128mul_64k_bbe(be128 *a, const struct gf128mul_64k *t) { u8 *ap = (u8 *)a; be128 r[1]; @@ -402,7 +402,7 @@ out: } EXPORT_SYMBOL(gf128mul_init_4k_bbe); -void gf128mul_4k_lle(be128 *a, struct gf128mul_4k *t) +void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t) { u8 *ap = (u8 *)a; be128 r[1]; @@ -417,7 +417,7 @@ void gf128mul_4k_lle(be128 *a, struct gf128mul_4k *t) } EXPORT_SYMBOL(gf128mul_4k_lle); -void gf128mul_4k_bbe(be128 *a, struct gf128mul_4k *t) +void gf128mul_4k_bbe(be128 *a, const struct gf128mul_4k *t) { u8 *ap = (u8 *)a; be128 r[1]; diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h index 9662c4538873..0bc9b5f1c45e 100644 --- a/include/crypto/gf128mul.h +++ b/include/crypto/gf128mul.h @@ -174,8 +174,8 @@ struct gf128mul_4k { struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g); struct gf128mul_4k *gf128mul_init_4k_bbe(const be128 *g); -void gf128mul_4k_lle(be128 *a, struct gf128mul_4k *t); -void gf128mul_4k_bbe(be128 *a, struct gf128mul_4k *t); +void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t); +void gf128mul_4k_bbe(be128 *a, const struct gf128mul_4k *t); static inline void gf128mul_free_4k(struct gf128mul_4k *t) { @@ -196,6 +196,6 @@ struct gf128mul_64k { */ struct gf128mul_64k *gf128mul_init_64k_bbe(const be128 *g); void gf128mul_free_64k(struct gf128mul_64k *t); -void gf128mul_64k_bbe(be128 *a, struct gf128mul_64k *t); +void gf128mul_64k_bbe(be128 *a, const struct gf128mul_64k *t); #endif /* _CRYPTO_GF128MUL_H */ From b56f5cbc7e08ec7d31c42fc41e5247677f20b143 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 14 Feb 2017 21:51:01 +0000 Subject: [PATCH 005/101] crypto: arm/aes-neonbs - resolve fallback cipher at runtime Currently, the bit sliced NEON AES code for ARM has a link time dependency on the scalar ARM asm implementation, which it uses as a fallback to perform CBC encryption and the encryption of the initial XTS tweak. The bit sliced NEON code is both fast and time invariant, which makes it a reasonable default on hardware that supports it. However, the ARM asm code it pulls in is not time invariant, and due to the way it is linked in, cannot be overridden by the new generic time invariant driver. In fact, it will not be used at all, given that the ARM asm code registers itself as a cipher with a priority that exceeds the priority of the fixed time cipher. So remove the link time dependency, and allocate the fallback cipher via the crypto API. Note that this requires this driver's module_init call to be replaced with late_initcall, so that the (possibly generic) fallback cipher is guaranteed to be available when the builtin test is performed at registration time. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/Kconfig | 2 +- arch/arm/crypto/aes-neonbs-glue.c | 60 +++++++++++++++++++++++-------- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index a8fce93137fb..b9adedcc5b2e 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -73,7 +73,7 @@ config CRYPTO_AES_ARM_BS depends on KERNEL_MODE_NEON select CRYPTO_BLKCIPHER select CRYPTO_SIMD - select CRYPTO_AES_ARM + select CRYPTO_AES help Use a faster and more secure NEON based implementation of AES in CBC, CTR and XTS modes diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index 2920b96dbd36..c76377961444 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -42,9 +42,6 @@ asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, int blocks, u8 iv[]); -asmlinkage void __aes_arm_encrypt(const u32 rk[], int rounds, const u8 in[], - u8 out[]); - struct aesbs_ctx { int rounds; u8 rk[13 * (8 * AES_BLOCK_SIZE) + 32] __aligned(AES_BLOCK_SIZE); @@ -52,12 +49,12 @@ struct aesbs_ctx { struct aesbs_cbc_ctx { struct aesbs_ctx key; - u32 enc[AES_MAX_KEYLENGTH_U32]; + struct crypto_cipher *enc_tfm; }; struct aesbs_xts_ctx { struct aesbs_ctx key; - u32 twkey[AES_MAX_KEYLENGTH_U32]; + struct crypto_cipher *tweak_tfm; }; static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key, @@ -132,20 +129,18 @@ static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key, ctx->key.rounds = 6 + key_len / 4; - memcpy(ctx->enc, rk.key_enc, sizeof(ctx->enc)); - kernel_neon_begin(); aesbs_convert_key(ctx->key.rk, rk.key_enc, ctx->key.rounds); kernel_neon_end(); - return 0; + return crypto_cipher_setkey(ctx->enc_tfm, in_key, key_len); } static void cbc_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) { struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); - __aes_arm_encrypt(ctx->enc, ctx->key.rounds, src, dst); + crypto_cipher_encrypt_one(ctx->enc_tfm, dst, src); } static int cbc_encrypt(struct skcipher_request *req) @@ -181,6 +176,23 @@ static int cbc_decrypt(struct skcipher_request *req) return err; } +static int cbc_init(struct crypto_tfm *tfm) +{ + struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->enc_tfm = crypto_alloc_cipher("aes", 0, 0); + if (IS_ERR(ctx->enc_tfm)) + return PTR_ERR(ctx->enc_tfm); + return 0; +} + +static void cbc_exit(struct crypto_tfm *tfm) +{ + struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_cipher(ctx->enc_tfm); +} + static int ctr_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -228,7 +240,6 @@ static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int key_len) { struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - struct crypto_aes_ctx rk; int err; err = xts_verify_key(tfm, in_key, key_len); @@ -236,15 +247,30 @@ static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, return err; key_len /= 2; - err = crypto_aes_expand_key(&rk, in_key + key_len, key_len); + err = crypto_cipher_setkey(ctx->tweak_tfm, in_key + key_len, key_len); if (err) return err; - memcpy(ctx->twkey, rk.key_enc, sizeof(ctx->twkey)); - return aesbs_setkey(tfm, in_key, key_len); } +static int xts_init(struct crypto_tfm *tfm) +{ + struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->tweak_tfm = crypto_alloc_cipher("aes", 0, 0); + if (IS_ERR(ctx->tweak_tfm)) + return PTR_ERR(ctx->tweak_tfm); + return 0; +} + +static void xts_exit(struct crypto_tfm *tfm) +{ + struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_cipher(ctx->tweak_tfm); +} + static int __xts_crypt(struct skcipher_request *req, void (*fn)(u8 out[], u8 const in[], u8 const rk[], int rounds, int blocks, u8 iv[])) @@ -256,7 +282,7 @@ static int __xts_crypt(struct skcipher_request *req, err = skcipher_walk_virt(&walk, req, true); - __aes_arm_encrypt(ctx->twkey, ctx->key.rounds, walk.iv, walk.iv); + crypto_cipher_encrypt_one(ctx->tweak_tfm, walk.iv, walk.iv); kernel_neon_begin(); while (walk.nbytes >= AES_BLOCK_SIZE) { @@ -309,6 +335,8 @@ static struct skcipher_alg aes_algs[] = { { .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), .base.cra_module = THIS_MODULE, .base.cra_flags = CRYPTO_ALG_INTERNAL, + .base.cra_init = cbc_init, + .base.cra_exit = cbc_exit, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -342,6 +370,8 @@ static struct skcipher_alg aes_algs[] = { { .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx), .base.cra_module = THIS_MODULE, .base.cra_flags = CRYPTO_ALG_INTERNAL, + .base.cra_init = xts_init, + .base.cra_exit = xts_exit, .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, @@ -402,5 +432,5 @@ unregister_simds: return err; } -module_init(aes_init); +late_initcall(aes_init); module_exit(aes_exit); From 27c539aeffe2851bf9aeeeba8a58038187a05019 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 14 Feb 2017 21:51:02 +0000 Subject: [PATCH 006/101] crypto: algapi - annotate expected branch behavior in crypto_inc() To prevent unnecessary branching, mark the exit condition of the primary loop as likely(), given that a carry in a 32-bit counter occurs very rarely. On arm64, the resulting code is emitted by GCC as 9a8: cmp w1, #0x3 9ac: add x3, x0, w1, uxtw 9b0: b.ls 9e0 9b4: ldr w2, [x3,#-4]! 9b8: rev w2, w2 9bc: add w2, w2, #0x1 9c0: rev w4, w2 9c4: str w4, [x3] 9c8: cbz w2, 9d0 9cc: ret where the two remaining branch conditions (one for size < 4 and one for the carry) are statically predicted as non-taken, resulting in optimal execution in the vast majority of cases. Also, replace the open coded alignment test with IS_ALIGNED(). Cc: Jason A. Donenfeld Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/algapi.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/algapi.c b/crypto/algapi.c index 6b52e8f0b95f..9eed4ef9c971 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -963,11 +963,11 @@ void crypto_inc(u8 *a, unsigned int size) u32 c; if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || - !((unsigned long)b & (__alignof__(*b) - 1))) + IS_ALIGNED((unsigned long)b, __alignof__(*b))) for (; size >= 4; size -= 4) { c = be32_to_cpu(*--b) + 1; *b = cpu_to_be32(c); - if (c) + if (likely(c)) return; } From 640035a2dc5534b49cd64580e41874b71f131a1c Mon Sep 17 00:00:00 2001 From: Mahipal Challa Date: Wed, 15 Feb 2017 10:45:08 +0530 Subject: [PATCH 007/101] crypto: zip - Add ThunderX ZIP driver core Add a driver for the ZIP engine found on Cavium ThunderX SOCs. The ZIP engine supports hardware accelerated compression and decompression. It includes 2 independent ZIP cores and supports: - DEFLATE compression and decompression (RFC 1951) - LZS compression and decompression (RFC 2395 and ANSI X3.241-1994) - ADLER32 and CRC32 checksums for ZLIB (RFC 1950) and GZIP (RFC 1952) The ZIP engine is presented as a PCI device. It supports DMA and scatter-gather. Signed-off-by: Mahipal Challa Signed-off-by: Jan Glauber Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 7 + drivers/crypto/Makefile | 1 + drivers/crypto/cavium/Makefile | 4 + drivers/crypto/cavium/zip/Makefile | 8 + drivers/crypto/cavium/zip/common.h | 202 ++++ drivers/crypto/cavium/zip/zip_crypto.h | 77 ++ drivers/crypto/cavium/zip/zip_device.c | 201 ++++ drivers/crypto/cavium/zip/zip_device.h | 108 ++ drivers/crypto/cavium/zip/zip_main.c | 552 ++++++++++ drivers/crypto/cavium/zip/zip_main.h | 106 ++ drivers/crypto/cavium/zip/zip_mem.c | 114 ++ drivers/crypto/cavium/zip/zip_mem.h | 78 ++ drivers/crypto/cavium/zip/zip_regs.h | 1347 ++++++++++++++++++++++++ 13 files changed, 2805 insertions(+) create mode 100644 drivers/crypto/cavium/Makefile create mode 100644 drivers/crypto/cavium/zip/Makefile create mode 100644 drivers/crypto/cavium/zip/common.h create mode 100644 drivers/crypto/cavium/zip/zip_crypto.h create mode 100644 drivers/crypto/cavium/zip/zip_device.c create mode 100644 drivers/crypto/cavium/zip/zip_device.h create mode 100644 drivers/crypto/cavium/zip/zip_main.c create mode 100644 drivers/crypto/cavium/zip/zip_main.h create mode 100644 drivers/crypto/cavium/zip/zip_mem.c create mode 100644 drivers/crypto/cavium/zip/zip_mem.h create mode 100644 drivers/crypto/cavium/zip/zip_regs.h diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 473d31288ad8..1a60626937e4 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -515,6 +515,13 @@ config CRYPTO_DEV_MXS_DCP source "drivers/crypto/qat/Kconfig" source "drivers/crypto/cavium/cpt/Kconfig" +config CRYPTO_DEV_CAVIUM_ZIP + tristate "Cavium ZIP driver" + depends on PCI && 64BIT && (ARM64 || COMPILE_TEST) + ---help--- + Select this option if you want to enable compression/decompression + acceleration on Cavium's ARM based SoCs + config CRYPTO_DEV_QCE tristate "Qualcomm crypto engine accelerator" depends on (ARCH_QCOM || COMPILE_TEST) && HAS_DMA && HAS_IOMEM diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 739609471169..41ca339e89d3 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -2,6 +2,7 @@ obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o obj-$(CONFIG_CRYPTO_DEV_BFIN_CRC) += bfin_crc.o +obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/ obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/ obj-$(CONFIG_CRYPTO_DEV_CPT) += cavium/cpt/ diff --git a/drivers/crypto/cavium/Makefile b/drivers/crypto/cavium/Makefile new file mode 100644 index 000000000000..641268b784be --- /dev/null +++ b/drivers/crypto/cavium/Makefile @@ -0,0 +1,4 @@ +# +# Makefile for Cavium crypto device drivers +# +obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += zip/ diff --git a/drivers/crypto/cavium/zip/Makefile b/drivers/crypto/cavium/zip/Makefile new file mode 100644 index 000000000000..2c0750885165 --- /dev/null +++ b/drivers/crypto/cavium/zip/Makefile @@ -0,0 +1,8 @@ +# +# Makefile for Cavium's ZIP Driver. +# + +obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += thunderx_zip.o +thunderx_zip-y := zip_main.o \ + zip_device.o \ + zip_mem.o diff --git a/drivers/crypto/cavium/zip/common.h b/drivers/crypto/cavium/zip/common.h new file mode 100644 index 000000000000..dc451e0a43c5 --- /dev/null +++ b/drivers/crypto/cavium/zip/common.h @@ -0,0 +1,202 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __COMMON_H__ +#define __COMMON_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Device specific zlib function definitions */ +#include "zip_device.h" + +/* ZIP device definitions */ +#include "zip_main.h" + +/* ZIP memory allocation/deallocation related definitions */ +#include "zip_mem.h" + +/* Device specific structure definitions */ +#include "zip_regs.h" + +#define ZIP_ERROR -1 + +#define ZIP_FLUSH_FINISH 4 + +#define RAW_FORMAT 0 /* for rawpipe */ +#define ZLIB_FORMAT 1 /* for zpipe */ +#define GZIP_FORMAT 2 /* for gzpipe */ +#define LZS_FORMAT 3 /* for lzspipe */ + +/* Max number of ZIP devices supported */ +#define MAX_ZIP_DEVICES 2 + +/* Configures the number of zip queues to be used */ +#define ZIP_NUM_QUEUES 2 + +#define DYNAMIC_STOP_EXCESS 1024 + +/* Maximum buffer sizes in direct mode */ +#define MAX_INPUT_BUFFER_SIZE (64 * 1024) +#define MAX_OUTPUT_BUFFER_SIZE (64 * 1024) + +/** + * struct zip_operation - common data structure for comp and decomp operations + * @input: Next input byte is read from here + * @output: Next output byte written here + * @ctx_addr: Inflate context buffer address + * @history: Pointer to the history buffer + * @input_len: Number of bytes available at next_in + * @input_total_len: Total number of input bytes read + * @output_len: Remaining free space at next_out + * @output_total_len: Total number of bytes output so far + * @csum: Checksum value of the uncompressed data + * @flush: Flush flag + * @format: Format (depends on stream's wrap) + * @speed: Speed depends on stream's level + * @ccode: Compression code ( stream's strategy) + * @lzs_flag: Flag for LZS support + * @begin_file: Beginning of file indication for inflate + * @history_len: Size of the history data + * @end_file: Ending of the file indication for inflate + * @compcode: Completion status of the ZIP invocation + * @bytes_read: Input bytes read in current instruction + * @bits_processed: Total bits processed for entire file + * @sizeofptr: To distinguish between ILP32 and LP64 + * @sizeofzops: Optional just for padding + * + * This structure is used to maintain the required meta data for the + * comp and decomp operations. + */ +struct zip_operation { + u8 *input; + u8 *output; + u64 ctx_addr; + u64 history; + + u32 input_len; + u32 input_total_len; + + u32 output_len; + u32 output_total_len; + + u32 csum; + u32 flush; + + u32 format; + u32 speed; + u32 ccode; + u32 lzs_flag; + + u32 begin_file; + u32 history_len; + + u32 end_file; + u32 compcode; + u32 bytes_read; + u32 bits_processed; + + u32 sizeofptr; + u32 sizeofzops; +}; + +/* error messages */ +#define zip_err(fmt, args...) pr_err("ZIP ERR:%s():%d: " \ + fmt "\n", __func__, __LINE__, ## args) + +#ifdef MSG_ENABLE +/* Enable all messages */ +#define zip_msg(fmt, args...) pr_info("ZIP_MSG:" fmt "\n", ## args) +#else +#define zip_msg(fmt, args...) +#endif + +#if defined(ZIP_DEBUG_ENABLE) && defined(MSG_ENABLE) + +#ifdef DEBUG_LEVEL + +#define FILE_NAME (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : \ + strrchr(__FILE__, '\\') ? strrchr(__FILE__, '\\') + 1 : __FILE__) + +#if DEBUG_LEVEL >= 4 + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s: %s() : %d: " \ + fmt "\n", FILE_NAME, __func__, __LINE__, ## args) + +#elif DEBUG_LEVEL >= 3 + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s: %s() : %d: " \ + fmt "\n", FILE_NAME, __func__, __LINE__, ## args) + +#elif DEBUG_LEVEL >= 2 + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG: %s() : %d: " \ + fmt "\n", __func__, __LINE__, ## args) + +#else + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG:" fmt "\n", ## args) + +#endif /* DEBUG LEVEL >=4 */ + +#else + +#define zip_dbg(fmt, args...) pr_info("ZIP DBG:" fmt "\n", ## args) + +#endif /* DEBUG_LEVEL */ +#else + +#define zip_dbg(fmt, args...) + +#endif /* ZIP_DEBUG_ENABLE && MSG_ENABLE*/ + +#endif diff --git a/drivers/crypto/cavium/zip/zip_crypto.h b/drivers/crypto/cavium/zip/zip_crypto.h new file mode 100644 index 000000000000..2d91b05a94b8 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_crypto.h @@ -0,0 +1,77 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_CRYPTO_H__ +#define __ZIP_CRYPTO_H__ + +#include +#include +#include "common.h" + +struct zip_kernel_ctx { + struct zip_operation zip_comp; + struct zip_operation zip_decomp; +}; + +int zip_alloc_comp_ctx_deflate(struct crypto_tfm *tfm); +int zip_alloc_comp_ctx_lzs(struct crypto_tfm *tfm); +void zip_free_comp_ctx(struct crypto_tfm *tfm); +int zip_comp_compress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen); +int zip_comp_decompress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen); + +void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm); +void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm); +void zip_free_scomp_ctx(struct crypto_scomp *tfm, void *zip_ctx); +int zip_scomp_compress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx); +int zip_scomp_decompress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx); +#endif diff --git a/drivers/crypto/cavium/zip/zip_device.c b/drivers/crypto/cavium/zip/zip_device.c new file mode 100644 index 000000000000..e980f0894546 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_device.c @@ -0,0 +1,201 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#include "common.h" + +/** + * zip_cmd_queue_consumed - Calculates the space consumed in the command queue. + * + * @zip_dev: Pointer to zip device structure + * @queue: Queue number + * + * Return: Bytes consumed in the command queue buffer. + */ +static inline u32 zip_cmd_queue_consumed(struct zip_device *zip_dev, int queue) +{ + return ((zip_dev->iq[queue].sw_head - zip_dev->iq[queue].sw_tail) * + sizeof(u64 *)); +} + +/** + * zip_load_instr - Submits the instruction into the ZIP command queue + * @instr: Pointer to the instruction to be submitted + * @zip_dev: Pointer to ZIP device structure to which the instruction is to + * be submitted + * + * This function copies the ZIP instruction to the command queue and rings the + * doorbell to notify the engine of the instruction submission. The command + * queue is maintained in a circular fashion. When there is space for exactly + * one instruction in the queue, next chunk pointer of the queue is made to + * point to the head of the queue, thus maintaining a circular queue. + * + * Return: Queue number to which the instruction was submitted + */ +u32 zip_load_instr(union zip_inst_s *instr, + struct zip_device *zip_dev) +{ + union zip_quex_doorbell dbell; + u32 queue = 0; + u32 consumed = 0; + u64 *ncb_ptr = NULL; + union zip_nptr_s ncp; + + /* + * Distribute the instructions between the enabled queues based on + * the CPU id. + */ + if (smp_processor_id() % 2 == 0) + queue = 0; + else + queue = 1; + + zip_dbg("CPU Core: %d Queue number:%d", smp_processor_id(), queue); + + /* Take cmd buffer lock */ + spin_lock(&zip_dev->iq[queue].lock); + + /* + * Command Queue implementation + * 1. If there is place for new instructions, push the cmd at sw_head. + * 2. If there is place for exactly one instruction, push the new cmd + * at the sw_head. Make sw_head point to the sw_tail to make it + * circular. Write sw_head's physical address to the "Next-Chunk + * Buffer Ptr" to make it cmd_hw_tail. + * 3. Ring the door bell. + */ + zip_dbg("sw_head : %lx", zip_dev->iq[queue].sw_head); + zip_dbg("sw_tail : %lx", zip_dev->iq[queue].sw_tail); + + consumed = zip_cmd_queue_consumed(zip_dev, queue); + /* Check if there is space to push just one cmd */ + if ((consumed + 128) == (ZIP_CMD_QBUF_SIZE - 8)) { + zip_dbg("Cmd queue space available for single command"); + /* Space for one cmd, pust it and make it circular queue */ + memcpy((u8 *)zip_dev->iq[queue].sw_head, (u8 *)instr, + sizeof(union zip_inst_s)); + zip_dev->iq[queue].sw_head += 16; /* 16 64_bit words = 128B */ + + /* Now, point the "Next-Chunk Buffer Ptr" to sw_head */ + ncb_ptr = zip_dev->iq[queue].sw_head; + + zip_dbg("ncb addr :0x%lx sw_head addr :0x%lx", + ncb_ptr, zip_dev->iq[queue].sw_head - 16); + + /* Using Circular command queue */ + zip_dev->iq[queue].sw_head = zip_dev->iq[queue].sw_tail; + /* Mark this buffer for free */ + zip_dev->iq[queue].free_flag = 1; + + /* Write new chunk buffer address at "Next-Chunk Buffer Ptr" */ + ncp.u_reg64 = 0ull; + ncp.s.addr = __pa(zip_dev->iq[queue].sw_head); + *ncb_ptr = ncp.u_reg64; + zip_dbg("*ncb_ptr :0x%lx sw_head[phys] :0x%lx", + *ncb_ptr, __pa(zip_dev->iq[queue].sw_head)); + + zip_dev->iq[queue].pend_cnt++; + + } else { + zip_dbg("Enough space is available for commands"); + /* Push this cmd to cmd queue buffer */ + memcpy((u8 *)zip_dev->iq[queue].sw_head, (u8 *)instr, + sizeof(union zip_inst_s)); + zip_dev->iq[queue].sw_head += 16; /* 16 64_bit words = 128B */ + + zip_dev->iq[queue].pend_cnt++; + } + zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", + zip_dev->iq[queue].sw_head, zip_dev->iq[queue].sw_tail, + zip_dev->iq[queue].hw_tail); + + zip_dbg(" Pushed the new cmd : pend_cnt : %d", + zip_dev->iq[queue].pend_cnt); + + /* Ring the doorbell */ + dbell.u_reg64 = 0ull; + dbell.s.dbell_cnt = 1; + zip_reg_write(dbell.u_reg64, + (zip_dev->reg_base + ZIP_QUEX_DOORBELL(queue))); + + /* Unlock cmd buffer lock */ + spin_unlock(&zip_dev->iq[queue].lock); + + return queue; +} + +/** + * zip_update_cmd_bufs - Updates the queue statistics after posting the + * instruction + * @zip_dev: Pointer to zip device structure + * @queue: Queue number + */ +void zip_update_cmd_bufs(struct zip_device *zip_dev, u32 queue) +{ + /* Take cmd buffer lock */ + spin_lock(&zip_dev->iq[queue].lock); + + /* Check if the previous buffer can be freed */ + if (zip_dev->iq[queue].free_flag == 1) { + zip_dbg("Free flag. Free cmd buffer, adjust sw head and tail"); + /* Reset the free flag */ + zip_dev->iq[queue].free_flag = 0; + + /* Point the hw_tail to start of the new chunk buffer */ + zip_dev->iq[queue].hw_tail = zip_dev->iq[queue].sw_head; + } else { + zip_dbg("Free flag not set. increment hw tail"); + zip_dev->iq[queue].hw_tail += 16; /* 16 64_bit words = 128B */ + } + + zip_dev->iq[queue].done_cnt++; + zip_dev->iq[queue].pend_cnt--; + + zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", + zip_dev->iq[queue].sw_head, zip_dev->iq[queue].sw_tail, + zip_dev->iq[queue].hw_tail); + zip_dbg(" Got CC : pend_cnt : %d\n", zip_dev->iq[queue].pend_cnt); + + spin_unlock(&zip_dev->iq[queue].lock); +} diff --git a/drivers/crypto/cavium/zip/zip_device.h b/drivers/crypto/cavium/zip/zip_device.h new file mode 100644 index 000000000000..9e18b3b93d38 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_device.h @@ -0,0 +1,108 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_DEVICE_H__ +#define __ZIP_DEVICE_H__ + +#include +#include "zip_main.h" + +struct sg_info { + /* + * Pointer to the input data when scatter_gather == 0 and + * pointer to the input gather list buffer when scatter_gather == 1 + */ + union zip_zptr_s *gather; + + /* + * Pointer to the output data when scatter_gather == 0 and + * pointer to the output scatter list buffer when scatter_gather == 1 + */ + union zip_zptr_s *scatter; + + /* + * Holds size of the output buffer pointed by scatter list + * when scatter_gather == 1 + */ + u64 scatter_buf_size; + + /* for gather data */ + u64 gather_enable; + + /* for scatter data */ + u64 scatter_enable; + + /* Number of gather list pointers for gather data */ + u32 gbuf_cnt; + + /* Number of scatter list pointers for scatter data */ + u32 sbuf_cnt; + + /* Buffers allocation state */ + u8 alloc_state; +}; + +/** + * struct zip_state - Structure representing the required information related + * to a command + * @zip_cmd: Pointer to zip instruction structure + * @result: Pointer to zip result structure + * @ctx: Context pointer for inflate + * @history: Decompression history pointer + * @sginfo: Scatter-gather info structure + */ +struct zip_state { + union zip_inst_s zip_cmd; + union zip_zres_s result; + union zip_zptr_s *ctx; + union zip_zptr_s *history; + struct sg_info sginfo; +}; + +#define ZIP_CONTEXT_SIZE 2048 +#define ZIP_INFLATE_HISTORY_SIZE 32768 +#define ZIP_DEFLATE_HISTORY_SIZE 32768 + +#endif diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c new file mode 100644 index 000000000000..89ea360d410d --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_main.c @@ -0,0 +1,552 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#include "common.h" +#include "zip_crypto.h" + +#define DRV_NAME "ThunderX-ZIP" + +static struct zip_device *zip_dev[MAX_ZIP_DEVICES]; + +static const struct pci_device_id zip_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDERX_ZIP) }, + { 0, } +}; + +void zip_reg_write(u64 val, u64 __iomem *addr) +{ + writeq(val, addr); +} + +u64 zip_reg_read(u64 __iomem *addr) +{ + return readq(addr); +} + +/* + * Allocates new ZIP device structure + * Returns zip_device pointer or NULL if cannot allocate memory for zip_device + */ +static struct zip_device *zip_alloc_device(struct pci_dev *pdev) +{ + struct zip_device *zip = NULL; + int idx; + + for (idx = 0; idx < MAX_ZIP_DEVICES; idx++) { + if (!zip_dev[idx]) + break; + } + + /* To ensure that the index is within the limit */ + if (idx < MAX_ZIP_DEVICES) + zip = devm_kzalloc(&pdev->dev, sizeof(*zip), GFP_KERNEL); + + if (!zip) + return NULL; + + zip_dev[idx] = zip; + zip->index = idx; + return zip; +} + +/** + * zip_get_device - Get ZIP device based on node id of cpu + * + * @node: Node id of the current cpu + * Return: Pointer to Zip device structure + */ +struct zip_device *zip_get_device(int node) +{ + if ((node < MAX_ZIP_DEVICES) && (node >= 0)) + return zip_dev[node]; + + zip_err("ZIP device not found for node id %d\n", node); + return NULL; +} + +/** + * zip_get_node_id - Get the node id of the current cpu + * + * Return: Node id of the current cpu + */ +int zip_get_node_id(void) +{ + return cpu_to_node(smp_processor_id()); +} + +/* Initializes the ZIP h/w sub-system */ +static int zip_init_hw(struct zip_device *zip) +{ + union zip_cmd_ctl cmd_ctl; + union zip_constants constants; + union zip_que_ena que_ena; + union zip_quex_map que_map; + union zip_que_pri que_pri; + + union zip_quex_sbuf_addr que_sbuf_addr; + union zip_quex_sbuf_ctl que_sbuf_ctl; + + int q = 0; + + /* Enable the ZIP Engine(Core) Clock */ + cmd_ctl.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CMD_CTL); + cmd_ctl.s.forceclk = 1; + zip_reg_write(cmd_ctl.u_reg64 & 0xFF, (zip->reg_base + ZIP_CMD_CTL)); + + zip_msg("ZIP_CMD_CTL : 0x%016llx", + zip_reg_read(zip->reg_base + ZIP_CMD_CTL)); + + constants.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CONSTANTS); + zip->depth = constants.s.depth; + zip->onfsize = constants.s.onfsize; + zip->ctxsize = constants.s.ctxsize; + + zip_msg("depth: 0x%016llx , onfsize : 0x%016llx , ctxsize : 0x%016llx", + zip->depth, zip->onfsize, zip->ctxsize); + + /* + * Program ZIP_QUE(0..7)_SBUF_ADDR and ZIP_QUE(0..7)_SBUF_CTL to + * have the correct buffer pointer and size configured for each + * instruction queue. + */ + for (q = 0; q < ZIP_NUM_QUEUES; q++) { + que_sbuf_ctl.u_reg64 = 0ull; + que_sbuf_ctl.s.size = (ZIP_CMD_QBUF_SIZE / sizeof(u64)); + que_sbuf_ctl.s.inst_be = 0; + que_sbuf_ctl.s.stream_id = 0; + zip_reg_write(que_sbuf_ctl.u_reg64, + (zip->reg_base + ZIP_QUEX_SBUF_CTL(q))); + + zip_msg("QUEX_SBUF_CTL[%d]: 0x%016llx", q, + zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_CTL(q))); + } + + for (q = 0; q < ZIP_NUM_QUEUES; q++) { + memset(&zip->iq[q], 0x0, sizeof(struct zip_iq)); + + spin_lock_init(&zip->iq[q].lock); + + if (zip_cmd_qbuf_alloc(zip, q)) { + while (q != 0) { + q--; + zip_cmd_qbuf_free(zip, q); + } + return -ENOMEM; + } + + /* Initialize tail ptr to head */ + zip->iq[q].sw_tail = zip->iq[q].sw_head; + zip->iq[q].hw_tail = zip->iq[q].sw_head; + + /* Write the physical addr to register */ + que_sbuf_addr.u_reg64 = 0ull; + que_sbuf_addr.s.ptr = (__pa(zip->iq[q].sw_head) >> + ZIP_128B_ALIGN); + + zip_msg("QUE[%d]_PTR(PHYS): 0x%016llx", q, + (u64)que_sbuf_addr.s.ptr); + + zip_reg_write(que_sbuf_addr.u_reg64, + (zip->reg_base + ZIP_QUEX_SBUF_ADDR(q))); + + zip_msg("QUEX_SBUF_ADDR[%d]: 0x%016llx", q, + zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_ADDR(q))); + + zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx", + zip->iq[q].sw_head, zip->iq[q].sw_tail, + zip->iq[q].hw_tail); + zip_dbg("sw_head phy addr : 0x%lx", que_sbuf_addr.s.ptr); + } + + /* + * Queue-to-ZIP core mapping + * If a queue is not mapped to a particular core, it is equivalent to + * the ZIP core being disabled. + */ + que_ena.u_reg64 = 0x0ull; + /* Enabling queues based on ZIP_NUM_QUEUES */ + for (q = 0; q < ZIP_NUM_QUEUES; q++) + que_ena.s.ena |= (0x1 << q); + zip_reg_write(que_ena.u_reg64, (zip->reg_base + ZIP_QUE_ENA)); + + zip_msg("QUE_ENA : 0x%016llx", + zip_reg_read(zip->reg_base + ZIP_QUE_ENA)); + + for (q = 0; q < ZIP_NUM_QUEUES; q++) { + que_map.u_reg64 = 0ull; + /* Mapping each queue to two ZIP cores */ + que_map.s.zce = 0x3; + zip_reg_write(que_map.u_reg64, + (zip->reg_base + ZIP_QUEX_MAP(q))); + + zip_msg("QUE_MAP(%d) : 0x%016llx", q, + zip_reg_read(zip->reg_base + ZIP_QUEX_MAP(q))); + } + + que_pri.u_reg64 = 0ull; + for (q = 0; q < ZIP_NUM_QUEUES; q++) + que_pri.s.pri |= (0x1 << q); /* Higher Priority RR */ + zip_reg_write(que_pri.u_reg64, (zip->reg_base + ZIP_QUE_PRI)); + + zip_msg("QUE_PRI %016llx", zip_reg_read(zip->reg_base + ZIP_QUE_PRI)); + + return 0; +} + +static int zip_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct zip_device *zip = NULL; + int err; + + zip = zip_alloc_device(pdev); + if (!zip) + return -ENOMEM; + + dev_info(dev, "Found ZIP device %d %x:%x on Node %d\n", zip->index, + pdev->vendor, pdev->device, dev_to_node(dev)); + + pci_set_drvdata(pdev, zip); + zip->pdev = pdev; + + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device"); + goto err_free_device; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x", err); + goto err_disable_device; + } + + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get usable DMA configuration\n"); + goto err_release_regions; + } + + err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get 48-bit DMA for allocations\n"); + goto err_release_regions; + } + + /* MAP configuration registers */ + zip->reg_base = pci_ioremap_bar(pdev, PCI_CFG_ZIP_PF_BAR0); + if (!zip->reg_base) { + dev_err(dev, "ZIP: Cannot map BAR0 CSR memory space, aborting"); + err = -ENOMEM; + goto err_release_regions; + } + + /* Initialize ZIP Hardware */ + err = zip_init_hw(zip); + if (err) + goto err_release_regions; + + return 0; + +err_release_regions: + if (zip->reg_base) + iounmap(zip->reg_base); + pci_release_regions(pdev); + +err_disable_device: + pci_disable_device(pdev); + +err_free_device: + pci_set_drvdata(pdev, NULL); + + /* Remove zip_dev from zip_device list, free the zip_device memory */ + zip_dev[zip->index] = NULL; + devm_kfree(dev, zip); + + return err; +} + +static void zip_remove(struct pci_dev *pdev) +{ + struct zip_device *zip = pci_get_drvdata(pdev); + union zip_cmd_ctl cmd_ctl; + int q = 0; + + if (!zip) + return; + + if (zip->reg_base) { + cmd_ctl.u_reg64 = 0x0ull; + cmd_ctl.s.reset = 1; /* Forces ZIP cores to do reset */ + zip_reg_write(cmd_ctl.u_reg64, (zip->reg_base + ZIP_CMD_CTL)); + iounmap(zip->reg_base); + } + + pci_release_regions(pdev); + pci_disable_device(pdev); + + /* + * Free Command Queue buffers. This free should be called for all + * the enabled Queues. + */ + for (q = 0; q < ZIP_NUM_QUEUES; q++) + zip_cmd_qbuf_free(zip, q); + + pci_set_drvdata(pdev, NULL); + /* remove zip device from zip device list */ + zip_dev[zip->index] = NULL; +} + +/* Dummy Functions */ +int zip_alloc_comp_ctx_deflate(struct crypto_tfm *tfm) +{ + return 0; +} + +int zip_alloc_comp_ctx_lzs(struct crypto_tfm *tfm) +{ + return 0; +} + +void zip_free_comp_ctx(struct crypto_tfm *tfm) +{ +} + +int zip_comp_compress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen) +{ + return 0; +} + +int zip_comp_decompress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen) +{ + return 0; +} + +void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm) +{ + return NULL; +} + +void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm) +{ + return NULL; +} + +void zip_free_scomp_ctx(struct crypto_scomp *tfm, void *zip_ctx) +{ +} + +int zip_scomp_compress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx) +{ + return 0; +} + +int zip_scomp_decompress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx) +{ + return 0; +} + +/* PCI Sub-System Interface */ +static struct pci_driver zip_driver = { + .name = DRV_NAME, + .id_table = zip_id_table, + .probe = zip_probe, + .remove = zip_remove, +}; + +/* Kernel Crypto Subsystem Interface */ + +static struct crypto_alg zip_comp_deflate = { + .cra_name = "deflate", + .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, + .cra_ctxsize = sizeof(struct zip_kernel_ctx), + .cra_priority = 300, + .cra_module = THIS_MODULE, + .cra_init = zip_alloc_comp_ctx_deflate, + .cra_exit = zip_free_comp_ctx, + .cra_u = { .compress = { + .coa_compress = zip_comp_compress, + .coa_decompress = zip_comp_decompress + } } +}; + +static struct crypto_alg zip_comp_lzs = { + .cra_name = "lzs", + .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, + .cra_ctxsize = sizeof(struct zip_kernel_ctx), + .cra_priority = 300, + .cra_module = THIS_MODULE, + .cra_init = zip_alloc_comp_ctx_lzs, + .cra_exit = zip_free_comp_ctx, + .cra_u = { .compress = { + .coa_compress = zip_comp_compress, + .coa_decompress = zip_comp_decompress + } } +}; + +static struct scomp_alg zip_scomp_deflate = { + .alloc_ctx = zip_alloc_scomp_ctx_deflate, + .free_ctx = zip_free_scomp_ctx, + .compress = zip_scomp_compress, + .decompress = zip_scomp_decompress, + .base = { + .cra_name = "deflate", + .cra_driver_name = "deflate-scomp", + .cra_module = THIS_MODULE, + .cra_priority = 300, + } +}; + +static struct scomp_alg zip_scomp_lzs = { + .alloc_ctx = zip_alloc_scomp_ctx_lzs, + .free_ctx = zip_free_scomp_ctx, + .compress = zip_scomp_compress, + .decompress = zip_scomp_decompress, + .base = { + .cra_name = "lzs", + .cra_driver_name = "lzs-scomp", + .cra_module = THIS_MODULE, + .cra_priority = 300, + } +}; + +static int zip_register_compression_device(void) +{ + int ret; + + ret = crypto_register_alg(&zip_comp_deflate); + if (ret < 0) { + zip_err("Deflate algorithm registration failed\n"); + return ret; + } + + ret = crypto_register_alg(&zip_comp_lzs); + if (ret < 0) { + zip_err("LZS algorithm registration failed\n"); + goto err_unregister_alg_deflate; + } + + ret = crypto_register_scomp(&zip_scomp_deflate); + if (ret < 0) { + zip_err("Deflate scomp algorithm registration failed\n"); + goto err_unregister_alg_lzs; + } + + ret = crypto_register_scomp(&zip_scomp_lzs); + if (ret < 0) { + zip_err("LZS scomp algorithm registration failed\n"); + goto err_unregister_scomp_deflate; + } + + return ret; + +err_unregister_scomp_deflate: + crypto_unregister_scomp(&zip_scomp_deflate); +err_unregister_alg_lzs: + crypto_unregister_alg(&zip_comp_lzs); +err_unregister_alg_deflate: + crypto_unregister_alg(&zip_comp_deflate); + + return ret; +} + +static void zip_unregister_compression_device(void) +{ + crypto_unregister_alg(&zip_comp_deflate); + crypto_unregister_alg(&zip_comp_lzs); + crypto_unregister_scomp(&zip_scomp_deflate); + crypto_unregister_scomp(&zip_scomp_lzs); +} + +static int __init zip_init_module(void) +{ + int ret; + + zip_msg("%s\n", DRV_NAME); + + ret = pci_register_driver(&zip_driver); + if (ret < 0) { + zip_err("ZIP: pci_register_driver() failed\n"); + return ret; + } + + /* Register with the Kernel Crypto Interface */ + ret = zip_register_compression_device(); + if (ret < 0) { + zip_err("ZIP: Kernel Crypto Registration failed\n"); + goto err_pci_unregister; + } + + return ret; + +err_pci_unregister: + pci_unregister_driver(&zip_driver); + return ret; +} + +static void __exit zip_cleanup_module(void) +{ + /* Unregister from the kernel crypto interface */ + zip_unregister_compression_device(); + + /* Unregister this driver for pci zip devices */ + pci_unregister_driver(&zip_driver); +} + +module_init(zip_init_module); +module_exit(zip_cleanup_module); + +MODULE_AUTHOR("Cavium Inc"); +MODULE_DESCRIPTION("Cavium Inc ThunderX ZIP Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_DEVICE_TABLE(pci, zip_id_table); diff --git a/drivers/crypto/cavium/zip/zip_main.h b/drivers/crypto/cavium/zip/zip_main.h new file mode 100644 index 000000000000..1775c77d2a48 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_main.h @@ -0,0 +1,106 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_MAIN_H__ +#define __ZIP_MAIN_H__ + +#include "zip_device.h" +#include "zip_regs.h" + +/* PCI device IDs */ +#define PCI_DEVICE_ID_THUNDERX_ZIP 0xA01A + +/* ZIP device BARs */ +#define PCI_CFG_ZIP_PF_BAR0 0 /* Base addr for normal regs */ + +/* Maximum available zip queues */ +#define ZIP_MAX_NUM_QUEUES 8 + +#define ZIP_128B_ALIGN 7 + +/* Command queue buffer size */ +#define ZIP_CMD_QBUF_SIZE (8064 + 8) + +struct zip_registers { + char *reg_name; + u64 reg_offset; +}; + +/* ZIP Instruction Queue */ +struct zip_iq { + u64 *sw_head; + u64 *sw_tail; + u64 *hw_tail; + u64 done_cnt; + u64 pend_cnt; + u64 free_flag; + + /* ZIP IQ lock */ + spinlock_t lock; +}; + +/* ZIP Device */ +struct zip_device { + u32 index; + void __iomem *reg_base; + struct pci_dev *pdev; + + /* Different ZIP Constants */ + u64 depth; + u64 onfsize; + u64 ctxsize; + + struct zip_iq iq[ZIP_MAX_NUM_QUEUES]; +}; + +/* Prototypes */ +struct zip_device *zip_get_device(int node_id); +int zip_get_node_id(void); +void zip_reg_write(u64 val, u64 __iomem *addr); +u64 zip_reg_read(u64 __iomem *addr); +void zip_update_cmd_bufs(struct zip_device *zip_dev, u32 queue); +u32 zip_load_instr(union zip_inst_s *instr, struct zip_device *zip_dev); + +#endif /* ZIP_MAIN_H */ diff --git a/drivers/crypto/cavium/zip/zip_mem.c b/drivers/crypto/cavium/zip/zip_mem.c new file mode 100644 index 000000000000..b3e0843a9169 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_mem.c @@ -0,0 +1,114 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#include +#include + +#include "common.h" + +/** + * zip_cmd_qbuf_alloc - Allocates a cmd buffer for ZIP Instruction Queue + * @zip: Pointer to zip device structure + * @q: Queue number to allocate bufffer to + * Return: 0 if successful, -ENOMEM otherwise + */ +int zip_cmd_qbuf_alloc(struct zip_device *zip, int q) +{ + zip->iq[q].sw_head = (u64 *)__get_free_pages((GFP_KERNEL | GFP_DMA), + get_order(ZIP_CMD_QBUF_SIZE)); + + if (!zip->iq[q].sw_head) + return -ENOMEM; + + memset(zip->iq[q].sw_head, 0, ZIP_CMD_QBUF_SIZE); + + zip_dbg("cmd_qbuf_alloc[%d] Success : %p\n", q, zip->iq[q].sw_head); + return 0; +} + +/** + * zip_cmd_qbuf_free - Frees the cmd Queue buffer + * @zip: Pointer to zip device structure + * @q: Queue number to free buffer of + */ +void zip_cmd_qbuf_free(struct zip_device *zip, int q) +{ + zip_dbg("Freeing cmd_qbuf 0x%lx\n", zip->iq[q].sw_tail); + + free_pages((u64)zip->iq[q].sw_tail, get_order(ZIP_CMD_QBUF_SIZE)); +} + +/** + * zip_data_buf_alloc - Allocates memory for a data bufffer + * @size: Size of the buffer to allocate + * Returns: Pointer to the buffer allocated + */ +u8 *zip_data_buf_alloc(u64 size) +{ + u8 *ptr; + + ptr = (u8 *)__get_free_pages((GFP_KERNEL | GFP_DMA), + get_order(size)); + + if (!ptr) + return NULL; + + memset(ptr, 0, size); + + zip_dbg("Data buffer allocation success\n"); + return ptr; +} + +/** + * zip_data_buf_free - Frees the memory of a data buffer + * @ptr: Pointer to the buffer + * @size: Buffer size + */ +void zip_data_buf_free(u8 *ptr, u64 size) +{ + zip_dbg("Freeing data buffer 0x%lx\n", ptr); + + free_pages((u64)ptr, get_order(size)); +} diff --git a/drivers/crypto/cavium/zip/zip_mem.h b/drivers/crypto/cavium/zip/zip_mem.h new file mode 100644 index 000000000000..f8f2f08c4a5c --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_mem.h @@ -0,0 +1,78 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_MEM_H__ +#define __ZIP_MEM_H__ + +/** + * zip_cmd_qbuf_free - Frees the cmd Queue buffer + * @zip: Pointer to zip device structure + * @q: Queue nmber to free buffer of + */ +void zip_cmd_qbuf_free(struct zip_device *zip, int q); + +/** + * zip_cmd_qbuf_alloc - Allocates a Chunk/cmd buffer for ZIP Inst(cmd) Queue + * @zip: Pointer to zip device structure + * @q: Queue number to allocate bufffer to + * Return: 0 if successful, 1 otherwise + */ +int zip_cmd_qbuf_alloc(struct zip_device *zip, int q); + +/** + * zip_data_buf_alloc - Allocates memory for a data bufffer + * @size: Size of the buffer to allocate + * Returns: Pointer to the buffer allocated + */ +u8 *zip_data_buf_alloc(u64 size); + +/** + * zip_data_buf_free - Frees the memory of a data buffer + * @ptr: Pointer to the buffer + * @size: Buffer size + */ +void zip_data_buf_free(u8 *ptr, u64 size); + +#endif diff --git a/drivers/crypto/cavium/zip/zip_regs.h b/drivers/crypto/cavium/zip/zip_regs.h new file mode 100644 index 000000000000..d0be682305c1 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_regs.h @@ -0,0 +1,1347 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_REGS_H__ +#define __ZIP_REGS_H__ + +/* + * Configuration and status register (CSR) address and type definitions for + * Cavium ZIP. + */ + +#include + +/* ZIP invocation result completion status codes */ +#define ZIP_CMD_NOTDONE 0x0 + +/* Successful completion. */ +#define ZIP_CMD_SUCCESS 0x1 + +/* Output truncated */ +#define ZIP_CMD_DTRUNC 0x2 + +/* Dynamic Stop */ +#define ZIP_CMD_DYNAMIC_STOP 0x3 + +/* Uncompress ran out of input data when IWORD0[EF] was set */ +#define ZIP_CMD_ITRUNC 0x4 + +/* Uncompress found the reserved block type 3 */ +#define ZIP_CMD_RBLOCK 0x5 + +/* + * Uncompress found LEN != ZIP_CMD_NLEN in an uncompressed block in the input. + */ +#define ZIP_CMD_NLEN 0x6 + +/* Uncompress found a bad code in the main Huffman codes. */ +#define ZIP_CMD_BADCODE 0x7 + +/* Uncompress found a bad code in the 19 Huffman codes encoding lengths. */ +#define ZIP_CMD_BADCODE2 0x8 + +/* Compress found a zero-length input. */ +#define ZIP_CMD_ZERO_LEN 0x9 + +/* The compress or decompress encountered an internal parity error. */ +#define ZIP_CMD_PARITY 0xA + +/* + * Uncompress found a string identifier that precedes the uncompressed data and + * decompression history. + */ +#define ZIP_CMD_FATAL 0xB + +/** + * enum zip_int_vec_e - ZIP MSI-X Vector Enumeration, enumerates the MSI-X + * interrupt vectors. + */ +enum zip_int_vec_e { + ZIP_INT_VEC_E_ECCE = 0x10, + ZIP_INT_VEC_E_FIFE = 0x11, + ZIP_INT_VEC_E_QUE0_DONE = 0x0, + ZIP_INT_VEC_E_QUE0_ERR = 0x8, + ZIP_INT_VEC_E_QUE1_DONE = 0x1, + ZIP_INT_VEC_E_QUE1_ERR = 0x9, + ZIP_INT_VEC_E_QUE2_DONE = 0x2, + ZIP_INT_VEC_E_QUE2_ERR = 0xa, + ZIP_INT_VEC_E_QUE3_DONE = 0x3, + ZIP_INT_VEC_E_QUE3_ERR = 0xb, + ZIP_INT_VEC_E_QUE4_DONE = 0x4, + ZIP_INT_VEC_E_QUE4_ERR = 0xc, + ZIP_INT_VEC_E_QUE5_DONE = 0x5, + ZIP_INT_VEC_E_QUE5_ERR = 0xd, + ZIP_INT_VEC_E_QUE6_DONE = 0x6, + ZIP_INT_VEC_E_QUE6_ERR = 0xe, + ZIP_INT_VEC_E_QUE7_DONE = 0x7, + ZIP_INT_VEC_E_QUE7_ERR = 0xf, + ZIP_INT_VEC_E_ENUM_LAST = 0x12, +}; + +/** + * union zip_zptr_addr_s - ZIP Generic Pointer Structure for ADDR. + * + * It is the generic format of pointers in ZIP_INST_S. + */ +union zip_zptr_addr_s { + u64 u_reg64; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_49_63 : 15; + u64 addr : 49; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 addr : 49; + u64 reserved_49_63 : 15; +#endif + } s; + +}; + +/** + * union zip_zptr_ctl_s - ZIP Generic Pointer Structure for CTL. + * + * It is the generic format of pointers in ZIP_INST_S. + */ +union zip_zptr_ctl_s { + u64 u_reg64; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_112_127 : 16; + u64 length : 16; + u64 reserved_67_95 : 29; + u64 fw : 1; + u64 nc : 1; + u64 data_be : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 data_be : 1; + u64 nc : 1; + u64 fw : 1; + u64 reserved_67_95 : 29; + u64 length : 16; + u64 reserved_112_127 : 16; +#endif + } s; +}; + +/** + * union zip_inst_s - ZIP Instruction Structure. + * Each ZIP instruction has 16 words (they are called IWORD0 to IWORD15 within + * the structure). + */ +union zip_inst_s { + u64 u_reg64[16]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 doneint : 1; + u64 reserved_56_62 : 7; + u64 totaloutputlength : 24; + u64 reserved_27_31 : 5; + u64 exn : 3; + u64 reserved_23_23 : 1; + u64 exbits : 7; + u64 reserved_12_15 : 4; + u64 sf : 1; + u64 ss : 2; + u64 cc : 2; + u64 ef : 1; + u64 bf : 1; + u64 ce : 1; + u64 reserved_3_3 : 1; + u64 ds : 1; + u64 dg : 1; + u64 hg : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 hg : 1; + u64 dg : 1; + u64 ds : 1; + u64 reserved_3_3 : 1; + u64 ce : 1; + u64 bf : 1; + u64 ef : 1; + u64 cc : 2; + u64 ss : 2; + u64 sf : 1; + u64 reserved_12_15 : 4; + u64 exbits : 7; + u64 reserved_23_23 : 1; + u64 exn : 3; + u64 reserved_27_31 : 5; + u64 totaloutputlength : 24; + u64 reserved_56_62 : 7; + u64 doneint : 1; +#endif +#if defined(__BIG_ENDIAN_BITFIELD) + u64 historylength : 16; + u64 reserved_96_111 : 16; + u64 adlercrc32 : 32; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 adlercrc32 : 32; + u64 reserved_96_111 : 16; + u64 historylength : 16; +#endif + union zip_zptr_addr_s ctx_ptr_addr; + union zip_zptr_ctl_s ctx_ptr_ctl; + union zip_zptr_addr_s his_ptr_addr; + union zip_zptr_ctl_s his_ptr_ctl; + union zip_zptr_addr_s inp_ptr_addr; + union zip_zptr_ctl_s inp_ptr_ctl; + union zip_zptr_addr_s out_ptr_addr; + union zip_zptr_ctl_s out_ptr_ctl; + union zip_zptr_addr_s res_ptr_addr; + union zip_zptr_ctl_s res_ptr_ctl; +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_817_831 : 15; + u64 wq_ptr : 49; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 wq_ptr : 49; + u64 reserved_817_831 : 15; +#endif +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_882_895 : 14; + u64 tt : 2; + u64 reserved_874_879 : 6; + u64 grp : 10; + u64 tag : 32; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 tag : 32; + u64 grp : 10; + u64 reserved_874_879 : 6; + u64 tt : 2; + u64 reserved_882_895 : 14; +#endif +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_896_959 : 64; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 reserved_896_959 : 64; +#endif +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_960_1023 : 64; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 reserved_960_1023 : 64; +#endif + } s; +}; + +/** + * union zip_nptr_s - ZIP Instruction Next-Chunk-Buffer Pointer (NPTR) + * Structure + * + * ZIP_NPTR structure is used to chain all the zip instruction buffers + * together. ZIP instruction buffers are managed (allocated and released) by + * the software. + */ +union zip_nptr_s { + u64 u_reg64; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_49_63 : 15; + u64 addr : 49; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 addr : 49; + u64 reserved_49_63 : 15; +#endif + } s; +}; + +/** + * union zip_zptr_s - ZIP Generic Pointer Structure. + * + * It is the generic format of pointers in ZIP_INST_S. + */ +union zip_zptr_s { + u64 u_reg64[2]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_49_63 : 15; + u64 addr : 49; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 addr : 49; + u64 reserved_49_63 : 15; +#endif +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_112_127 : 16; + u64 length : 16; + u64 reserved_67_95 : 29; + u64 fw : 1; + u64 nc : 1; + u64 data_be : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 data_be : 1; + u64 nc : 1; + u64 fw : 1; + u64 reserved_67_95 : 29; + u64 length : 16; + u64 reserved_112_127 : 16; +#endif + } s; +}; + +/** + * union zip_zres_s - ZIP Result Structure + * + * The ZIP coprocessor writes the result structure after it completes the + * invocation. The result structure is exactly 24 bytes, and each invocation of + * the ZIP coprocessor produces exactly one result structure. + */ +union zip_zres_s { + u64 u_reg64[3]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 crc32 : 32; + u64 adler32 : 32; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 adler32 : 32; + u64 crc32 : 32; +#endif +#if defined(__BIG_ENDIAN_BITFIELD) + u64 totalbyteswritten : 32; + u64 totalbytesread : 32; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 totalbytesread : 32; + u64 totalbyteswritten : 32; +#endif +#if defined(__BIG_ENDIAN_BITFIELD) + u64 totalbitsprocessed : 32; + u64 doneint : 1; + u64 reserved_155_158 : 4; + u64 exn : 3; + u64 reserved_151_151 : 1; + u64 exbits : 7; + u64 reserved_137_143 : 7; + u64 ef : 1; + + volatile u64 compcode : 8; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + + volatile u64 compcode : 8; + u64 ef : 1; + u64 reserved_137_143 : 7; + u64 exbits : 7; + u64 reserved_151_151 : 1; + u64 exn : 3; + u64 reserved_155_158 : 4; + u64 doneint : 1; + u64 totalbitsprocessed : 32; +#endif + } s; +}; + +/** + * union zip_cmd_ctl - Structure representing the register that controls + * clock and reset. + */ +union zip_cmd_ctl { + u64 u_reg64; + struct zip_cmd_ctl_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_2_63 : 62; + u64 forceclk : 1; + u64 reset : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 reset : 1; + u64 forceclk : 1; + u64 reserved_2_63 : 62; +#endif + } s; +}; + +#define ZIP_CMD_CTL 0x0ull + +/** + * union zip_constants - Data structure representing the register that contains + * all of the current implementation-related parameters of the zip core in this + * chip. + */ +union zip_constants { + u64 u_reg64; + struct zip_constants_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 nexec : 8; + u64 reserved_49_55 : 7; + u64 syncflush_capable : 1; + u64 depth : 16; + u64 onfsize : 12; + u64 ctxsize : 12; + u64 reserved_1_7 : 7; + u64 disabled : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 disabled : 1; + u64 reserved_1_7 : 7; + u64 ctxsize : 12; + u64 onfsize : 12; + u64 depth : 16; + u64 syncflush_capable : 1; + u64 reserved_49_55 : 7; + u64 nexec : 8; +#endif + } s; +}; + +#define ZIP_CONSTANTS 0x00A0ull + +/** + * union zip_corex_bist_status - Represents registers which have the BIST + * status of memories in zip cores. + * + * Each bit is the BIST result of an individual memory + * (per bit, 0 = pass and 1 = fail). + */ +union zip_corex_bist_status { + u64 u_reg64; + struct zip_corex_bist_status_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_53_63 : 11; + u64 bstatus : 53; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 bstatus : 53; + u64 reserved_53_63 : 11; +#endif + } s; +}; + +static inline u64 ZIP_COREX_BIST_STATUS(u64 param1) +{ + if (((param1 <= 1))) + return 0x0520ull + (param1 & 1) * 0x8ull; + pr_err("ZIP_COREX_BIST_STATUS: %llu\n", param1); + return 0; +} + +/** + * union zip_ctl_bist_status - Represents register that has the BIST status of + * memories in ZIP_CTL (instruction buffer, G/S pointer FIFO, input data + * buffer, output data buffers). + * + * Each bit is the BIST result of an individual memory + * (per bit, 0 = pass and 1 = fail). + */ +union zip_ctl_bist_status { + u64 u_reg64; + struct zip_ctl_bist_status_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_9_63 : 55; + u64 bstatus : 9; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 bstatus : 9; + u64 reserved_9_63 : 55; +#endif + } s; +}; + +#define ZIP_CTL_BIST_STATUS 0x0510ull + +/** + * union zip_ctl_cfg - Represents the register that controls the behavior of + * the ZIP DMA engines. + * + * It is recommended to keep default values for normal operation. Changing the + * values of the fields may be useful for diagnostics. + */ +union zip_ctl_cfg { + u64 u_reg64; + struct zip_ctl_cfg_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_52_63 : 12; + u64 ildf : 4; + u64 reserved_36_47 : 12; + u64 drtf : 4; + u64 reserved_27_31 : 5; + u64 stcf : 3; + u64 reserved_19_23 : 5; + u64 ldf : 3; + u64 reserved_2_15 : 14; + u64 busy : 1; + u64 reserved_0_0 : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 reserved_0_0 : 1; + u64 busy : 1; + u64 reserved_2_15 : 14; + u64 ldf : 3; + u64 reserved_19_23 : 5; + u64 stcf : 3; + u64 reserved_27_31 : 5; + u64 drtf : 4; + u64 reserved_36_47 : 12; + u64 ildf : 4; + u64 reserved_52_63 : 12; +#endif + } s; +}; + +#define ZIP_CTL_CFG 0x0560ull + +/** + * union zip_dbg_corex_inst - Represents the registers that reflect the status + * of the current instruction that the ZIP core is executing or has executed. + * + * These registers are only for debug use. + */ +union zip_dbg_corex_inst { + u64 u_reg64; + struct zip_dbg_corex_inst_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 busy : 1; + u64 reserved_35_62 : 28; + u64 qid : 3; + u64 iid : 32; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 iid : 32; + u64 qid : 3; + u64 reserved_35_62 : 28; + u64 busy : 1; +#endif + } s; +}; + +static inline u64 ZIP_DBG_COREX_INST(u64 param1) +{ + if (((param1 <= 1))) + return 0x0640ull + (param1 & 1) * 0x8ull; + pr_err("ZIP_DBG_COREX_INST: %llu\n", param1); + return 0; +} + +/** + * union zip_dbg_corex_sta - Represents registers that reflect the status of + * the zip cores. + * + * They are for debug use only. + */ +union zip_dbg_corex_sta { + u64 u_reg64; + struct zip_dbg_corex_sta_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 busy : 1; + u64 reserved_37_62 : 26; + u64 ist : 5; + u64 nie : 32; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 nie : 32; + u64 ist : 5; + u64 reserved_37_62 : 26; + u64 busy : 1; +#endif + } s; +}; + +static inline u64 ZIP_DBG_COREX_STA(u64 param1) +{ + if (((param1 <= 1))) + return 0x0680ull + (param1 & 1) * 0x8ull; + pr_err("ZIP_DBG_COREX_STA: %llu\n", param1); + return 0; +} + +/** + * union zip_dbg_quex_sta - Represets registers that reflect status of the zip + * instruction queues. + * + * They are for debug use only. + */ +union zip_dbg_quex_sta { + u64 u_reg64; + struct zip_dbg_quex_sta_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 busy : 1; + u64 reserved_56_62 : 7; + u64 rqwc : 24; + u64 nii : 32; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 nii : 32; + u64 rqwc : 24; + u64 reserved_56_62 : 7; + u64 busy : 1; +#endif + } s; +}; + +static inline u64 ZIP_DBG_QUEX_STA(u64 param1) +{ + if (((param1 <= 7))) + return 0x1800ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_DBG_QUEX_STA: %llu\n", param1); + return 0; +} + +/** + * union zip_ecc_ctl - Represents the register that enables ECC for each + * individual internal memory that requires ECC. + * + * For debug purpose, it can also flip one or two bits in the ECC data. + */ +union zip_ecc_ctl { + u64 u_reg64; + struct zip_ecc_ctl_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_19_63 : 45; + u64 vmem_cdis : 1; + u64 vmem_fs : 2; + u64 reserved_15_15 : 1; + u64 idf1_cdis : 1; + u64 idf1_fs : 2; + u64 reserved_11_11 : 1; + u64 idf0_cdis : 1; + u64 idf0_fs : 2; + u64 reserved_7_7 : 1; + u64 gspf_cdis : 1; + u64 gspf_fs : 2; + u64 reserved_3_3 : 1; + u64 iqf_cdis : 1; + u64 iqf_fs : 2; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 iqf_fs : 2; + u64 iqf_cdis : 1; + u64 reserved_3_3 : 1; + u64 gspf_fs : 2; + u64 gspf_cdis : 1; + u64 reserved_7_7 : 1; + u64 idf0_fs : 2; + u64 idf0_cdis : 1; + u64 reserved_11_11 : 1; + u64 idf1_fs : 2; + u64 idf1_cdis : 1; + u64 reserved_15_15 : 1; + u64 vmem_fs : 2; + u64 vmem_cdis : 1; + u64 reserved_19_63 : 45; +#endif + } s; +}; + +#define ZIP_ECC_CTL 0x0568ull + +/* NCB - zip_ecce_ena_w1c */ +union zip_ecce_ena_w1c { + u64 u_reg64; + struct zip_ecce_ena_w1c_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_37_63 : 27; + u64 dbe : 5; + u64 reserved_5_31 : 27; + u64 sbe : 5; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 sbe : 5; + u64 reserved_5_31 : 27; + u64 dbe : 5; + u64 reserved_37_63 : 27; +#endif + } s; +}; + +#define ZIP_ECCE_ENA_W1C 0x0598ull + +/* NCB - zip_ecce_ena_w1s */ +union zip_ecce_ena_w1s { + u64 u_reg64; + struct zip_ecce_ena_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_37_63 : 27; + u64 dbe : 5; + u64 reserved_5_31 : 27; + u64 sbe : 5; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 sbe : 5; + u64 reserved_5_31 : 27; + u64 dbe : 5; + u64 reserved_37_63 : 27; +#endif + } s; +}; + +#define ZIP_ECCE_ENA_W1S 0x0590ull + +/** + * union zip_ecce_int - Represents the register that contains the status of the + * ECC interrupt sources. + */ +union zip_ecce_int { + u64 u_reg64; + struct zip_ecce_int_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_37_63 : 27; + u64 dbe : 5; + u64 reserved_5_31 : 27; + u64 sbe : 5; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 sbe : 5; + u64 reserved_5_31 : 27; + u64 dbe : 5; + u64 reserved_37_63 : 27; +#endif + } s; +}; + +#define ZIP_ECCE_INT 0x0580ull + +/* NCB - zip_ecce_int_w1s */ +union zip_ecce_int_w1s { + u64 u_reg64; + struct zip_ecce_int_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_37_63 : 27; + u64 dbe : 5; + u64 reserved_5_31 : 27; + u64 sbe : 5; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 sbe : 5; + u64 reserved_5_31 : 27; + u64 dbe : 5; + u64 reserved_37_63 : 27; +#endif + } s; +}; + +#define ZIP_ECCE_INT_W1S 0x0588ull + +/* NCB - zip_fife_ena_w1c */ +union zip_fife_ena_w1c { + u64 u_reg64; + struct zip_fife_ena_w1c_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_42_63 : 22; + u64 asserts : 42; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 asserts : 42; + u64 reserved_42_63 : 22; +#endif + } s; +}; + +#define ZIP_FIFE_ENA_W1C 0x0090ull + +/* NCB - zip_fife_ena_w1s */ +union zip_fife_ena_w1s { + u64 u_reg64; + struct zip_fife_ena_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_42_63 : 22; + u64 asserts : 42; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 asserts : 42; + u64 reserved_42_63 : 22; +#endif + } s; +}; + +#define ZIP_FIFE_ENA_W1S 0x0088ull + +/* NCB - zip_fife_int */ +union zip_fife_int { + u64 u_reg64; + struct zip_fife_int_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_42_63 : 22; + u64 asserts : 42; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 asserts : 42; + u64 reserved_42_63 : 22; +#endif + } s; +}; + +#define ZIP_FIFE_INT 0x0078ull + +/* NCB - zip_fife_int_w1s */ +union zip_fife_int_w1s { + u64 u_reg64; + struct zip_fife_int_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_42_63 : 22; + u64 asserts : 42; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 asserts : 42; + u64 reserved_42_63 : 22; +#endif + } s; +}; + +#define ZIP_FIFE_INT_W1S 0x0080ull + +/** + * union zip_msix_pbax - Represents the register that is the MSI-X PBA table + * + * The bit number is indexed by the ZIP_INT_VEC_E enumeration. + */ +union zip_msix_pbax { + u64 u_reg64; + struct zip_msix_pbax_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 pend : 64; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 pend : 64; +#endif + } s; +}; + +static inline u64 ZIP_MSIX_PBAX(u64 param1) +{ + if (((param1 == 0))) + return 0x0000838000FF0000ull; + pr_err("ZIP_MSIX_PBAX: %llu\n", param1); + return 0; +} + +/** + * union zip_msix_vecx_addr - Represents the register that is the MSI-X vector + * table, indexed by the ZIP_INT_VEC_E enumeration. + */ +union zip_msix_vecx_addr { + u64 u_reg64; + struct zip_msix_vecx_addr_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_49_63 : 15; + u64 addr : 47; + u64 reserved_1_1 : 1; + u64 secvec : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 secvec : 1; + u64 reserved_1_1 : 1; + u64 addr : 47; + u64 reserved_49_63 : 15; +#endif + } s; +}; + +static inline u64 ZIP_MSIX_VECX_ADDR(u64 param1) +{ + if (((param1 <= 17))) + return 0x0000838000F00000ull + (param1 & 31) * 0x10ull; + pr_err("ZIP_MSIX_VECX_ADDR: %llu\n", param1); + return 0; +} + +/** + * union zip_msix_vecx_ctl - Represents the register that is the MSI-X vector + * table, indexed by the ZIP_INT_VEC_E enumeration. + */ +union zip_msix_vecx_ctl { + u64 u_reg64; + struct zip_msix_vecx_ctl_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_33_63 : 31; + u64 mask : 1; + u64 reserved_20_31 : 12; + u64 data : 20; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 data : 20; + u64 reserved_20_31 : 12; + u64 mask : 1; + u64 reserved_33_63 : 31; +#endif + } s; +}; + +static inline u64 ZIP_MSIX_VECX_CTL(u64 param1) +{ + if (((param1 <= 17))) + return 0x0000838000F00008ull + (param1 & 31) * 0x10ull; + pr_err("ZIP_MSIX_VECX_CTL: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_done - Represents the registers that contain the per-queue + * instruction done count. + */ +union zip_quex_done { + u64 u_reg64; + struct zip_quex_done_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_20_63 : 44; + u64 done : 20; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 done : 20; + u64 reserved_20_63 : 44; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_DONE(u64 param1) +{ + if (((param1 <= 7))) + return 0x2000ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_DONE: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_done_ack - Represents the registers on write to which will + * decrement the per-queue instructiona done count. + */ +union zip_quex_done_ack { + u64 u_reg64; + struct zip_quex_done_ack_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_20_63 : 44; + u64 done_ack : 20; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 done_ack : 20; + u64 reserved_20_63 : 44; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_DONE_ACK(u64 param1) +{ + if (((param1 <= 7))) + return 0x2200ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_DONE_ACK: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_done_ena_w1c - Represents the register which when written + * 1 to will disable the DONEINT interrupt for the queue. + */ +union zip_quex_done_ena_w1c { + u64 u_reg64; + struct zip_quex_done_ena_w1c_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_1_63 : 63; + u64 done_ena : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 done_ena : 1; + u64 reserved_1_63 : 63; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_DONE_ENA_W1C(u64 param1) +{ + if (((param1 <= 7))) + return 0x2600ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_DONE_ENA_W1C: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_done_ena_w1s - Represents the register that when written 1 to + * will enable the DONEINT interrupt for the queue. + */ +union zip_quex_done_ena_w1s { + u64 u_reg64; + struct zip_quex_done_ena_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_1_63 : 63; + u64 done_ena : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 done_ena : 1; + u64 reserved_1_63 : 63; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_DONE_ENA_W1S(u64 param1) +{ + if (((param1 <= 7))) + return 0x2400ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_DONE_ENA_W1S: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_done_wait - Represents the register that specifies the per + * queue interrupt coalescing settings. + */ +union zip_quex_done_wait { + u64 u_reg64; + struct zip_quex_done_wait_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_48_63 : 16; + u64 time_wait : 16; + u64 reserved_20_31 : 12; + u64 num_wait : 20; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 num_wait : 20; + u64 reserved_20_31 : 12; + u64 time_wait : 16; + u64 reserved_48_63 : 16; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_DONE_WAIT(u64 param1) +{ + if (((param1 <= 7))) + return 0x2800ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_DONE_WAIT: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_doorbell - Represents doorbell registers for the ZIP + * instruction queues. + */ +union zip_quex_doorbell { + u64 u_reg64; + struct zip_quex_doorbell_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_20_63 : 44; + u64 dbell_cnt : 20; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 dbell_cnt : 20; + u64 reserved_20_63 : 44; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_DOORBELL(u64 param1) +{ + if (((param1 <= 7))) + return 0x4000ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_DOORBELL: %llu\n", param1); + return 0; +} + +union zip_quex_err_ena_w1c { + u64 u_reg64; + struct zip_quex_err_ena_w1c_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_5_63 : 59; + u64 mdbe : 1; + u64 nwrp : 1; + u64 nrrp : 1; + u64 irde : 1; + u64 dovf : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 dovf : 1; + u64 irde : 1; + u64 nrrp : 1; + u64 nwrp : 1; + u64 mdbe : 1; + u64 reserved_5_63 : 59; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_ERR_ENA_W1C(u64 param1) +{ + if (((param1 <= 7))) + return 0x3600ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_ERR_ENA_W1C: %llu\n", param1); + return 0; +} + +union zip_quex_err_ena_w1s { + u64 u_reg64; + struct zip_quex_err_ena_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_5_63 : 59; + u64 mdbe : 1; + u64 nwrp : 1; + u64 nrrp : 1; + u64 irde : 1; + u64 dovf : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 dovf : 1; + u64 irde : 1; + u64 nrrp : 1; + u64 nwrp : 1; + u64 mdbe : 1; + u64 reserved_5_63 : 59; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_ERR_ENA_W1S(u64 param1) +{ + if (((param1 <= 7))) + return 0x3400ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_ERR_ENA_W1S: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_err_int - Represents registers that contain the per-queue + * error interrupts. + */ +union zip_quex_err_int { + u64 u_reg64; + struct zip_quex_err_int_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_5_63 : 59; + u64 mdbe : 1; + u64 nwrp : 1; + u64 nrrp : 1; + u64 irde : 1; + u64 dovf : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 dovf : 1; + u64 irde : 1; + u64 nrrp : 1; + u64 nwrp : 1; + u64 mdbe : 1; + u64 reserved_5_63 : 59; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_ERR_INT(u64 param1) +{ + if (((param1 <= 7))) + return 0x3000ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_ERR_INT: %llu\n", param1); + return 0; +} + +/* NCB - zip_que#_err_int_w1s */ +union zip_quex_err_int_w1s { + u64 u_reg64; + struct zip_quex_err_int_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_5_63 : 59; + u64 mdbe : 1; + u64 nwrp : 1; + u64 nrrp : 1; + u64 irde : 1; + u64 dovf : 1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 dovf : 1; + u64 irde : 1; + u64 nrrp : 1; + u64 nwrp : 1; + u64 mdbe : 1; + u64 reserved_5_63 : 59; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_ERR_INT_W1S(u64 param1) +{ + if (((param1 <= 7))) + return 0x3200ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_ERR_INT_W1S: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_gcfg - Represents the registers that reflect status of the + * zip instruction queues,debug use only. + */ +union zip_quex_gcfg { + u64 u_reg64; + struct zip_quex_gcfg_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_4_63 : 60; + u64 iqb_ldwb : 1; + u64 cbw_sty : 1; + u64 l2ld_cmd : 2; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 l2ld_cmd : 2; + u64 cbw_sty : 1; + u64 iqb_ldwb : 1; + u64 reserved_4_63 : 60; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_GCFG(u64 param1) +{ + if (((param1 <= 7))) + return 0x1A00ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_GCFG: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_map - Represents the registers that control how each + * instruction queue maps to zip cores. + */ +union zip_quex_map { + u64 u_reg64; + struct zip_quex_map_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_2_63 : 62; + u64 zce : 2; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 zce : 2; + u64 reserved_2_63 : 62; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_MAP(u64 param1) +{ + if (((param1 <= 7))) + return 0x1400ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_MAP: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_sbuf_addr - Represents the registers that set the buffer + * parameters for the instruction queues. + * + * When quiescent (i.e. outstanding doorbell count is 0), it is safe to rewrite + * this register to effectively reset the command buffer state machine. + * These registers must be programmed after SW programs the corresponding + * ZIP_QUE(0..7)_SBUF_CTL. + */ +union zip_quex_sbuf_addr { + u64 u_reg64; + struct zip_quex_sbuf_addr_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_49_63 : 15; + u64 ptr : 42; + u64 off : 7; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 off : 7; + u64 ptr : 42; + u64 reserved_49_63 : 15; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_SBUF_ADDR(u64 param1) +{ + if (((param1 <= 7))) + return 0x1000ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_SBUF_ADDR: %llu\n", param1); + return 0; +} + +/** + * union zip_quex_sbuf_ctl - Represents the registers that set the buffer + * parameters for the instruction queues. + * + * When quiescent (i.e. outstanding doorbell count is 0), it is safe to rewrite + * this register to effectively reset the command buffer state machine. + * These registers must be programmed before SW programs the corresponding + * ZIP_QUE(0..7)_SBUF_ADDR. + */ +union zip_quex_sbuf_ctl { + u64 u_reg64; + struct zip_quex_sbuf_ctl_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_45_63 : 19; + u64 size : 13; + u64 inst_be : 1; + u64 reserved_24_30 : 7; + u64 stream_id : 8; + u64 reserved_12_15 : 4; + u64 aura : 12; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 aura : 12; + u64 reserved_12_15 : 4; + u64 stream_id : 8; + u64 reserved_24_30 : 7; + u64 inst_be : 1; + u64 size : 13; + u64 reserved_45_63 : 19; +#endif + } s; +}; + +static inline u64 ZIP_QUEX_SBUF_CTL(u64 param1) +{ + if (((param1 <= 7))) + return 0x1200ull + (param1 & 7) * 0x8ull; + pr_err("ZIP_QUEX_SBUF_CTL: %llu\n", param1); + return 0; +} + +/** + * union zip_que_ena - Represents queue enable register + * + * If a queue is disabled, ZIP_CTL stops fetching instructions from the queue. + */ +union zip_que_ena { + u64 u_reg64; + struct zip_que_ena_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_8_63 : 56; + u64 ena : 8; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 ena : 8; + u64 reserved_8_63 : 56; +#endif + } s; +}; + +#define ZIP_QUE_ENA 0x0500ull + +/** + * union zip_que_pri - Represents the register that defines the priority + * between instruction queues. + */ +union zip_que_pri { + u64 u_reg64; + struct zip_que_pri_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_8_63 : 56; + u64 pri : 8; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 pri : 8; + u64 reserved_8_63 : 56; +#endif + } s; +}; + +#define ZIP_QUE_PRI 0x0508ull + +/** + * union zip_throttle - Represents the register that controls the maximum + * number of in-flight X2I data fetch transactions. + * + * Writing 0 to this register causes the ZIP module to temporarily suspend NCB + * accesses; it is not recommended for normal operation, but may be useful for + * diagnostics. + */ +union zip_throttle { + u64 u_reg64; + struct zip_throttle_s { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_6_63 : 58; + u64 ld_infl : 6; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 ld_infl : 6; + u64 reserved_6_63 : 58; +#endif + } s; +}; + +#define ZIP_THROTTLE 0x0010ull + +#endif /* _CSRS_ZIP__ */ From f05845fcba12579ad84f58386b60adbfc14397b2 Mon Sep 17 00:00:00 2001 From: Mahipal Challa Date: Wed, 15 Feb 2017 10:45:09 +0530 Subject: [PATCH 008/101] crypto: zip - Wire-up Compression / decompression HW offload This contains changes for adding compression/decompression h/w offload functionality for both DEFLATE and LZS. Signed-off-by: Mahipal Challa Signed-off-by: Jan Glauber Signed-off-by: Herbert Xu --- drivers/crypto/cavium/zip/Makefile | 5 +- drivers/crypto/cavium/zip/zip_crypto.c | 313 ++++++++++++++++++++++++ drivers/crypto/cavium/zip/zip_crypto.h | 2 + drivers/crypto/cavium/zip/zip_deflate.c | 190 ++++++++++++++ drivers/crypto/cavium/zip/zip_deflate.h | 62 +++++ drivers/crypto/cavium/zip/zip_device.c | 1 + drivers/crypto/cavium/zip/zip_inflate.c | 211 ++++++++++++++++ drivers/crypto/cavium/zip/zip_inflate.h | 62 +++++ drivers/crypto/cavium/zip/zip_main.c | 57 ----- 9 files changed, 845 insertions(+), 58 deletions(-) create mode 100644 drivers/crypto/cavium/zip/zip_crypto.c create mode 100644 drivers/crypto/cavium/zip/zip_deflate.c create mode 100644 drivers/crypto/cavium/zip/zip_deflate.h create mode 100644 drivers/crypto/cavium/zip/zip_inflate.c create mode 100644 drivers/crypto/cavium/zip/zip_inflate.h diff --git a/drivers/crypto/cavium/zip/Makefile b/drivers/crypto/cavium/zip/Makefile index 2c0750885165..b2f3baaff757 100644 --- a/drivers/crypto/cavium/zip/Makefile +++ b/drivers/crypto/cavium/zip/Makefile @@ -5,4 +5,7 @@ obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += thunderx_zip.o thunderx_zip-y := zip_main.o \ zip_device.o \ - zip_mem.o + zip_crypto.o \ + zip_mem.o \ + zip_deflate.o \ + zip_inflate.o diff --git a/drivers/crypto/cavium/zip/zip_crypto.c b/drivers/crypto/cavium/zip/zip_crypto.c new file mode 100644 index 000000000000..8df4d26cf9d4 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_crypto.c @@ -0,0 +1,313 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#include "zip_crypto.h" + +static void zip_static_init_zip_ops(struct zip_operation *zip_ops, + int lzs_flag) +{ + zip_ops->flush = ZIP_FLUSH_FINISH; + + /* equivalent to level 6 of opensource zlib */ + zip_ops->speed = 1; + + if (!lzs_flag) { + zip_ops->ccode = 0; /* Auto Huffman */ + zip_ops->lzs_flag = 0; + zip_ops->format = ZLIB_FORMAT; + } else { + zip_ops->ccode = 3; /* LZS Encoding */ + zip_ops->lzs_flag = 1; + zip_ops->format = LZS_FORMAT; + } + zip_ops->begin_file = 1; + zip_ops->history_len = 0; + zip_ops->end_file = 1; + zip_ops->compcode = 0; + zip_ops->csum = 1; /* Adler checksum desired */ +} + +int zip_ctx_init(struct zip_kernel_ctx *zip_ctx, int lzs_flag) +{ + struct zip_operation *comp_ctx = &zip_ctx->zip_comp; + struct zip_operation *decomp_ctx = &zip_ctx->zip_decomp; + + zip_static_init_zip_ops(comp_ctx, lzs_flag); + zip_static_init_zip_ops(decomp_ctx, lzs_flag); + + comp_ctx->input = zip_data_buf_alloc(MAX_INPUT_BUFFER_SIZE); + if (!comp_ctx->input) + return -ENOMEM; + + comp_ctx->output = zip_data_buf_alloc(MAX_OUTPUT_BUFFER_SIZE); + if (!comp_ctx->output) + goto err_comp_input; + + decomp_ctx->input = zip_data_buf_alloc(MAX_INPUT_BUFFER_SIZE); + if (!decomp_ctx->input) + goto err_comp_output; + + decomp_ctx->output = zip_data_buf_alloc(MAX_OUTPUT_BUFFER_SIZE); + if (!decomp_ctx->output) + goto err_decomp_input; + + return 0; + +err_decomp_input: + zip_data_buf_free(decomp_ctx->input, MAX_INPUT_BUFFER_SIZE); + +err_comp_output: + zip_data_buf_free(comp_ctx->output, MAX_OUTPUT_BUFFER_SIZE); + +err_comp_input: + zip_data_buf_free(comp_ctx->input, MAX_INPUT_BUFFER_SIZE); + + return -ENOMEM; +} + +void zip_ctx_exit(struct zip_kernel_ctx *zip_ctx) +{ + struct zip_operation *comp_ctx = &zip_ctx->zip_comp; + struct zip_operation *dec_ctx = &zip_ctx->zip_decomp; + + zip_data_buf_free(comp_ctx->input, MAX_INPUT_BUFFER_SIZE); + zip_data_buf_free(comp_ctx->output, MAX_OUTPUT_BUFFER_SIZE); + + zip_data_buf_free(dec_ctx->input, MAX_INPUT_BUFFER_SIZE); + zip_data_buf_free(dec_ctx->output, MAX_OUTPUT_BUFFER_SIZE); +} + +int zip_compress(const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, + struct zip_kernel_ctx *zip_ctx) +{ + struct zip_operation *zip_ops = NULL; + struct zip_state zip_state; + struct zip_device *zip = NULL; + int ret; + + if (!zip_ctx || !src || !dst || !dlen) + return -ENOMEM; + + zip = zip_get_device(zip_get_node_id()); + if (!zip) + return -ENODEV; + + memset(&zip_state, 0, sizeof(struct zip_state)); + zip_ops = &zip_ctx->zip_comp; + + zip_ops->input_len = slen; + zip_ops->output_len = *dlen; + memcpy(zip_ops->input, src, slen); + + ret = zip_deflate(zip_ops, &zip_state, zip); + + if (!ret) { + *dlen = zip_ops->output_len; + memcpy(dst, zip_ops->output, *dlen); + } + + return ret; +} + +int zip_decompress(const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, + struct zip_kernel_ctx *zip_ctx) +{ + struct zip_operation *zip_ops = NULL; + struct zip_state zip_state; + struct zip_device *zip = NULL; + int ret; + + if (!zip_ctx || !src || !dst || !dlen) + return -ENOMEM; + + zip = zip_get_device(zip_get_node_id()); + if (!zip) + return -ENODEV; + + memset(&zip_state, 0, sizeof(struct zip_state)); + zip_ops = &zip_ctx->zip_decomp; + memcpy(zip_ops->input, src, slen); + + /* Work around for a bug in zlib which needs an extra bytes sometimes */ + if (zip_ops->ccode != 3) /* Not LZS Encoding */ + zip_ops->input[slen++] = 0; + + zip_ops->input_len = slen; + zip_ops->output_len = *dlen; + + ret = zip_inflate(zip_ops, &zip_state, zip); + + if (!ret) { + *dlen = zip_ops->output_len; + memcpy(dst, zip_ops->output, *dlen); + } + + return ret; +} + +/* Legacy Compress framework start */ +int zip_alloc_comp_ctx_deflate(struct crypto_tfm *tfm) +{ + int ret; + struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); + + ret = zip_ctx_init(zip_ctx, 0); + + return ret; +} + +int zip_alloc_comp_ctx_lzs(struct crypto_tfm *tfm) +{ + int ret; + struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); + + ret = zip_ctx_init(zip_ctx, 1); + + return ret; +} + +void zip_free_comp_ctx(struct crypto_tfm *tfm) +{ + struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); + + zip_ctx_exit(zip_ctx); +} + +int zip_comp_compress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen) +{ + int ret; + struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); + + ret = zip_compress(src, slen, dst, dlen, zip_ctx); + + return ret; +} + +int zip_comp_decompress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen) +{ + int ret; + struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); + + ret = zip_decompress(src, slen, dst, dlen, zip_ctx); + + return ret; +} /* Legacy compress framework end */ + +/* SCOMP framework start */ +void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm) +{ + int ret; + struct zip_kernel_ctx *zip_ctx; + + zip_ctx = kzalloc(sizeof(*zip_ctx), GFP_KERNEL); + if (!zip_ctx) + return ERR_PTR(-ENOMEM); + + ret = zip_ctx_init(zip_ctx, 0); + + if (ret) { + kzfree(zip_ctx); + return ERR_PTR(ret); + } + + return zip_ctx; +} + +void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm) +{ + int ret; + struct zip_kernel_ctx *zip_ctx; + + zip_ctx = kzalloc(sizeof(*zip_ctx), GFP_KERNEL); + if (!zip_ctx) + return ERR_PTR(-ENOMEM); + + ret = zip_ctx_init(zip_ctx, 1); + + if (ret) { + kzfree(zip_ctx); + return ERR_PTR(ret); + } + + return zip_ctx; +} + +void zip_free_scomp_ctx(struct crypto_scomp *tfm, void *ctx) +{ + struct zip_kernel_ctx *zip_ctx = ctx; + + zip_ctx_exit(zip_ctx); + kzfree(zip_ctx); +} + +int zip_scomp_compress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx) +{ + int ret; + struct zip_kernel_ctx *zip_ctx = ctx; + + ret = zip_compress(src, slen, dst, dlen, zip_ctx); + + return ret; +} + +int zip_scomp_decompress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx) +{ + int ret; + struct zip_kernel_ctx *zip_ctx = ctx; + + ret = zip_decompress(src, slen, dst, dlen, zip_ctx); + + return ret; +} /* SCOMP framework end */ diff --git a/drivers/crypto/cavium/zip/zip_crypto.h b/drivers/crypto/cavium/zip/zip_crypto.h index 2d91b05a94b8..b59ddfcacd34 100644 --- a/drivers/crypto/cavium/zip/zip_crypto.h +++ b/drivers/crypto/cavium/zip/zip_crypto.h @@ -49,6 +49,8 @@ #include #include #include "common.h" +#include "zip_deflate.h" +#include "zip_inflate.h" struct zip_kernel_ctx { struct zip_operation zip_comp; diff --git a/drivers/crypto/cavium/zip/zip_deflate.c b/drivers/crypto/cavium/zip/zip_deflate.c new file mode 100644 index 000000000000..b02789c09201 --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_deflate.c @@ -0,0 +1,190 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#include +#include + +#include "common.h" +#include "zip_deflate.h" + +/* Prepares the deflate zip command */ +static int prepare_zip_command(struct zip_operation *zip_ops, + struct zip_state *s, union zip_inst_s *zip_cmd) +{ + union zip_zres_s *result_ptr = &s->result; + + memset(zip_cmd, 0, sizeof(s->zip_cmd)); + memset(result_ptr, 0, sizeof(s->result)); + + /* IWORD #0 */ + /* History gather */ + zip_cmd->s.hg = 0; + /* compression enable = 1 for deflate */ + zip_cmd->s.ce = 1; + /* sf (sync flush) */ + zip_cmd->s.sf = 1; + /* ef (end of file) */ + if (zip_ops->flush == ZIP_FLUSH_FINISH) { + zip_cmd->s.ef = 1; + zip_cmd->s.sf = 0; + } + + zip_cmd->s.cc = zip_ops->ccode; + /* ss (compression speed/storage) */ + zip_cmd->s.ss = zip_ops->speed; + + /* IWORD #1 */ + /* adler checksum */ + zip_cmd->s.adlercrc32 = zip_ops->csum; + zip_cmd->s.historylength = zip_ops->history_len; + zip_cmd->s.dg = 0; + + /* IWORD # 6 and 7 - compression input/history pointer */ + zip_cmd->s.inp_ptr_addr.s.addr = __pa(zip_ops->input); + zip_cmd->s.inp_ptr_ctl.s.length = (zip_ops->input_len + + zip_ops->history_len); + zip_cmd->s.ds = 0; + + /* IWORD # 8 and 9 - Output pointer */ + zip_cmd->s.out_ptr_addr.s.addr = __pa(zip_ops->output); + zip_cmd->s.out_ptr_ctl.s.length = zip_ops->output_len; + /* maximum number of output-stream bytes that can be written */ + zip_cmd->s.totaloutputlength = zip_ops->output_len; + + /* IWORD # 10 and 11 - Result pointer */ + zip_cmd->s.res_ptr_addr.s.addr = __pa(result_ptr); + /* Clearing completion code */ + result_ptr->s.compcode = 0; + + return 0; +} + +/** + * zip_deflate - API to offload deflate operation to hardware + * @zip_ops: Pointer to zip operation structure + * @s: Pointer to the structure representing zip state + * @zip_dev: Pointer to zip device structure + * + * This function prepares the zip deflate command and submits it to the zip + * engine for processing. + * + * Return: 0 if successful or error code + */ +int zip_deflate(struct zip_operation *zip_ops, struct zip_state *s, + struct zip_device *zip_dev) +{ + union zip_inst_s *zip_cmd = &s->zip_cmd; + union zip_zres_s *result_ptr = &s->result; + u32 queue; + + /* Prepares zip command based on the input parameters */ + prepare_zip_command(zip_ops, s, zip_cmd); + + /* Loads zip command into command queues and rings door bell */ + queue = zip_load_instr(zip_cmd, zip_dev); + + while (!result_ptr->s.compcode) + continue; + + zip_ops->compcode = result_ptr->s.compcode; + switch (zip_ops->compcode) { + case ZIP_CMD_NOTDONE: + zip_dbg("Zip instruction not yet completed"); + return ZIP_ERROR; + + case ZIP_CMD_SUCCESS: + zip_dbg("Zip instruction completed successfully"); + zip_update_cmd_bufs(zip_dev, queue); + break; + + case ZIP_CMD_DTRUNC: + zip_dbg("Output Truncate error"); + /* Returning ZIP_ERROR to avoid copy to user */ + return ZIP_ERROR; + + default: + zip_err("Zip instruction failed. Code:%d", zip_ops->compcode); + return ZIP_ERROR; + } + + /* Update the CRC depending on the format */ + switch (zip_ops->format) { + case RAW_FORMAT: + zip_dbg("RAW Format: %d ", zip_ops->format); + /* Get checksum from engine, need to feed it again */ + zip_ops->csum = result_ptr->s.adler32; + break; + + case ZLIB_FORMAT: + zip_dbg("ZLIB Format: %d ", zip_ops->format); + zip_ops->csum = result_ptr->s.adler32; + break; + + case GZIP_FORMAT: + zip_dbg("GZIP Format: %d ", zip_ops->format); + zip_ops->csum = result_ptr->s.crc32; + break; + + case LZS_FORMAT: + zip_dbg("LZS Format: %d ", zip_ops->format); + break; + + default: + zip_err("Unknown Format:%d\n", zip_ops->format); + } + + /* Update output_len */ + if (zip_ops->output_len < result_ptr->s.totalbyteswritten) { + /* Dynamic stop && strm->output_len < zipconstants[onfsize] */ + zip_err("output_len (%d) < total bytes written(%d)\n", + zip_ops->output_len, result_ptr->s.totalbyteswritten); + zip_ops->output_len = 0; + + } else { + zip_ops->output_len = result_ptr->s.totalbyteswritten; + } + + return 0; +} diff --git a/drivers/crypto/cavium/zip/zip_deflate.h b/drivers/crypto/cavium/zip/zip_deflate.h new file mode 100644 index 000000000000..1d32e76edc4d --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_deflate.h @@ -0,0 +1,62 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_DEFLATE_H__ +#define __ZIP_DEFLATE_H__ + +/** + * zip_deflate - API to offload deflate operation to hardware + * @zip_ops: Pointer to zip operation structure + * @s: Pointer to the structure representing zip state + * @zip_dev: Pointer to the structure representing zip device + * + * This function prepares the zip deflate command and submits it to the zip + * engine by ringing the doorbell. + * + * Return: 0 if successful or error code + */ +int zip_deflate(struct zip_operation *zip_ops, struct zip_state *s, + struct zip_device *zip_dev); +#endif diff --git a/drivers/crypto/cavium/zip/zip_device.c b/drivers/crypto/cavium/zip/zip_device.c index e980f0894546..ccf21fb91513 100644 --- a/drivers/crypto/cavium/zip/zip_device.c +++ b/drivers/crypto/cavium/zip/zip_device.c @@ -44,6 +44,7 @@ ***********************license end**************************************/ #include "common.h" +#include "zip_deflate.h" /** * zip_cmd_queue_consumed - Calculates the space consumed in the command queue. diff --git a/drivers/crypto/cavium/zip/zip_inflate.c b/drivers/crypto/cavium/zip/zip_inflate.c new file mode 100644 index 000000000000..5f43a8cb982b --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_inflate.c @@ -0,0 +1,211 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#include +#include + +#include "common.h" +#include "zip_inflate.h" + +static int prepare_inflate_zcmd(struct zip_operation *zip_ops, + struct zip_state *s, union zip_inst_s *zip_cmd) +{ + union zip_zres_s *result_ptr = &s->result; + + memset(zip_cmd, 0, sizeof(s->zip_cmd)); + memset(result_ptr, 0, sizeof(s->result)); + + /* IWORD#0 */ + + /* Decompression History Gather list - no gather list */ + zip_cmd->s.hg = 0; + /* For decompression, CE must be 0x0. */ + zip_cmd->s.ce = 0; + /* For decompression, SS must be 0x0. */ + zip_cmd->s.ss = 0; + /* For decompression, SF should always be set. */ + zip_cmd->s.sf = 1; + + /* Begin File */ + if (zip_ops->begin_file == 0) + zip_cmd->s.bf = 0; + else + zip_cmd->s.bf = 1; + + zip_cmd->s.ef = 1; + /* 0: for Deflate decompression, 3: for LZS decompression */ + zip_cmd->s.cc = zip_ops->ccode; + + /* IWORD #1*/ + + /* adler checksum */ + zip_cmd->s.adlercrc32 = zip_ops->csum; + + /* + * HISTORYLENGTH must be 0x0 for any ZIP decompress operation. + * History data is added to a decompression operation via IWORD3. + */ + zip_cmd->s.historylength = 0; + zip_cmd->s.ds = 0; + + /* IWORD # 8 and 9 - Output pointer */ + zip_cmd->s.out_ptr_addr.s.addr = __pa(zip_ops->output); + zip_cmd->s.out_ptr_ctl.s.length = zip_ops->output_len; + + /* Maximum number of output-stream bytes that can be written */ + zip_cmd->s.totaloutputlength = zip_ops->output_len; + + zip_dbg("Data Direct Input case "); + + /* IWORD # 6 and 7 - input pointer */ + zip_cmd->s.dg = 0; + zip_cmd->s.inp_ptr_addr.s.addr = __pa((u8 *)zip_ops->input); + zip_cmd->s.inp_ptr_ctl.s.length = zip_ops->input_len; + + /* IWORD # 10 and 11 - Result pointer */ + zip_cmd->s.res_ptr_addr.s.addr = __pa(result_ptr); + + /* Clearing completion code */ + result_ptr->s.compcode = 0; + + /* Returning 0 for time being.*/ + return 0; +} + +/** + * zip_inflate - API to offload inflate operation to hardware + * @zip_ops: Pointer to zip operation structure + * @s: Pointer to the structure representing zip state + * @zip_dev: Pointer to zip device structure + * + * This function prepares the zip inflate command and submits it to the zip + * engine for processing. + * + * Return: 0 if successful or error code + */ +int zip_inflate(struct zip_operation *zip_ops, struct zip_state *s, + struct zip_device *zip_dev) +{ + union zip_inst_s *zip_cmd = &s->zip_cmd; + union zip_zres_s *result_ptr = &s->result; + u32 queue; + + /* Prepare inflate zip command */ + prepare_inflate_zcmd(zip_ops, s, zip_cmd); + + /* Load inflate command to zip queue and ring the doorbell */ + queue = zip_load_instr(zip_cmd, zip_dev); + + while (!result_ptr->s.compcode) + continue; + + zip_ops->compcode = result_ptr->s.compcode; + switch (zip_ops->compcode) { + case ZIP_CMD_NOTDONE: + zip_dbg("Zip Instruction not yet completed\n"); + return ZIP_ERROR; + + case ZIP_CMD_SUCCESS: + zip_dbg("Zip Instruction completed successfully\n"); + break; + + case ZIP_CMD_DYNAMIC_STOP: + zip_dbg(" Dynamic stop Initiated\n"); + break; + + default: + zip_dbg("Instruction failed. Code = %d\n", zip_ops->compcode); + zip_update_cmd_bufs(zip_dev, queue); + return ZIP_ERROR; + } + + zip_update_cmd_bufs(zip_dev, queue); + + if ((zip_ops->ccode == 3) && (zip_ops->flush == 4) && + (zip_ops->compcode != ZIP_CMD_DYNAMIC_STOP)) + result_ptr->s.ef = 1; + + zip_ops->csum = result_ptr->s.adler32; + + if (zip_ops->output_len < result_ptr->s.totalbyteswritten) { + zip_err("output_len (%d) < total bytes written (%d)\n", + zip_ops->output_len, result_ptr->s.totalbyteswritten); + zip_ops->output_len = 0; + } else { + zip_ops->output_len = result_ptr->s.totalbyteswritten; + } + + zip_ops->bytes_read = result_ptr->s.totalbytesread; + zip_ops->bits_processed = result_ptr->s.totalbitsprocessed; + zip_ops->end_file = result_ptr->s.ef; + if (zip_ops->end_file) { + switch (zip_ops->format) { + case RAW_FORMAT: + zip_dbg("RAW Format: %d ", zip_ops->format); + /* Get checksum from engine */ + zip_ops->csum = result_ptr->s.adler32; + break; + + case ZLIB_FORMAT: + zip_dbg("ZLIB Format: %d ", zip_ops->format); + zip_ops->csum = result_ptr->s.adler32; + break; + + case GZIP_FORMAT: + zip_dbg("GZIP Format: %d ", zip_ops->format); + zip_ops->csum = result_ptr->s.crc32; + break; + + case LZS_FORMAT: + zip_dbg("LZS Format: %d ", zip_ops->format); + break; + + default: + zip_err("Format error:%d\n", zip_ops->format); + } + } + + return 0; +} diff --git a/drivers/crypto/cavium/zip/zip_inflate.h b/drivers/crypto/cavium/zip/zip_inflate.h new file mode 100644 index 000000000000..6b20f179978e --- /dev/null +++ b/drivers/crypto/cavium/zip/zip_inflate.h @@ -0,0 +1,62 @@ +/***********************license start************************************ + * Copyright (c) 2003-2017 Cavium, Inc. + * All rights reserved. + * + * License: one of 'Cavium License' or 'GNU General Public License Version 2' + * + * This file is provided under the terms of the Cavium License (see below) + * or under the terms of GNU General Public License, Version 2, as + * published by the Free Software Foundation. When using or redistributing + * this file, you may do so under either license. + * + * Cavium License: Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of Cavium Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import + * regulations in other countries. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" + * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS + * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH + * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY + * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT + * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) + * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A + * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET + * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE + * ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES + * WITH YOU. + ***********************license end**************************************/ + +#ifndef __ZIP_INFLATE_H__ +#define __ZIP_INFLATE_H__ + +/** + * zip_inflate - API to offload inflate operation to hardware + * @zip_ops: Pointer to zip operation structure + * @s: Pointer to the structure representing zip state + * @zip_dev: Pointer to the structure representing zip device + * + * This function prepares the zip inflate command and submits it to the zip + * engine for processing. + * + * Return: 0 if successful or error code + */ +int zip_inflate(struct zip_operation *zip_ops, struct zip_state *s, + struct zip_device *zip_dev); +#endif diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c index 89ea360d410d..dd5814a12c9a 100644 --- a/drivers/crypto/cavium/zip/zip_main.c +++ b/drivers/crypto/cavium/zip/zip_main.c @@ -339,63 +339,6 @@ static void zip_remove(struct pci_dev *pdev) zip_dev[zip->index] = NULL; } -/* Dummy Functions */ -int zip_alloc_comp_ctx_deflate(struct crypto_tfm *tfm) -{ - return 0; -} - -int zip_alloc_comp_ctx_lzs(struct crypto_tfm *tfm) -{ - return 0; -} - -void zip_free_comp_ctx(struct crypto_tfm *tfm) -{ -} - -int zip_comp_compress(struct crypto_tfm *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen) -{ - return 0; -} - -int zip_comp_decompress(struct crypto_tfm *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen) -{ - return 0; -} - -void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm) -{ - return NULL; -} - -void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm) -{ - return NULL; -} - -void zip_free_scomp_ctx(struct crypto_scomp *tfm, void *zip_ctx) -{ -} - -int zip_scomp_compress(struct crypto_scomp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx) -{ - return 0; -} - -int zip_scomp_decompress(struct crypto_scomp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen, void *ctx) -{ - return 0; -} - /* PCI Sub-System Interface */ static struct pci_driver zip_driver = { .name = DRV_NAME, From 09ae5d37e0935c8f0bcb63d80543347daa209e25 Mon Sep 17 00:00:00 2001 From: Mahipal Challa Date: Wed, 15 Feb 2017 10:45:10 +0530 Subject: [PATCH 009/101] crypto: zip - Add Compression/Decompression statistics Add statistics for compression/decompression hardware offload under debugfs. Signed-off-by: Mahipal Challa Signed-off-by: Jan Glauber Signed-off-by: Herbert Xu --- drivers/crypto/cavium/zip/zip_deflate.c | 10 + drivers/crypto/cavium/zip/zip_inflate.c | 12 ++ drivers/crypto/cavium/zip/zip_main.c | 234 ++++++++++++++++++++++++ drivers/crypto/cavium/zip/zip_main.h | 15 ++ 4 files changed, 271 insertions(+) diff --git a/drivers/crypto/cavium/zip/zip_deflate.c b/drivers/crypto/cavium/zip/zip_deflate.c index b02789c09201..9a944b8c1e29 100644 --- a/drivers/crypto/cavium/zip/zip_deflate.c +++ b/drivers/crypto/cavium/zip/zip_deflate.c @@ -122,12 +122,19 @@ int zip_deflate(struct zip_operation *zip_ops, struct zip_state *s, /* Prepares zip command based on the input parameters */ prepare_zip_command(zip_ops, s, zip_cmd); + atomic64_add(zip_ops->input_len, &zip_dev->stats.comp_in_bytes); /* Loads zip command into command queues and rings door bell */ queue = zip_load_instr(zip_cmd, zip_dev); + /* Stats update for compression requests submitted */ + atomic64_inc(&zip_dev->stats.comp_req_submit); + while (!result_ptr->s.compcode) continue; + /* Stats update for compression requests completed */ + atomic64_inc(&zip_dev->stats.comp_req_complete); + zip_ops->compcode = result_ptr->s.compcode; switch (zip_ops->compcode) { case ZIP_CMD_NOTDONE: @@ -175,6 +182,9 @@ int zip_deflate(struct zip_operation *zip_ops, struct zip_state *s, zip_err("Unknown Format:%d\n", zip_ops->format); } + atomic64_add(result_ptr->s.totalbyteswritten, + &zip_dev->stats.comp_out_bytes); + /* Update output_len */ if (zip_ops->output_len < result_ptr->s.totalbyteswritten) { /* Dynamic stop && strm->output_len < zipconstants[onfsize] */ diff --git a/drivers/crypto/cavium/zip/zip_inflate.c b/drivers/crypto/cavium/zip/zip_inflate.c index 5f43a8cb982b..50cbdd83dbf2 100644 --- a/drivers/crypto/cavium/zip/zip_inflate.c +++ b/drivers/crypto/cavium/zip/zip_inflate.c @@ -135,12 +135,20 @@ int zip_inflate(struct zip_operation *zip_ops, struct zip_state *s, /* Prepare inflate zip command */ prepare_inflate_zcmd(zip_ops, s, zip_cmd); + atomic64_add(zip_ops->input_len, &zip_dev->stats.decomp_in_bytes); + /* Load inflate command to zip queue and ring the doorbell */ queue = zip_load_instr(zip_cmd, zip_dev); + /* Decompression requests submitted stats update */ + atomic64_inc(&zip_dev->stats.decomp_req_submit); + while (!result_ptr->s.compcode) continue; + /* Decompression requests completed stats update */ + atomic64_inc(&zip_dev->stats.decomp_req_complete); + zip_ops->compcode = result_ptr->s.compcode; switch (zip_ops->compcode) { case ZIP_CMD_NOTDONE: @@ -157,6 +165,7 @@ int zip_inflate(struct zip_operation *zip_ops, struct zip_state *s, default: zip_dbg("Instruction failed. Code = %d\n", zip_ops->compcode); + atomic64_inc(&zip_dev->stats.decomp_bad_reqs); zip_update_cmd_bufs(zip_dev, queue); return ZIP_ERROR; } @@ -169,6 +178,9 @@ int zip_inflate(struct zip_operation *zip_ops, struct zip_state *s, zip_ops->csum = result_ptr->s.adler32; + atomic64_add(result_ptr->s.totalbyteswritten, + &zip_dev->stats.decomp_out_bytes); + if (zip_ops->output_len < result_ptr->s.totalbyteswritten) { zip_err("output_len (%d) < total bytes written (%d)\n", zip_ops->output_len, result_ptr->s.totalbyteswritten); diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c index dd5814a12c9a..0951e20b395b 100644 --- a/drivers/crypto/cavium/zip/zip_main.c +++ b/drivers/crypto/cavium/zip/zip_main.c @@ -451,6 +451,228 @@ static void zip_unregister_compression_device(void) crypto_unregister_scomp(&zip_scomp_lzs); } +/* + * debugfs functions + */ +#ifdef CONFIG_DEBUG_FS +#include + +/* Displays ZIP device statistics */ +static int zip_show_stats(struct seq_file *s, void *unused) +{ + u64 val = 0ull; + u64 avg_chunk = 0ull, avg_cr = 0ull; + u32 q = 0; + + int index = 0; + struct zip_device *zip; + struct zip_stats *st; + + for (index = 0; index < MAX_ZIP_DEVICES; index++) { + if (zip_dev[index]) { + zip = zip_dev[index]; + st = &zip->stats; + + /* Get all the pending requests */ + for (q = 0; q < ZIP_NUM_QUEUES; q++) { + val = zip_reg_read((zip->reg_base + + ZIP_DBG_COREX_STA(q))); + val = (val >> 32); + val = val & 0xffffff; + atomic64_add(val, &st->pending_req); + } + + avg_chunk = (atomic64_read(&st->comp_in_bytes) / + atomic64_read(&st->comp_req_complete)); + avg_cr = (atomic64_read(&st->comp_in_bytes) / + atomic64_read(&st->comp_out_bytes)); + seq_printf(s, " ZIP Device %d Stats\n" + "-----------------------------------\n" + "Comp Req Submitted : \t%ld\n" + "Comp Req Completed : \t%ld\n" + "Compress In Bytes : \t%ld\n" + "Compressed Out Bytes : \t%ld\n" + "Average Chunk size : \t%llu\n" + "Average Compression ratio : \t%llu\n" + "Decomp Req Submitted : \t%ld\n" + "Decomp Req Completed : \t%ld\n" + "Decompress In Bytes : \t%ld\n" + "Decompressed Out Bytes : \t%ld\n" + "Decompress Bad requests : \t%ld\n" + "Pending Req : \t%ld\n" + "---------------------------------\n", + index, + atomic64_read(&st->comp_req_submit), + atomic64_read(&st->comp_req_complete), + atomic64_read(&st->comp_in_bytes), + atomic64_read(&st->comp_out_bytes), + avg_chunk, + avg_cr, + atomic64_read(&st->decomp_req_submit), + atomic64_read(&st->decomp_req_complete), + atomic64_read(&st->decomp_in_bytes), + atomic64_read(&st->decomp_out_bytes), + atomic64_read(&st->decomp_bad_reqs), + atomic64_read(&st->pending_req)); + + /* Reset pending requests count */ + atomic64_set(&st->pending_req, 0); + } + } + return 0; +} + +/* Clears stats data */ +static int zip_clear_stats(struct seq_file *s, void *unused) +{ + int index = 0; + + for (index = 0; index < MAX_ZIP_DEVICES; index++) { + if (zip_dev[index]) { + memset(&zip_dev[index]->stats, 0, + sizeof(struct zip_state)); + seq_printf(s, "Cleared stats for zip %d\n", index); + } + } + + return 0; +} + +static struct zip_registers zipregs[64] = { + {"ZIP_CMD_CTL ", 0x0000ull}, + {"ZIP_THROTTLE ", 0x0010ull}, + {"ZIP_CONSTANTS ", 0x00A0ull}, + {"ZIP_QUE0_MAP ", 0x1400ull}, + {"ZIP_QUE1_MAP ", 0x1408ull}, + {"ZIP_QUE_ENA ", 0x0500ull}, + {"ZIP_QUE_PRI ", 0x0508ull}, + {"ZIP_QUE0_DONE ", 0x2000ull}, + {"ZIP_QUE1_DONE ", 0x2008ull}, + {"ZIP_QUE0_DOORBELL ", 0x4000ull}, + {"ZIP_QUE1_DOORBELL ", 0x4008ull}, + {"ZIP_QUE0_SBUF_ADDR ", 0x1000ull}, + {"ZIP_QUE1_SBUF_ADDR ", 0x1008ull}, + {"ZIP_QUE0_SBUF_CTL ", 0x1200ull}, + {"ZIP_QUE1_SBUF_CTL ", 0x1208ull}, + { NULL, 0} +}; + +/* Prints registers' contents */ +static int zip_print_regs(struct seq_file *s, void *unused) +{ + u64 val = 0; + int i = 0, index = 0; + + for (index = 0; index < MAX_ZIP_DEVICES; index++) { + if (zip_dev[index]) { + seq_printf(s, "--------------------------------\n" + " ZIP Device %d Registers\n" + "--------------------------------\n", + index); + + i = 0; + + while (zipregs[i].reg_name) { + val = zip_reg_read((zip_dev[index]->reg_base + + zipregs[i].reg_offset)); + seq_printf(s, "%s: 0x%016llx\n", + zipregs[i].reg_name, val); + i++; + } + } + } + return 0; +} + +static int zip_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, zip_show_stats, NULL); +} + +static const struct file_operations zip_stats_fops = { + .owner = THIS_MODULE, + .open = zip_stats_open, + .read = seq_read, +}; + +static int zip_clear_open(struct inode *inode, struct file *file) +{ + return single_open(file, zip_clear_stats, NULL); +} + +static const struct file_operations zip_clear_fops = { + .owner = THIS_MODULE, + .open = zip_clear_open, + .read = seq_read, +}; + +static int zip_regs_open(struct inode *inode, struct file *file) +{ + return single_open(file, zip_print_regs, NULL); +} + +static const struct file_operations zip_regs_fops = { + .owner = THIS_MODULE, + .open = zip_regs_open, + .read = seq_read, +}; + +/* Root directory for thunderx_zip debugfs entry */ +static struct dentry *zip_debugfs_root; + +static int __init zip_debugfs_init(void) +{ + struct dentry *zip_stats, *zip_clear, *zip_regs; + + if (!debugfs_initialized()) + return -ENODEV; + + zip_debugfs_root = debugfs_create_dir("thunderx_zip", NULL); + if (!zip_debugfs_root) + return -ENOMEM; + + /* Creating files for entries inside thunderx_zip directory */ + zip_stats = debugfs_create_file("zip_stats", 0444, + zip_debugfs_root, + NULL, &zip_stats_fops); + if (!zip_stats) + goto failed_to_create; + + zip_clear = debugfs_create_file("zip_clear", 0444, + zip_debugfs_root, + NULL, &zip_clear_fops); + if (!zip_clear) + goto failed_to_create; + + zip_regs = debugfs_create_file("zip_regs", 0444, + zip_debugfs_root, + NULL, &zip_regs_fops); + if (!zip_regs) + goto failed_to_create; + + return 0; + +failed_to_create: + debugfs_remove_recursive(zip_debugfs_root); + return -ENOENT; +} + +static void __exit zip_debugfs_exit(void) +{ + debugfs_remove_recursive(zip_debugfs_root); +} + +#else +static int __init zip_debugfs_init(void) +{ + return 0; +} + +static void __exit zip_debugfs_exit(void) { } + +#endif +/* debugfs - end */ + static int __init zip_init_module(void) { int ret; @@ -470,8 +692,18 @@ static int __init zip_init_module(void) goto err_pci_unregister; } + /* comp-decomp statistics are handled with debugfs interface */ + ret = zip_debugfs_init(); + if (ret < 0) { + zip_err("ZIP: debugfs initialization failed\n"); + goto err_crypto_unregister; + } + return ret; +err_crypto_unregister: + zip_unregister_compression_device(); + err_pci_unregister: pci_unregister_driver(&zip_driver); return ret; @@ -479,6 +711,8 @@ err_pci_unregister: static void __exit zip_cleanup_module(void) { + zip_debugfs_exit(); + /* Unregister from the kernel crypto interface */ zip_unregister_compression_device(); diff --git a/drivers/crypto/cavium/zip/zip_main.h b/drivers/crypto/cavium/zip/zip_main.h index 1775c77d2a48..64e051f60784 100644 --- a/drivers/crypto/cavium/zip/zip_main.h +++ b/drivers/crypto/cavium/zip/zip_main.h @@ -68,6 +68,20 @@ struct zip_registers { u64 reg_offset; }; +/* ZIP Compression - Decompression stats */ +struct zip_stats { + atomic64_t comp_req_submit; + atomic64_t comp_req_complete; + atomic64_t decomp_req_submit; + atomic64_t decomp_req_complete; + atomic64_t pending_req; + atomic64_t comp_in_bytes; + atomic64_t comp_out_bytes; + atomic64_t decomp_in_bytes; + atomic64_t decomp_out_bytes; + atomic64_t decomp_bad_reqs; +}; + /* ZIP Instruction Queue */ struct zip_iq { u64 *sw_head; @@ -93,6 +107,7 @@ struct zip_device { u64 ctxsize; struct zip_iq iq[ZIP_MAX_NUM_QUEUES]; + struct zip_stats stats; }; /* Prototypes */ From cd220ae6c7fbb0c80c3301e011e1baf4f352e497 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 22 Feb 2017 23:51:38 +0000 Subject: [PATCH 010/101] crypto: bcm - fix spelling mistake: "genereate" -> "generate" trivial fix to spelling mistake in pr_err message Signed-off-by: Colin Ian King Acked-by: Steve Lin Signed-off-by: Herbert Xu --- drivers/crypto/bcm/util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/bcm/util.c b/drivers/crypto/bcm/util.c index 0502f460dacd..430c5570ea87 100644 --- a/drivers/crypto/bcm/util.c +++ b/drivers/crypto/bcm/util.c @@ -312,7 +312,7 @@ int do_shash(unsigned char *name, unsigned char *result, } rc = crypto_shash_final(&sdesc->shash, result); if (rc) - pr_err("%s: Could not genereate %s hash", __func__, name); + pr_err("%s: Could not generate %s hash", __func__, name); do_shash_err: crypto_free_shash(hash); From 5527dfb6ddac2aac98c2939f27840cb47abd5693 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 24 Feb 2017 15:46:58 -0800 Subject: [PATCH 011/101] crypto: kpp - constify buffer passed to crypto_kpp_set_secret() Constify the buffer passed to crypto_kpp_set_secret() and kpp_alg.set_secret, since it is never modified. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/dh.c | 3 ++- crypto/ecdh.c | 3 ++- drivers/crypto/qat/qat_common/qat_asym_algs.c | 2 +- include/crypto/kpp.h | 6 +++--- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crypto/dh.c b/crypto/dh.c index ddcb528ab2cc..87e3542cf1b8 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -79,7 +79,8 @@ static int dh_set_params(struct dh_ctx *ctx, struct dh *params) return 0; } -static int dh_set_secret(struct crypto_kpp *tfm, void *buf, unsigned int len) +static int dh_set_secret(struct crypto_kpp *tfm, const void *buf, + unsigned int len) { struct dh_ctx *ctx = dh_get_ctx(tfm); struct dh params; diff --git a/crypto/ecdh.c b/crypto/ecdh.c index 3de289806d67..63ca33771e4e 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -38,7 +38,8 @@ static unsigned int ecdh_supported_curve(unsigned int curve_id) } } -static int ecdh_set_secret(struct crypto_kpp *tfm, void *buf, unsigned int len) +static int ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, + unsigned int len) { struct ecdh_ctx *ctx = ecdh_get_ctx(tfm); struct ecdh params; diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 0d35dca2e925..2aab80bc241f 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -491,7 +491,7 @@ static void qat_dh_clear_ctx(struct device *dev, struct qat_dh_ctx *ctx) ctx->g2 = false; } -static int qat_dh_set_secret(struct crypto_kpp *tfm, void *buf, +static int qat_dh_set_secret(struct crypto_kpp *tfm, const void *buf, unsigned int len) { struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); diff --git a/include/crypto/kpp.h b/include/crypto/kpp.h index 4307a2f2365f..ce8e1f79374b 100644 --- a/include/crypto/kpp.h +++ b/include/crypto/kpp.h @@ -74,7 +74,7 @@ struct crypto_kpp { * @base: Common crypto API algorithm data structure */ struct kpp_alg { - int (*set_secret)(struct crypto_kpp *tfm, void *buffer, + int (*set_secret)(struct crypto_kpp *tfm, const void *buffer, unsigned int len); int (*generate_public_key)(struct kpp_request *req); int (*compute_shared_secret)(struct kpp_request *req); @@ -273,8 +273,8 @@ struct kpp_secret { * * Return: zero on success; error code in case of error */ -static inline int crypto_kpp_set_secret(struct crypto_kpp *tfm, void *buffer, - unsigned int len) +static inline int crypto_kpp_set_secret(struct crypto_kpp *tfm, + const void *buffer, unsigned int len) { struct kpp_alg *alg = crypto_kpp_alg(tfm); From b13b1e0c6b171b4f6ad94dd99020708719983494 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 24 Feb 2017 15:46:59 -0800 Subject: [PATCH 012/101] crypto: testmgr - constify all test vectors Cryptographic test vectors should never be modified, so constify them to enforce this at both compile-time and run-time. This moves a significant amount of data from .data to .rodata when the crypto tests are enabled. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/testmgr.c | 71 ++++--- crypto/testmgr.h | 512 +++++++++++++++++++++++------------------------ 2 files changed, 297 insertions(+), 286 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index f9c378af3907..89f1dd1f4b13 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -83,47 +83,47 @@ struct tcrypt_result { struct aead_test_suite { struct { - struct aead_testvec *vecs; + const struct aead_testvec *vecs; unsigned int count; } enc, dec; }; struct cipher_test_suite { struct { - struct cipher_testvec *vecs; + const struct cipher_testvec *vecs; unsigned int count; } enc, dec; }; struct comp_test_suite { struct { - struct comp_testvec *vecs; + const struct comp_testvec *vecs; unsigned int count; } comp, decomp; }; struct hash_test_suite { - struct hash_testvec *vecs; + const struct hash_testvec *vecs; unsigned int count; }; struct cprng_test_suite { - struct cprng_testvec *vecs; + const struct cprng_testvec *vecs; unsigned int count; }; struct drbg_test_suite { - struct drbg_testvec *vecs; + const struct drbg_testvec *vecs; unsigned int count; }; struct akcipher_test_suite { - struct akcipher_testvec *vecs; + const struct akcipher_testvec *vecs; unsigned int count; }; struct kpp_test_suite { - struct kpp_testvec *vecs; + const struct kpp_testvec *vecs; unsigned int count; }; @@ -145,7 +145,8 @@ struct alg_test_desc { } suite; }; -static unsigned int IDX[8] = { IDX1, IDX2, IDX3, IDX4, IDX5, IDX6, IDX7, IDX8 }; +static const unsigned int IDX[8] = { + IDX1, IDX2, IDX3, IDX4, IDX5, IDX6, IDX7, IDX8 }; static void hexdump(unsigned char *buf, unsigned int len) { @@ -203,7 +204,7 @@ static int wait_async_op(struct tcrypt_result *tr, int ret) } static int ahash_partial_update(struct ahash_request **preq, - struct crypto_ahash *tfm, struct hash_testvec *template, + struct crypto_ahash *tfm, const struct hash_testvec *template, void *hash_buff, int k, int temp, struct scatterlist *sg, const char *algo, char *result, struct tcrypt_result *tresult) { @@ -260,9 +261,9 @@ out_nostate: return ret; } -static int __test_hash(struct crypto_ahash *tfm, struct hash_testvec *template, - unsigned int tcount, bool use_digest, - const int align_offset) +static int __test_hash(struct crypto_ahash *tfm, + const struct hash_testvec *template, unsigned int tcount, + bool use_digest, const int align_offset) { const char *algo = crypto_tfm_alg_driver_name(crypto_ahash_tfm(tfm)); size_t digest_size = crypto_ahash_digestsize(tfm); @@ -538,7 +539,8 @@ out_nobuf: return ret; } -static int test_hash(struct crypto_ahash *tfm, struct hash_testvec *template, +static int test_hash(struct crypto_ahash *tfm, + const struct hash_testvec *template, unsigned int tcount, bool use_digest) { unsigned int alignmask; @@ -566,7 +568,7 @@ static int test_hash(struct crypto_ahash *tfm, struct hash_testvec *template, } static int __test_aead(struct crypto_aead *tfm, int enc, - struct aead_testvec *template, unsigned int tcount, + const struct aead_testvec *template, unsigned int tcount, const bool diff_dst, const int align_offset) { const char *algo = crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)); @@ -957,7 +959,7 @@ out_noxbuf: } static int test_aead(struct crypto_aead *tfm, int enc, - struct aead_testvec *template, unsigned int tcount) + const struct aead_testvec *template, unsigned int tcount) { unsigned int alignmask; int ret; @@ -990,7 +992,8 @@ static int test_aead(struct crypto_aead *tfm, int enc, } static int test_cipher(struct crypto_cipher *tfm, int enc, - struct cipher_testvec *template, unsigned int tcount) + const struct cipher_testvec *template, + unsigned int tcount) { const char *algo = crypto_tfm_alg_driver_name(crypto_cipher_tfm(tfm)); unsigned int i, j, k; @@ -1068,7 +1071,8 @@ out_nobuf: } static int __test_skcipher(struct crypto_skcipher *tfm, int enc, - struct cipher_testvec *template, unsigned int tcount, + const struct cipher_testvec *template, + unsigned int tcount, const bool diff_dst, const int align_offset) { const char *algo = @@ -1332,7 +1336,8 @@ out_nobuf: } static int test_skcipher(struct crypto_skcipher *tfm, int enc, - struct cipher_testvec *template, unsigned int tcount) + const struct cipher_testvec *template, + unsigned int tcount) { unsigned int alignmask; int ret; @@ -1364,8 +1369,10 @@ static int test_skcipher(struct crypto_skcipher *tfm, int enc, return 0; } -static int test_comp(struct crypto_comp *tfm, struct comp_testvec *ctemplate, - struct comp_testvec *dtemplate, int ctcount, int dtcount) +static int test_comp(struct crypto_comp *tfm, + const struct comp_testvec *ctemplate, + const struct comp_testvec *dtemplate, + int ctcount, int dtcount) { const char *algo = crypto_tfm_alg_driver_name(crypto_comp_tfm(tfm)); unsigned int i; @@ -1444,8 +1451,10 @@ out: return ret; } -static int test_acomp(struct crypto_acomp *tfm, struct comp_testvec *ctemplate, - struct comp_testvec *dtemplate, int ctcount, int dtcount) +static int test_acomp(struct crypto_acomp *tfm, + const struct comp_testvec *ctemplate, + const struct comp_testvec *dtemplate, + int ctcount, int dtcount) { const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)); unsigned int i; @@ -1588,7 +1597,8 @@ out: return ret; } -static int test_cprng(struct crypto_rng *tfm, struct cprng_testvec *template, +static int test_cprng(struct crypto_rng *tfm, + const struct cprng_testvec *template, unsigned int tcount) { const char *algo = crypto_tfm_alg_driver_name(crypto_rng_tfm(tfm)); @@ -1865,7 +1875,7 @@ static int alg_test_cprng(const struct alg_test_desc *desc, const char *driver, } -static int drbg_cavs_test(struct drbg_testvec *test, int pr, +static int drbg_cavs_test(const struct drbg_testvec *test, int pr, const char *driver, u32 type, u32 mask) { int ret = -EAGAIN; @@ -1939,7 +1949,7 @@ static int alg_test_drbg(const struct alg_test_desc *desc, const char *driver, int err = 0; int pr = 0; int i = 0; - struct drbg_testvec *template = desc->suite.drbg.vecs; + const struct drbg_testvec *template = desc->suite.drbg.vecs; unsigned int tcount = desc->suite.drbg.count; if (0 == memcmp(driver, "drbg_pr_", 8)) @@ -1958,7 +1968,7 @@ static int alg_test_drbg(const struct alg_test_desc *desc, const char *driver, } -static int do_test_kpp(struct crypto_kpp *tfm, struct kpp_testvec *vec, +static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec, const char *alg) { struct kpp_request *req; @@ -2050,7 +2060,7 @@ free_req: } static int test_kpp(struct crypto_kpp *tfm, const char *alg, - struct kpp_testvec *vecs, unsigned int tcount) + const struct kpp_testvec *vecs, unsigned int tcount) { int ret, i; @@ -2086,7 +2096,7 @@ static int alg_test_kpp(const struct alg_test_desc *desc, const char *driver, } static int test_akcipher_one(struct crypto_akcipher *tfm, - struct akcipher_testvec *vecs) + const struct akcipher_testvec *vecs) { char *xbuf[XBUFSIZE]; struct akcipher_request *req; @@ -2206,7 +2216,8 @@ free_xbuf: } static int test_akcipher(struct crypto_akcipher *tfm, const char *alg, - struct akcipher_testvec *vecs, unsigned int tcount) + const struct akcipher_testvec *vecs, + unsigned int tcount) { const char *algo = crypto_tfm_alg_driver_name(crypto_akcipher_tfm(tfm)); diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 03f473116f78..15c043f74b18 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -34,9 +34,9 @@ struct hash_testvec { /* only used with keyed hash algorithms */ - char *key; - char *plaintext; - char *digest; + const char *key; + const char *plaintext; + const char *digest; unsigned char tap[MAX_TAP]; unsigned short psize; unsigned char np; @@ -63,11 +63,11 @@ struct hash_testvec { */ struct cipher_testvec { - char *key; - char *iv; - char *iv_out; - char *input; - char *result; + const char *key; + const char *iv; + const char *iv_out; + const char *input; + const char *result; unsigned short tap[MAX_TAP]; int np; unsigned char also_non_np; @@ -80,11 +80,11 @@ struct cipher_testvec { }; struct aead_testvec { - char *key; - char *iv; - char *input; - char *assoc; - char *result; + const char *key; + const char *iv; + const char *input; + const char *assoc; + const char *result; unsigned char tap[MAX_TAP]; unsigned char atap[MAX_TAP]; int np; @@ -99,10 +99,10 @@ struct aead_testvec { }; struct cprng_testvec { - char *key; - char *dt; - char *v; - char *result; + const char *key; + const char *dt; + const char *v; + const char *result; unsigned char klen; unsigned short dtlen; unsigned short vlen; @@ -111,24 +111,24 @@ struct cprng_testvec { }; struct drbg_testvec { - unsigned char *entropy; + const unsigned char *entropy; size_t entropylen; - unsigned char *entpra; - unsigned char *entprb; + const unsigned char *entpra; + const unsigned char *entprb; size_t entprlen; - unsigned char *addtla; - unsigned char *addtlb; + const unsigned char *addtla; + const unsigned char *addtlb; size_t addtllen; - unsigned char *pers; + const unsigned char *pers; size_t perslen; - unsigned char *expected; + const unsigned char *expected; size_t expectedlen; }; struct akcipher_testvec { - unsigned char *key; - unsigned char *m; - unsigned char *c; + const unsigned char *key; + const unsigned char *m; + const unsigned char *c; unsigned int key_len; unsigned int m_size; unsigned int c_size; @@ -136,22 +136,22 @@ struct akcipher_testvec { }; struct kpp_testvec { - unsigned char *secret; - unsigned char *b_public; - unsigned char *expected_a_public; - unsigned char *expected_ss; + const unsigned char *secret; + const unsigned char *b_public; + const unsigned char *expected_a_public; + const unsigned char *expected_ss; unsigned short secret_size; unsigned short b_public_size; unsigned short expected_a_public_size; unsigned short expected_ss_size; }; -static char zeroed_string[48]; +static const char zeroed_string[48]; /* * RSA test vectors. Borrowed from openSSL. */ -static struct akcipher_testvec rsa_tv_template[] = { +static const struct akcipher_testvec rsa_tv_template[] = { { #ifndef CONFIG_CRYPTO_FIPS .key = @@ -538,7 +538,7 @@ static struct akcipher_testvec rsa_tv_template[] = { } }; -struct kpp_testvec dh_tv_template[] = { +static const struct kpp_testvec dh_tv_template[] = { { .secret = #ifdef __LITTLE_ENDIAN @@ -755,7 +755,7 @@ struct kpp_testvec dh_tv_template[] = { } }; -struct kpp_testvec ecdh_tv_template[] = { +static const struct kpp_testvec ecdh_tv_template[] = { { #ifndef CONFIG_CRYPTO_FIPS .secret = @@ -846,7 +846,7 @@ struct kpp_testvec ecdh_tv_template[] = { /* * MD4 test vectors from RFC1320 */ -static struct hash_testvec md4_tv_template [] = { +static const struct hash_testvec md4_tv_template[] = { { .plaintext = "", .digest = "\x31\xd6\xcf\xe0\xd1\x6a\xe9\x31" @@ -887,7 +887,7 @@ static struct hash_testvec md4_tv_template [] = { }, }; -static struct hash_testvec sha3_224_tv_template[] = { +static const struct hash_testvec sha3_224_tv_template[] = { { .plaintext = "", .digest = "\x6b\x4e\x03\x42\x36\x67\xdb\xb7" @@ -912,7 +912,7 @@ static struct hash_testvec sha3_224_tv_template[] = { }, }; -static struct hash_testvec sha3_256_tv_template[] = { +static const struct hash_testvec sha3_256_tv_template[] = { { .plaintext = "", .digest = "\xa7\xff\xc6\xf8\xbf\x1e\xd7\x66" @@ -938,7 +938,7 @@ static struct hash_testvec sha3_256_tv_template[] = { }; -static struct hash_testvec sha3_384_tv_template[] = { +static const struct hash_testvec sha3_384_tv_template[] = { { .plaintext = "", .digest = "\x0c\x63\xa7\x5b\x84\x5e\x4f\x7d" @@ -970,7 +970,7 @@ static struct hash_testvec sha3_384_tv_template[] = { }; -static struct hash_testvec sha3_512_tv_template[] = { +static const struct hash_testvec sha3_512_tv_template[] = { { .plaintext = "", .digest = "\xa6\x9f\x73\xcc\xa2\x3a\x9a\xc5" @@ -1011,7 +1011,7 @@ static struct hash_testvec sha3_512_tv_template[] = { /* * MD5 test vectors from RFC1321 */ -static struct hash_testvec md5_tv_template[] = { +static const struct hash_testvec md5_tv_template[] = { { .digest = "\xd4\x1d\x8c\xd9\x8f\x00\xb2\x04" "\xe9\x80\x09\x98\xec\xf8\x42\x7e", @@ -1055,7 +1055,7 @@ static struct hash_testvec md5_tv_template[] = { /* * RIPEMD-128 test vectors from ISO/IEC 10118-3:2004(E) */ -static struct hash_testvec rmd128_tv_template[] = { +static const struct hash_testvec rmd128_tv_template[] = { { .digest = "\xcd\xf2\x62\x13\xa1\x50\xdc\x3e" "\xcb\x61\x0f\x18\xf6\xb3\x8b\x46", @@ -1117,7 +1117,7 @@ static struct hash_testvec rmd128_tv_template[] = { /* * RIPEMD-160 test vectors from ISO/IEC 10118-3:2004(E) */ -static struct hash_testvec rmd160_tv_template[] = { +static const struct hash_testvec rmd160_tv_template[] = { { .digest = "\x9c\x11\x85\xa5\xc5\xe9\xfc\x54\x61\x28" "\x08\x97\x7e\xe8\xf5\x48\xb2\x25\x8d\x31", @@ -1179,7 +1179,7 @@ static struct hash_testvec rmd160_tv_template[] = { /* * RIPEMD-256 test vectors */ -static struct hash_testvec rmd256_tv_template[] = { +static const struct hash_testvec rmd256_tv_template[] = { { .digest = "\x02\xba\x4c\x4e\x5f\x8e\xcd\x18" "\x77\xfc\x52\xd6\x4d\x30\xe3\x7a" @@ -1245,7 +1245,7 @@ static struct hash_testvec rmd256_tv_template[] = { /* * RIPEMD-320 test vectors */ -static struct hash_testvec rmd320_tv_template[] = { +static const struct hash_testvec rmd320_tv_template[] = { { .digest = "\x22\xd6\x5d\x56\x61\x53\x6c\xdc\x75\xc1" "\xfd\xf5\xc6\xde\x7b\x41\xb9\xf2\x73\x25" @@ -1308,7 +1308,7 @@ static struct hash_testvec rmd320_tv_template[] = { } }; -static struct hash_testvec crct10dif_tv_template[] = { +static const struct hash_testvec crct10dif_tv_template[] = { { .plaintext = "abc", .psize = 3, @@ -1358,7 +1358,7 @@ static struct hash_testvec crct10dif_tv_template[] = { * SHA1 test vectors from from FIPS PUB 180-1 * Long vector from CAVS 5.0 */ -static struct hash_testvec sha1_tv_template[] = { +static const struct hash_testvec sha1_tv_template[] = { { .plaintext = "", .psize = 0, @@ -1548,7 +1548,7 @@ static struct hash_testvec sha1_tv_template[] = { /* * SHA224 test vectors from from FIPS PUB 180-2 */ -static struct hash_testvec sha224_tv_template[] = { +static const struct hash_testvec sha224_tv_template[] = { { .plaintext = "", .psize = 0, @@ -1720,7 +1720,7 @@ static struct hash_testvec sha224_tv_template[] = { /* * SHA256 test vectors from from NIST */ -static struct hash_testvec sha256_tv_template[] = { +static const struct hash_testvec sha256_tv_template[] = { { .plaintext = "", .psize = 0, @@ -1891,7 +1891,7 @@ static struct hash_testvec sha256_tv_template[] = { /* * SHA384 test vectors from from NIST and kerneli */ -static struct hash_testvec sha384_tv_template[] = { +static const struct hash_testvec sha384_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2083,7 +2083,7 @@ static struct hash_testvec sha384_tv_template[] = { /* * SHA512 test vectors from from NIST and kerneli */ -static struct hash_testvec sha512_tv_template[] = { +static const struct hash_testvec sha512_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2290,7 +2290,7 @@ static struct hash_testvec sha512_tv_template[] = { * by Vincent Rijmen and Paulo S. L. M. Barreto as part of the NESSIE * submission */ -static struct hash_testvec wp512_tv_template[] = { +static const struct hash_testvec wp512_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2386,7 +2386,7 @@ static struct hash_testvec wp512_tv_template[] = { }, }; -static struct hash_testvec wp384_tv_template[] = { +static const struct hash_testvec wp384_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2466,7 +2466,7 @@ static struct hash_testvec wp384_tv_template[] = { }, }; -static struct hash_testvec wp256_tv_template[] = { +static const struct hash_testvec wp256_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2533,7 +2533,7 @@ static struct hash_testvec wp256_tv_template[] = { /* * TIGER test vectors from Tiger website */ -static struct hash_testvec tgr192_tv_template[] = { +static const struct hash_testvec tgr192_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2576,7 +2576,7 @@ static struct hash_testvec tgr192_tv_template[] = { }, }; -static struct hash_testvec tgr160_tv_template[] = { +static const struct hash_testvec tgr160_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2619,7 +2619,7 @@ static struct hash_testvec tgr160_tv_template[] = { }, }; -static struct hash_testvec tgr128_tv_template[] = { +static const struct hash_testvec tgr128_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2656,7 +2656,7 @@ static struct hash_testvec tgr128_tv_template[] = { }, }; -static struct hash_testvec ghash_tv_template[] = +static const struct hash_testvec ghash_tv_template[] = { { .key = "\xdf\xa6\xbf\x4d\xed\x81\xdb\x03" @@ -2771,7 +2771,7 @@ static struct hash_testvec ghash_tv_template[] = * HMAC-MD5 test vectors from RFC2202 * (These need to be fixed to not use strlen). */ -static struct hash_testvec hmac_md5_tv_template[] = +static const struct hash_testvec hmac_md5_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", @@ -2851,7 +2851,7 @@ static struct hash_testvec hmac_md5_tv_template[] = /* * HMAC-RIPEMD128 test vectors from RFC2286 */ -static struct hash_testvec hmac_rmd128_tv_template[] = { +static const struct hash_testvec hmac_rmd128_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", .ksize = 16, @@ -2930,7 +2930,7 @@ static struct hash_testvec hmac_rmd128_tv_template[] = { /* * HMAC-RIPEMD160 test vectors from RFC2286 */ -static struct hash_testvec hmac_rmd160_tv_template[] = { +static const struct hash_testvec hmac_rmd160_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", .ksize = 20, @@ -3009,7 +3009,7 @@ static struct hash_testvec hmac_rmd160_tv_template[] = { /* * HMAC-SHA1 test vectors from RFC2202 */ -static struct hash_testvec hmac_sha1_tv_template[] = { +static const struct hash_testvec hmac_sha1_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", .ksize = 20, @@ -3090,7 +3090,7 @@ static struct hash_testvec hmac_sha1_tv_template[] = { /* * SHA224 HMAC test vectors from RFC4231 */ -static struct hash_testvec hmac_sha224_tv_template[] = { +static const struct hash_testvec hmac_sha224_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" @@ -3203,7 +3203,7 @@ static struct hash_testvec hmac_sha224_tv_template[] = { * HMAC-SHA256 test vectors from * draft-ietf-ipsec-ciph-sha-256-01.txt */ -static struct hash_testvec hmac_sha256_tv_template[] = { +static const struct hash_testvec hmac_sha256_tv_template[] = { { .key = "\x01\x02\x03\x04\x05\x06\x07\x08" "\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" @@ -3338,7 +3338,7 @@ static struct hash_testvec hmac_sha256_tv_template[] = { }, }; -static struct hash_testvec aes_cmac128_tv_template[] = { +static const struct hash_testvec aes_cmac128_tv_template[] = { { /* From NIST Special Publication 800-38B, AES-128 */ .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", @@ -3413,7 +3413,7 @@ static struct hash_testvec aes_cmac128_tv_template[] = { } }; -static struct hash_testvec aes_cbcmac_tv_template[] = { +static const struct hash_testvec aes_cbcmac_tv_template[] = { { .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", @@ -3473,7 +3473,7 @@ static struct hash_testvec aes_cbcmac_tv_template[] = { } }; -static struct hash_testvec des3_ede_cmac64_tv_template[] = { +static const struct hash_testvec des3_ede_cmac64_tv_template[] = { /* * From NIST Special Publication 800-38B, Three Key TDEA * Corrected test vectors from: @@ -3519,7 +3519,7 @@ static struct hash_testvec des3_ede_cmac64_tv_template[] = { } }; -static struct hash_testvec aes_xcbc128_tv_template[] = { +static const struct hash_testvec aes_xcbc128_tv_template[] = { { .key = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", @@ -3585,35 +3585,35 @@ static struct hash_testvec aes_xcbc128_tv_template[] = { } }; -static char vmac_string1[128] = {'\x01', '\x01', '\x01', '\x01', - '\x02', '\x03', '\x02', '\x02', - '\x02', '\x04', '\x01', '\x07', - '\x04', '\x01', '\x04', '\x03',}; -static char vmac_string2[128] = {'a', 'b', 'c',}; -static char vmac_string3[128] = {'a', 'b', 'c', 'a', 'b', 'c', - 'a', 'b', 'c', 'a', 'b', 'c', - 'a', 'b', 'c', 'a', 'b', 'c', - 'a', 'b', 'c', 'a', 'b', 'c', - 'a', 'b', 'c', 'a', 'b', 'c', - 'a', 'b', 'c', 'a', 'b', 'c', - 'a', 'b', 'c', 'a', 'b', 'c', - 'a', 'b', 'c', 'a', 'b', 'c', - }; +static const char vmac_string1[128] = {'\x01', '\x01', '\x01', '\x01', + '\x02', '\x03', '\x02', '\x02', + '\x02', '\x04', '\x01', '\x07', + '\x04', '\x01', '\x04', '\x03',}; +static const char vmac_string2[128] = {'a', 'b', 'c',}; +static const char vmac_string3[128] = {'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + }; -static char vmac_string4[17] = {'b', 'c', 'e', 'f', - 'i', 'j', 'l', 'm', - 'o', 'p', 'r', 's', - 't', 'u', 'w', 'x', 'z'}; +static const char vmac_string4[17] = {'b', 'c', 'e', 'f', + 'i', 'j', 'l', 'm', + 'o', 'p', 'r', 's', + 't', 'u', 'w', 'x', 'z'}; -static char vmac_string5[127] = {'r', 'm', 'b', 't', 'c', - 'o', 'l', 'k', ']', '%', - '9', '2', '7', '!', 'A'}; +static const char vmac_string5[127] = {'r', 'm', 'b', 't', 'c', + 'o', 'l', 'k', ']', '%', + '9', '2', '7', '!', 'A'}; -static char vmac_string6[129] = {'p', 't', '*', '7', 'l', - 'i', '!', '#', 'w', '0', - 'z', '/', '4', 'A', 'n'}; +static const char vmac_string6[129] = {'p', 't', '*', '7', 'l', + 'i', '!', '#', 'w', '0', + 'z', '/', '4', 'A', 'n'}; -static struct hash_testvec aes_vmac128_tv_template[] = { +static const struct hash_testvec aes_vmac128_tv_template[] = { { .key = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", @@ -3691,7 +3691,7 @@ static struct hash_testvec aes_vmac128_tv_template[] = { * SHA384 HMAC test vectors from RFC4231 */ -static struct hash_testvec hmac_sha384_tv_template[] = { +static const struct hash_testvec hmac_sha384_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" @@ -3789,7 +3789,7 @@ static struct hash_testvec hmac_sha384_tv_template[] = { * SHA512 HMAC test vectors from RFC4231 */ -static struct hash_testvec hmac_sha512_tv_template[] = { +static const struct hash_testvec hmac_sha512_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" @@ -3894,7 +3894,7 @@ static struct hash_testvec hmac_sha512_tv_template[] = { }, }; -static struct hash_testvec hmac_sha3_224_tv_template[] = { +static const struct hash_testvec hmac_sha3_224_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" @@ -3983,7 +3983,7 @@ static struct hash_testvec hmac_sha3_224_tv_template[] = { }, }; -static struct hash_testvec hmac_sha3_256_tv_template[] = { +static const struct hash_testvec hmac_sha3_256_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" @@ -4072,7 +4072,7 @@ static struct hash_testvec hmac_sha3_256_tv_template[] = { }, }; -static struct hash_testvec hmac_sha3_384_tv_template[] = { +static const struct hash_testvec hmac_sha3_384_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" @@ -4169,7 +4169,7 @@ static struct hash_testvec hmac_sha3_384_tv_template[] = { }, }; -static struct hash_testvec hmac_sha3_512_tv_template[] = { +static const struct hash_testvec hmac_sha3_512_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" @@ -4278,7 +4278,7 @@ static struct hash_testvec hmac_sha3_512_tv_template[] = { * Poly1305 test vectors from RFC7539 A.3. */ -static struct hash_testvec poly1305_tv_template[] = { +static const struct hash_testvec poly1305_tv_template[] = { { /* Test Vector #1 */ .plaintext = "\x00\x00\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -4523,7 +4523,7 @@ static struct hash_testvec poly1305_tv_template[] = { /* * DES test vectors. */ -static struct cipher_testvec des_enc_tv_template[] = { +static const struct cipher_testvec des_enc_tv_template[] = { { /* From Applied Cryptography */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", .klen = 8, @@ -4697,7 +4697,7 @@ static struct cipher_testvec des_enc_tv_template[] = { }, }; -static struct cipher_testvec des_dec_tv_template[] = { +static const struct cipher_testvec des_dec_tv_template[] = { { /* From Applied Cryptography */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", .klen = 8, @@ -4807,7 +4807,7 @@ static struct cipher_testvec des_dec_tv_template[] = { }, }; -static struct cipher_testvec des_cbc_enc_tv_template[] = { +static const struct cipher_testvec des_cbc_enc_tv_template[] = { { /* From OpenSSL */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", .klen = 8, @@ -4933,7 +4933,7 @@ static struct cipher_testvec des_cbc_enc_tv_template[] = { }, }; -static struct cipher_testvec des_cbc_dec_tv_template[] = { +static const struct cipher_testvec des_cbc_dec_tv_template[] = { { /* FIPS Pub 81 */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", .klen = 8, @@ -5042,7 +5042,7 @@ static struct cipher_testvec des_cbc_dec_tv_template[] = { }, }; -static struct cipher_testvec des_ctr_enc_tv_template[] = { +static const struct cipher_testvec des_ctr_enc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55", .klen = 8, @@ -5188,7 +5188,7 @@ static struct cipher_testvec des_ctr_enc_tv_template[] = { }, }; -static struct cipher_testvec des_ctr_dec_tv_template[] = { +static const struct cipher_testvec des_ctr_dec_tv_template[] = { { /* Generated with Crypto++ */ .key = "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55", .klen = 8, @@ -5334,7 +5334,7 @@ static struct cipher_testvec des_ctr_dec_tv_template[] = { }, }; -static struct cipher_testvec des3_ede_enc_tv_template[] = { +static const struct cipher_testvec des3_ede_enc_tv_template[] = { { /* These are from openssl */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" "\x55\x55\x55\x55\x55\x55\x55\x55" @@ -5499,7 +5499,7 @@ static struct cipher_testvec des3_ede_enc_tv_template[] = { }, }; -static struct cipher_testvec des3_ede_dec_tv_template[] = { +static const struct cipher_testvec des3_ede_dec_tv_template[] = { { /* These are from openssl */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" "\x55\x55\x55\x55\x55\x55\x55\x55" @@ -5664,7 +5664,7 @@ static struct cipher_testvec des3_ede_dec_tv_template[] = { }, }; -static struct cipher_testvec des3_ede_cbc_enc_tv_template[] = { +static const struct cipher_testvec des3_ede_cbc_enc_tv_template[] = { { /* Generated from openssl */ .key = "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24" "\x44\x4D\x99\x5A\x12\xD6\x40\xC0" @@ -5844,7 +5844,7 @@ static struct cipher_testvec des3_ede_cbc_enc_tv_template[] = { }, }; -static struct cipher_testvec des3_ede_cbc_dec_tv_template[] = { +static const struct cipher_testvec des3_ede_cbc_dec_tv_template[] = { { /* Generated from openssl */ .key = "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24" "\x44\x4D\x99\x5A\x12\xD6\x40\xC0" @@ -6024,7 +6024,7 @@ static struct cipher_testvec des3_ede_cbc_dec_tv_template[] = { }, }; -static struct cipher_testvec des3_ede_ctr_enc_tv_template[] = { +static const struct cipher_testvec des3_ede_ctr_enc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x9C\xD6\xF3\x9C\xB9\x5A\x67\x00" "\x5A\x67\x00\x2D\xCE\xEB\x2D\xCE" @@ -6302,7 +6302,7 @@ static struct cipher_testvec des3_ede_ctr_enc_tv_template[] = { }, }; -static struct cipher_testvec des3_ede_ctr_dec_tv_template[] = { +static const struct cipher_testvec des3_ede_ctr_dec_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x9C\xD6\xF3\x9C\xB9\x5A\x67\x00" "\x5A\x67\x00\x2D\xCE\xEB\x2D\xCE" @@ -6583,7 +6583,7 @@ static struct cipher_testvec des3_ede_ctr_dec_tv_template[] = { /* * Blowfish test vectors. */ -static struct cipher_testvec bf_enc_tv_template[] = { +static const struct cipher_testvec bf_enc_tv_template[] = { { /* DES test vectors from OpenSSL */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00", .klen = 8, @@ -6775,7 +6775,7 @@ static struct cipher_testvec bf_enc_tv_template[] = { }, }; -static struct cipher_testvec bf_dec_tv_template[] = { +static const struct cipher_testvec bf_dec_tv_template[] = { { /* DES test vectors from OpenSSL */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00", .klen = 8, @@ -6967,7 +6967,7 @@ static struct cipher_testvec bf_dec_tv_template[] = { }, }; -static struct cipher_testvec bf_cbc_enc_tv_template[] = { +static const struct cipher_testvec bf_cbc_enc_tv_template[] = { { /* From OpenSSL */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87", @@ -7124,7 +7124,7 @@ static struct cipher_testvec bf_cbc_enc_tv_template[] = { }, }; -static struct cipher_testvec bf_cbc_dec_tv_template[] = { +static const struct cipher_testvec bf_cbc_dec_tv_template[] = { { /* From OpenSSL */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87", @@ -7281,7 +7281,7 @@ static struct cipher_testvec bf_cbc_dec_tv_template[] = { }, }; -static struct cipher_testvec bf_ctr_enc_tv_template[] = { +static const struct cipher_testvec bf_ctr_enc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -7693,7 +7693,7 @@ static struct cipher_testvec bf_ctr_enc_tv_template[] = { }, }; -static struct cipher_testvec bf_ctr_dec_tv_template[] = { +static const struct cipher_testvec bf_ctr_dec_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -8108,7 +8108,7 @@ static struct cipher_testvec bf_ctr_dec_tv_template[] = { /* * Twofish test vectors. */ -static struct cipher_testvec tf_enc_tv_template[] = { +static const struct cipher_testvec tf_enc_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -8276,7 +8276,7 @@ static struct cipher_testvec tf_enc_tv_template[] = { }, }; -static struct cipher_testvec tf_dec_tv_template[] = { +static const struct cipher_testvec tf_dec_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -8444,7 +8444,7 @@ static struct cipher_testvec tf_dec_tv_template[] = { }, }; -static struct cipher_testvec tf_cbc_enc_tv_template[] = { +static const struct cipher_testvec tf_cbc_enc_tv_template[] = { { /* Generated with Nettle */ .key = zeroed_string, .klen = 16, @@ -8627,7 +8627,7 @@ static struct cipher_testvec tf_cbc_enc_tv_template[] = { }, }; -static struct cipher_testvec tf_cbc_dec_tv_template[] = { +static const struct cipher_testvec tf_cbc_dec_tv_template[] = { { /* Reverse of the first four above */ .key = zeroed_string, .klen = 16, @@ -8810,7 +8810,7 @@ static struct cipher_testvec tf_cbc_dec_tv_template[] = { }, }; -static struct cipher_testvec tf_ctr_enc_tv_template[] = { +static const struct cipher_testvec tf_ctr_enc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -9221,7 +9221,7 @@ static struct cipher_testvec tf_ctr_enc_tv_template[] = { }, }; -static struct cipher_testvec tf_ctr_dec_tv_template[] = { +static const struct cipher_testvec tf_ctr_dec_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -9632,7 +9632,7 @@ static struct cipher_testvec tf_ctr_dec_tv_template[] = { }, }; -static struct cipher_testvec tf_lrw_enc_tv_template[] = { +static const struct cipher_testvec tf_lrw_enc_tv_template[] = { /* Generated from AES-LRW test vectors */ { .key = "\x45\x62\xac\x25\xf8\x28\x17\x6d" @@ -9884,7 +9884,7 @@ static struct cipher_testvec tf_lrw_enc_tv_template[] = { }, }; -static struct cipher_testvec tf_lrw_dec_tv_template[] = { +static const struct cipher_testvec tf_lrw_dec_tv_template[] = { /* Generated from AES-LRW test vectors */ /* same as enc vectors with input and result reversed */ { @@ -10137,7 +10137,7 @@ static struct cipher_testvec tf_lrw_dec_tv_template[] = { }, }; -static struct cipher_testvec tf_xts_enc_tv_template[] = { +static const struct cipher_testvec tf_xts_enc_tv_template[] = { /* Generated from AES-XTS test vectors */ { .key = "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -10479,7 +10479,7 @@ static struct cipher_testvec tf_xts_enc_tv_template[] = { }, }; -static struct cipher_testvec tf_xts_dec_tv_template[] = { +static const struct cipher_testvec tf_xts_dec_tv_template[] = { /* Generated from AES-XTS test vectors */ /* same as enc vectors with input and result reversed */ { @@ -10826,7 +10826,7 @@ static struct cipher_testvec tf_xts_dec_tv_template[] = { * Serpent test vectors. These are backwards because Serpent writes * octet sequences in right-to-left mode. */ -static struct cipher_testvec serpent_enc_tv_template[] = { +static const struct cipher_testvec serpent_enc_tv_template[] = { { .input = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", @@ -11002,7 +11002,7 @@ static struct cipher_testvec serpent_enc_tv_template[] = { }, }; -static struct cipher_testvec tnepres_enc_tv_template[] = { +static const struct cipher_testvec tnepres_enc_tv_template[] = { { /* KeySize=128, PT=0, I=1 */ .input = "\x00\x00\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00\x00\x00\x00\x00", @@ -11052,7 +11052,7 @@ static struct cipher_testvec tnepres_enc_tv_template[] = { }; -static struct cipher_testvec serpent_dec_tv_template[] = { +static const struct cipher_testvec serpent_dec_tv_template[] = { { .input = "\x12\x07\xfc\xce\x9b\xd0\xd6\x47" "\x6a\xe9\x8f\xbe\xd1\x43\xa0\xe2", @@ -11228,7 +11228,7 @@ static struct cipher_testvec serpent_dec_tv_template[] = { }, }; -static struct cipher_testvec tnepres_dec_tv_template[] = { +static const struct cipher_testvec tnepres_dec_tv_template[] = { { .input = "\x41\xcc\x6b\x31\x59\x31\x45\x97" "\x6d\x6f\xbb\x38\x4b\x37\x21\x28", @@ -11269,7 +11269,7 @@ static struct cipher_testvec tnepres_dec_tv_template[] = { }, }; -static struct cipher_testvec serpent_cbc_enc_tv_template[] = { +static const struct cipher_testvec serpent_cbc_enc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -11410,7 +11410,7 @@ static struct cipher_testvec serpent_cbc_enc_tv_template[] = { }, }; -static struct cipher_testvec serpent_cbc_dec_tv_template[] = { +static const struct cipher_testvec serpent_cbc_dec_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -11551,7 +11551,7 @@ static struct cipher_testvec serpent_cbc_dec_tv_template[] = { }, }; -static struct cipher_testvec serpent_ctr_enc_tv_template[] = { +static const struct cipher_testvec serpent_ctr_enc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -11962,7 +11962,7 @@ static struct cipher_testvec serpent_ctr_enc_tv_template[] = { }, }; -static struct cipher_testvec serpent_ctr_dec_tv_template[] = { +static const struct cipher_testvec serpent_ctr_dec_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -12373,7 +12373,7 @@ static struct cipher_testvec serpent_ctr_dec_tv_template[] = { }, }; -static struct cipher_testvec serpent_lrw_enc_tv_template[] = { +static const struct cipher_testvec serpent_lrw_enc_tv_template[] = { /* Generated from AES-LRW test vectors */ { .key = "\x45\x62\xac\x25\xf8\x28\x17\x6d" @@ -12625,7 +12625,7 @@ static struct cipher_testvec serpent_lrw_enc_tv_template[] = { }, }; -static struct cipher_testvec serpent_lrw_dec_tv_template[] = { +static const struct cipher_testvec serpent_lrw_dec_tv_template[] = { /* Generated from AES-LRW test vectors */ /* same as enc vectors with input and result reversed */ { @@ -12878,7 +12878,7 @@ static struct cipher_testvec serpent_lrw_dec_tv_template[] = { }, }; -static struct cipher_testvec serpent_xts_enc_tv_template[] = { +static const struct cipher_testvec serpent_xts_enc_tv_template[] = { /* Generated from AES-XTS test vectors */ { .key = "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -13220,7 +13220,7 @@ static struct cipher_testvec serpent_xts_enc_tv_template[] = { }, }; -static struct cipher_testvec serpent_xts_dec_tv_template[] = { +static const struct cipher_testvec serpent_xts_dec_tv_template[] = { /* Generated from AES-XTS test vectors */ /* same as enc vectors with input and result reversed */ { @@ -13564,7 +13564,7 @@ static struct cipher_testvec serpent_xts_dec_tv_template[] = { }; /* Cast6 test vectors from RFC 2612 */ -static struct cipher_testvec cast6_enc_tv_template[] = { +static const struct cipher_testvec cast6_enc_tv_template[] = { { .key = "\x23\x42\xbb\x9e\xfa\x38\x54\x2c" "\x0a\xf7\x56\x47\xf2\x9f\x61\x5d", @@ -13735,7 +13735,7 @@ static struct cipher_testvec cast6_enc_tv_template[] = { }, }; -static struct cipher_testvec cast6_dec_tv_template[] = { +static const struct cipher_testvec cast6_dec_tv_template[] = { { .key = "\x23\x42\xbb\x9e\xfa\x38\x54\x2c" "\x0a\xf7\x56\x47\xf2\x9f\x61\x5d", @@ -13906,7 +13906,7 @@ static struct cipher_testvec cast6_dec_tv_template[] = { }, }; -static struct cipher_testvec cast6_cbc_enc_tv_template[] = { +static const struct cipher_testvec cast6_cbc_enc_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -14047,7 +14047,7 @@ static struct cipher_testvec cast6_cbc_enc_tv_template[] = { }, }; -static struct cipher_testvec cast6_cbc_dec_tv_template[] = { +static const struct cipher_testvec cast6_cbc_dec_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -14188,7 +14188,7 @@ static struct cipher_testvec cast6_cbc_dec_tv_template[] = { }, }; -static struct cipher_testvec cast6_ctr_enc_tv_template[] = { +static const struct cipher_testvec cast6_ctr_enc_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -14345,7 +14345,7 @@ static struct cipher_testvec cast6_ctr_enc_tv_template[] = { }, }; -static struct cipher_testvec cast6_ctr_dec_tv_template[] = { +static const struct cipher_testvec cast6_ctr_dec_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -14502,7 +14502,7 @@ static struct cipher_testvec cast6_ctr_dec_tv_template[] = { }, }; -static struct cipher_testvec cast6_lrw_enc_tv_template[] = { +static const struct cipher_testvec cast6_lrw_enc_tv_template[] = { { /* Generated from TF test vectors */ .key = "\xf8\xd4\x76\xff\xd6\x46\xee\x6c" "\x23\x84\xcb\x1c\x77\xd6\x19\x5d" @@ -14649,7 +14649,7 @@ static struct cipher_testvec cast6_lrw_enc_tv_template[] = { }, }; -static struct cipher_testvec cast6_lrw_dec_tv_template[] = { +static const struct cipher_testvec cast6_lrw_dec_tv_template[] = { { /* Generated from TF test vectors */ .key = "\xf8\xd4\x76\xff\xd6\x46\xee\x6c" "\x23\x84\xcb\x1c\x77\xd6\x19\x5d" @@ -14796,7 +14796,7 @@ static struct cipher_testvec cast6_lrw_dec_tv_template[] = { }, }; -static struct cipher_testvec cast6_xts_enc_tv_template[] = { +static const struct cipher_testvec cast6_xts_enc_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x27\x18\x28\x18\x28\x45\x90\x45" "\x23\x53\x60\x28\x74\x71\x35\x26" @@ -14945,7 +14945,7 @@ static struct cipher_testvec cast6_xts_enc_tv_template[] = { }, }; -static struct cipher_testvec cast6_xts_dec_tv_template[] = { +static const struct cipher_testvec cast6_xts_dec_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x27\x18\x28\x18\x28\x45\x90\x45" "\x23\x53\x60\x28\x74\x71\x35\x26" @@ -15098,7 +15098,7 @@ static struct cipher_testvec cast6_xts_dec_tv_template[] = { /* * AES test vectors. */ -static struct cipher_testvec aes_enc_tv_template[] = { +static const struct cipher_testvec aes_enc_tv_template[] = { { /* From FIPS-197 */ .key = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", @@ -15270,7 +15270,7 @@ static struct cipher_testvec aes_enc_tv_template[] = { }, }; -static struct cipher_testvec aes_dec_tv_template[] = { +static const struct cipher_testvec aes_dec_tv_template[] = { { /* From FIPS-197 */ .key = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", @@ -15442,7 +15442,7 @@ static struct cipher_testvec aes_dec_tv_template[] = { }, }; -static struct cipher_testvec aes_cbc_enc_tv_template[] = { +static const struct cipher_testvec aes_cbc_enc_tv_template[] = { { /* From RFC 3602 */ .key = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b" "\x51\x2e\x03\xd5\x34\x12\x00\x06", @@ -15664,7 +15664,7 @@ static struct cipher_testvec aes_cbc_enc_tv_template[] = { }, }; -static struct cipher_testvec aes_cbc_dec_tv_template[] = { +static const struct cipher_testvec aes_cbc_dec_tv_template[] = { { /* From RFC 3602 */ .key = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b" "\x51\x2e\x03\xd5\x34\x12\x00\x06", @@ -15886,7 +15886,7 @@ static struct cipher_testvec aes_cbc_dec_tv_template[] = { }, }; -static struct aead_testvec hmac_md5_ecb_cipher_null_enc_tv_template[] = { +static const struct aead_testvec hmac_md5_ecb_cipher_null_enc_tv_template[] = { { /* Input data from RFC 2410 Case 1 */ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -15928,7 +15928,7 @@ static struct aead_testvec hmac_md5_ecb_cipher_null_enc_tv_template[] = { }, }; -static struct aead_testvec hmac_md5_ecb_cipher_null_dec_tv_template[] = { +static const struct aead_testvec hmac_md5_ecb_cipher_null_dec_tv_template[] = { { #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -15970,7 +15970,7 @@ static struct aead_testvec hmac_md5_ecb_cipher_null_dec_tv_template[] = { }, }; -static struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = { { /* RFC 3602 Case 1 */ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -16239,7 +16239,7 @@ static struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = { }, }; -static struct aead_testvec hmac_sha1_ecb_cipher_null_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha1_ecb_cipher_null_enc_tv_temp[] = { { /* Input data from RFC 2410 Case 1 */ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -16285,7 +16285,7 @@ static struct aead_testvec hmac_sha1_ecb_cipher_null_enc_tv_temp[] = { }, }; -static struct aead_testvec hmac_sha1_ecb_cipher_null_dec_tv_temp[] = { +static const struct aead_testvec hmac_sha1_ecb_cipher_null_dec_tv_temp[] = { { #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -16331,7 +16331,7 @@ static struct aead_testvec hmac_sha1_ecb_cipher_null_dec_tv_temp[] = { }, }; -static struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = { { /* RFC 3602 Case 1 */ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -16614,7 +16614,7 @@ static struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = { }, }; -static struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = { { /* RFC 3602 Case 1 */ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -16953,7 +16953,7 @@ static struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = { }, }; -static struct aead_testvec hmac_sha1_des_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha1_des_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17012,7 +17012,7 @@ static struct aead_testvec hmac_sha1_des_cbc_enc_tv_temp[] = { }, }; -static struct aead_testvec hmac_sha224_des_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha224_des_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17071,7 +17071,7 @@ static struct aead_testvec hmac_sha224_des_cbc_enc_tv_temp[] = { }, }; -static struct aead_testvec hmac_sha256_des_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha256_des_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17132,7 +17132,7 @@ static struct aead_testvec hmac_sha256_des_cbc_enc_tv_temp[] = { }, }; -static struct aead_testvec hmac_sha384_des_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha384_des_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17197,7 +17197,7 @@ static struct aead_testvec hmac_sha384_des_cbc_enc_tv_temp[] = { }, }; -static struct aead_testvec hmac_sha512_des_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha512_des_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17266,7 +17266,7 @@ static struct aead_testvec hmac_sha512_des_cbc_enc_tv_temp[] = { }, }; -static struct aead_testvec hmac_sha1_des3_ede_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha1_des3_ede_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17327,7 +17327,7 @@ static struct aead_testvec hmac_sha1_des3_ede_cbc_enc_tv_temp[] = { }, }; -static struct aead_testvec hmac_sha224_des3_ede_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha224_des3_ede_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17388,7 +17388,7 @@ static struct aead_testvec hmac_sha224_des3_ede_cbc_enc_tv_temp[] = { }, }; -static struct aead_testvec hmac_sha256_des3_ede_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha256_des3_ede_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17451,7 +17451,7 @@ static struct aead_testvec hmac_sha256_des3_ede_cbc_enc_tv_temp[] = { }, }; -static struct aead_testvec hmac_sha384_des3_ede_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha384_des3_ede_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17518,7 +17518,7 @@ static struct aead_testvec hmac_sha384_des3_ede_cbc_enc_tv_temp[] = { }, }; -static struct aead_testvec hmac_sha512_des3_ede_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha512_des3_ede_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17589,7 +17589,7 @@ static struct aead_testvec hmac_sha512_des3_ede_cbc_enc_tv_temp[] = { }, }; -static struct cipher_testvec aes_lrw_enc_tv_template[] = { +static const struct cipher_testvec aes_lrw_enc_tv_template[] = { /* from http://grouper.ieee.org/groups/1619/email/pdf00017.pdf */ { /* LRW-32-AES 1 */ .key = "\x45\x62\xac\x25\xf8\x28\x17\x6d" @@ -17842,7 +17842,7 @@ static struct cipher_testvec aes_lrw_enc_tv_template[] = { } }; -static struct cipher_testvec aes_lrw_dec_tv_template[] = { +static const struct cipher_testvec aes_lrw_dec_tv_template[] = { /* from http://grouper.ieee.org/groups/1619/email/pdf00017.pdf */ /* same as enc vectors with input and result reversed */ { /* LRW-32-AES 1 */ @@ -18096,7 +18096,7 @@ static struct cipher_testvec aes_lrw_dec_tv_template[] = { } }; -static struct cipher_testvec aes_xts_enc_tv_template[] = { +static const struct cipher_testvec aes_xts_enc_tv_template[] = { /* http://grouper.ieee.org/groups/1619/email/pdf00086.pdf */ { /* XTS-AES 1 */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -18439,7 +18439,7 @@ static struct cipher_testvec aes_xts_enc_tv_template[] = { } }; -static struct cipher_testvec aes_xts_dec_tv_template[] = { +static const struct cipher_testvec aes_xts_dec_tv_template[] = { /* http://grouper.ieee.org/groups/1619/email/pdf00086.pdf */ { /* XTS-AES 1 */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -18783,7 +18783,7 @@ static struct cipher_testvec aes_xts_dec_tv_template[] = { }; -static struct cipher_testvec aes_ctr_enc_tv_template[] = { +static const struct cipher_testvec aes_ctr_enc_tv_template[] = { { /* From NIST Special Publication 800-38A, Appendix F.5 */ .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", @@ -19138,7 +19138,7 @@ static struct cipher_testvec aes_ctr_enc_tv_template[] = { }, }; -static struct cipher_testvec aes_ctr_dec_tv_template[] = { +static const struct cipher_testvec aes_ctr_dec_tv_template[] = { { /* From NIST Special Publication 800-38A, Appendix F.5 */ .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", @@ -19493,7 +19493,7 @@ static struct cipher_testvec aes_ctr_dec_tv_template[] = { }, }; -static struct cipher_testvec aes_ctr_rfc3686_enc_tv_template[] = { +static const struct cipher_testvec aes_ctr_rfc3686_enc_tv_template[] = { { /* From RFC 3686 */ .key = "\xae\x68\x52\xf8\x12\x10\x67\xcc" "\x4b\xf7\xa5\x76\x55\x77\xf3\x9e" @@ -20625,7 +20625,7 @@ static struct cipher_testvec aes_ctr_rfc3686_enc_tv_template[] = { }, }; -static struct cipher_testvec aes_ctr_rfc3686_dec_tv_template[] = { +static const struct cipher_testvec aes_ctr_rfc3686_dec_tv_template[] = { { /* From RFC 3686 */ .key = "\xae\x68\x52\xf8\x12\x10\x67\xcc" "\x4b\xf7\xa5\x76\x55\x77\xf3\x9e" @@ -20716,7 +20716,7 @@ static struct cipher_testvec aes_ctr_rfc3686_dec_tv_template[] = { }, }; -static struct cipher_testvec aes_ofb_enc_tv_template[] = { +static const struct cipher_testvec aes_ofb_enc_tv_template[] = { /* From NIST Special Publication 800-38A, Appendix F.5 */ { .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" @@ -20745,7 +20745,7 @@ static struct cipher_testvec aes_ofb_enc_tv_template[] = { } }; -static struct cipher_testvec aes_ofb_dec_tv_template[] = { +static const struct cipher_testvec aes_ofb_dec_tv_template[] = { /* From NIST Special Publication 800-38A, Appendix F.5 */ { .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" @@ -20774,7 +20774,7 @@ static struct cipher_testvec aes_ofb_dec_tv_template[] = { } }; -static struct aead_testvec aes_gcm_enc_tv_template[] = { +static const struct aead_testvec aes_gcm_enc_tv_template[] = { { /* From McGrew & Viega - http://citeseer.ist.psu.edu/656989.html */ .key = zeroed_string, .klen = 16, @@ -20934,7 +20934,7 @@ static struct aead_testvec aes_gcm_enc_tv_template[] = { } }; -static struct aead_testvec aes_gcm_dec_tv_template[] = { +static const struct aead_testvec aes_gcm_dec_tv_template[] = { { /* From McGrew & Viega - http://citeseer.ist.psu.edu/656989.html */ .key = zeroed_string, .klen = 32, @@ -21136,7 +21136,7 @@ static struct aead_testvec aes_gcm_dec_tv_template[] = { } }; -static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { +static const struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { { /* Generated using Crypto++ */ .key = zeroed_string, .klen = 20, @@ -21749,7 +21749,7 @@ static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { } }; -static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { +static const struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { { /* Generated using Crypto++ */ .key = zeroed_string, .klen = 20, @@ -22363,7 +22363,7 @@ static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { } }; -static struct aead_testvec aes_gcm_rfc4543_enc_tv_template[] = { +static const struct aead_testvec aes_gcm_rfc4543_enc_tv_template[] = { { /* From draft-mcgrew-gcm-test-01 */ .key = "\x4c\x80\xcd\xef\xbb\x5d\x10\xda" "\x90\x6a\xc7\x3c\x36\x13\xa6\x34" @@ -22394,7 +22394,7 @@ static struct aead_testvec aes_gcm_rfc4543_enc_tv_template[] = { } }; -static struct aead_testvec aes_gcm_rfc4543_dec_tv_template[] = { +static const struct aead_testvec aes_gcm_rfc4543_dec_tv_template[] = { { /* From draft-mcgrew-gcm-test-01 */ .key = "\x4c\x80\xcd\xef\xbb\x5d\x10\xda" "\x90\x6a\xc7\x3c\x36\x13\xa6\x34" @@ -22453,7 +22453,7 @@ static struct aead_testvec aes_gcm_rfc4543_dec_tv_template[] = { }, }; -static struct aead_testvec aes_ccm_enc_tv_template[] = { +static const struct aead_testvec aes_ccm_enc_tv_template[] = { { /* From RFC 3610 */ .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf", @@ -22737,7 +22737,7 @@ static struct aead_testvec aes_ccm_enc_tv_template[] = { } }; -static struct aead_testvec aes_ccm_dec_tv_template[] = { +static const struct aead_testvec aes_ccm_dec_tv_template[] = { { /* From RFC 3610 */ .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf", @@ -23069,7 +23069,7 @@ static struct aead_testvec aes_ccm_dec_tv_template[] = { * These vectors are copied/generated from the ones for rfc4106 with * the key truncated by one byte.. */ -static struct aead_testvec aes_ccm_rfc4309_enc_tv_template[] = { +static const struct aead_testvec aes_ccm_rfc4309_enc_tv_template[] = { { /* Generated using Crypto++ */ .key = zeroed_string, .klen = 19, @@ -23682,7 +23682,7 @@ static struct aead_testvec aes_ccm_rfc4309_enc_tv_template[] = { } }; -static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { +static const struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { { /* Generated using Crypto++ */ .key = zeroed_string, .klen = 19, @@ -24298,7 +24298,7 @@ static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { /* * ChaCha20-Poly1305 AEAD test vectors from RFC7539 2.8.2./A.5. */ -static struct aead_testvec rfc7539_enc_tv_template[] = { +static const struct aead_testvec rfc7539_enc_tv_template[] = { { .key = "\x80\x81\x82\x83\x84\x85\x86\x87" "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" @@ -24430,7 +24430,7 @@ static struct aead_testvec rfc7539_enc_tv_template[] = { }, }; -static struct aead_testvec rfc7539_dec_tv_template[] = { +static const struct aead_testvec rfc7539_dec_tv_template[] = { { .key = "\x80\x81\x82\x83\x84\x85\x86\x87" "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" @@ -24565,7 +24565,7 @@ static struct aead_testvec rfc7539_dec_tv_template[] = { /* * draft-irtf-cfrg-chacha20-poly1305 */ -static struct aead_testvec rfc7539esp_enc_tv_template[] = { +static const struct aead_testvec rfc7539esp_enc_tv_template[] = { { .key = "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a" "\xf3\x33\x88\x86\x04\xf6\xb5\xf0" @@ -24653,7 +24653,7 @@ static struct aead_testvec rfc7539esp_enc_tv_template[] = { }, }; -static struct aead_testvec rfc7539esp_dec_tv_template[] = { +static const struct aead_testvec rfc7539esp_dec_tv_template[] = { { .key = "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a" "\xf3\x33\x88\x86\x04\xf6\xb5\xf0" @@ -24749,7 +24749,7 @@ static struct aead_testvec rfc7539esp_dec_tv_template[] = { * semiblock of the ciphertext from the test vector. For decryption, iv is * the first semiblock of the ciphertext. */ -static struct cipher_testvec aes_kw_enc_tv_template[] = { +static const struct cipher_testvec aes_kw_enc_tv_template[] = { { .key = "\x75\x75\xda\x3a\x93\x60\x7c\xc2" "\xbf\xd8\xce\xc7\xaa\xdf\xd9\xa6", @@ -24764,7 +24764,7 @@ static struct cipher_testvec aes_kw_enc_tv_template[] = { }, }; -static struct cipher_testvec aes_kw_dec_tv_template[] = { +static const struct cipher_testvec aes_kw_dec_tv_template[] = { { .key = "\x80\xaa\x99\x73\x27\xa4\x80\x6b" "\x6a\x7a\x41\xa5\x2b\x86\xc3\x71" @@ -24787,7 +24787,7 @@ static struct cipher_testvec aes_kw_dec_tv_template[] = { * http://csrc.nist.gov/groups/STM/cavp/documents/rng/RNGVS.pdf * Only AES-128 is supported at this time. */ -static struct cprng_testvec ansi_cprng_aes_tv_template[] = { +static const struct cprng_testvec ansi_cprng_aes_tv_template[] = { { .key = "\xf3\xb1\x66\x6d\x13\x60\x72\x42" "\xed\x06\x1c\xab\xb8\xd4\x62\x02", @@ -24883,7 +24883,7 @@ static struct cprng_testvec ansi_cprng_aes_tv_template[] = { * (Hash, HMAC, CTR) are tested with all permutations of use cases (w/ and * w/o personalization string, w/ and w/o additional input string). */ -static struct drbg_testvec drbg_pr_sha256_tv_template[] = { +static const struct drbg_testvec drbg_pr_sha256_tv_template[] = { { .entropy = (unsigned char *) "\x72\x88\x4c\xcd\x6c\x85\x57\x70\xf7\x0b\x8b\x86" @@ -25041,7 +25041,7 @@ static struct drbg_testvec drbg_pr_sha256_tv_template[] = { }, }; -static struct drbg_testvec drbg_pr_hmac_sha256_tv_template[] = { +static const struct drbg_testvec drbg_pr_hmac_sha256_tv_template[] = { { .entropy = (unsigned char *) "\x99\x69\xe5\x4b\x47\x03\xff\x31\x78\x5b\x87\x9a" @@ -25199,7 +25199,7 @@ static struct drbg_testvec drbg_pr_hmac_sha256_tv_template[] = { }, }; -static struct drbg_testvec drbg_pr_ctr_aes128_tv_template[] = { +static const struct drbg_testvec drbg_pr_ctr_aes128_tv_template[] = { { .entropy = (unsigned char *) "\xd1\x44\xc6\x61\x81\x6d\xca\x9d\x15\x28\x8a\x42" @@ -25323,7 +25323,7 @@ static struct drbg_testvec drbg_pr_ctr_aes128_tv_template[] = { * (Hash, HMAC, CTR) are tested with all permutations of use cases (w/ and * w/o personalization string, w/ and w/o additional input string). */ -static struct drbg_testvec drbg_nopr_sha256_tv_template[] = { +static const struct drbg_testvec drbg_nopr_sha256_tv_template[] = { { .entropy = (unsigned char *) "\xa6\x5a\xd0\xf3\x45\xdb\x4e\x0e\xff\xe8\x75\xc3" @@ -25445,7 +25445,7 @@ static struct drbg_testvec drbg_nopr_sha256_tv_template[] = { }, }; -static struct drbg_testvec drbg_nopr_hmac_sha256_tv_template[] = { +static const struct drbg_testvec drbg_nopr_hmac_sha256_tv_template[] = { { .entropy = (unsigned char *) "\xca\x85\x19\x11\x34\x93\x84\xbf\xfe\x89\xde\x1c" @@ -25567,7 +25567,7 @@ static struct drbg_testvec drbg_nopr_hmac_sha256_tv_template[] = { }, }; -static struct drbg_testvec drbg_nopr_ctr_aes192_tv_template[] = { +static const struct drbg_testvec drbg_nopr_ctr_aes192_tv_template[] = { { .entropy = (unsigned char *) "\xc3\x5c\x2f\xa2\xa8\x9d\x52\xa1\x1f\xa3\x2a\xa9" @@ -25591,7 +25591,7 @@ static struct drbg_testvec drbg_nopr_ctr_aes192_tv_template[] = { }, }; -static struct drbg_testvec drbg_nopr_ctr_aes256_tv_template[] = { +static const struct drbg_testvec drbg_nopr_ctr_aes256_tv_template[] = { { .entropy = (unsigned char *) "\x36\x40\x19\x40\xfa\x8b\x1f\xba\x91\xa1\x66\x1f" @@ -25615,7 +25615,7 @@ static struct drbg_testvec drbg_nopr_ctr_aes256_tv_template[] = { }, }; -static struct drbg_testvec drbg_nopr_ctr_aes128_tv_template[] = { +static const struct drbg_testvec drbg_nopr_ctr_aes128_tv_template[] = { { .entropy = (unsigned char *) "\x87\xe1\xc5\x32\x99\x7f\x57\xa3\x5c\x28\x6d\xe8" @@ -25704,7 +25704,7 @@ static struct drbg_testvec drbg_nopr_ctr_aes128_tv_template[] = { }; /* Cast5 test vectors from RFC 2144 */ -static struct cipher_testvec cast5_enc_tv_template[] = { +static const struct cipher_testvec cast5_enc_tv_template[] = { { .key = "\x01\x23\x45\x67\x12\x34\x56\x78" "\x23\x45\x67\x89\x34\x56\x78\x9a", @@ -25865,7 +25865,7 @@ static struct cipher_testvec cast5_enc_tv_template[] = { }, }; -static struct cipher_testvec cast5_dec_tv_template[] = { +static const struct cipher_testvec cast5_dec_tv_template[] = { { .key = "\x01\x23\x45\x67\x12\x34\x56\x78" "\x23\x45\x67\x89\x34\x56\x78\x9a", @@ -26026,7 +26026,7 @@ static struct cipher_testvec cast5_dec_tv_template[] = { }, }; -static struct cipher_testvec cast5_cbc_enc_tv_template[] = { +static const struct cipher_testvec cast5_cbc_enc_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A", @@ -26164,7 +26164,7 @@ static struct cipher_testvec cast5_cbc_enc_tv_template[] = { }, }; -static struct cipher_testvec cast5_cbc_dec_tv_template[] = { +static const struct cipher_testvec cast5_cbc_dec_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A", @@ -26302,7 +26302,7 @@ static struct cipher_testvec cast5_cbc_dec_tv_template[] = { }, }; -static struct cipher_testvec cast5_ctr_enc_tv_template[] = { +static const struct cipher_testvec cast5_ctr_enc_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A", @@ -26453,7 +26453,7 @@ static struct cipher_testvec cast5_ctr_enc_tv_template[] = { }, }; -static struct cipher_testvec cast5_ctr_dec_tv_template[] = { +static const struct cipher_testvec cast5_ctr_dec_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A", @@ -26607,7 +26607,7 @@ static struct cipher_testvec cast5_ctr_dec_tv_template[] = { /* * ARC4 test vectors from OpenSSL */ -static struct cipher_testvec arc4_enc_tv_template[] = { +static const struct cipher_testvec arc4_enc_tv_template[] = { { .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", .klen = 8, @@ -26673,7 +26673,7 @@ static struct cipher_testvec arc4_enc_tv_template[] = { }, }; -static struct cipher_testvec arc4_dec_tv_template[] = { +static const struct cipher_testvec arc4_dec_tv_template[] = { { .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", .klen = 8, @@ -26742,7 +26742,7 @@ static struct cipher_testvec arc4_dec_tv_template[] = { /* * TEA test vectors */ -static struct cipher_testvec tea_enc_tv_template[] = { +static const struct cipher_testvec tea_enc_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -26785,7 +26785,7 @@ static struct cipher_testvec tea_enc_tv_template[] = { } }; -static struct cipher_testvec tea_dec_tv_template[] = { +static const struct cipher_testvec tea_dec_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -26831,7 +26831,7 @@ static struct cipher_testvec tea_dec_tv_template[] = { /* * XTEA test vectors */ -static struct cipher_testvec xtea_enc_tv_template[] = { +static const struct cipher_testvec xtea_enc_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -26874,7 +26874,7 @@ static struct cipher_testvec xtea_enc_tv_template[] = { } }; -static struct cipher_testvec xtea_dec_tv_template[] = { +static const struct cipher_testvec xtea_dec_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -26920,7 +26920,7 @@ static struct cipher_testvec xtea_dec_tv_template[] = { /* * KHAZAD test vectors. */ -static struct cipher_testvec khazad_enc_tv_template[] = { +static const struct cipher_testvec khazad_enc_tv_template[] = { { .key = "\x80\x00\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00\x00\x00\x00\x00", @@ -26966,7 +26966,7 @@ static struct cipher_testvec khazad_enc_tv_template[] = { }, }; -static struct cipher_testvec khazad_dec_tv_template[] = { +static const struct cipher_testvec khazad_dec_tv_template[] = { { .key = "\x80\x00\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00\x00\x00\x00\x00", @@ -27016,7 +27016,7 @@ static struct cipher_testvec khazad_dec_tv_template[] = { * Anubis test vectors. */ -static struct cipher_testvec anubis_enc_tv_template[] = { +static const struct cipher_testvec anubis_enc_tv_template[] = { { .key = "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe" "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe", @@ -27079,7 +27079,7 @@ static struct cipher_testvec anubis_enc_tv_template[] = { }, }; -static struct cipher_testvec anubis_dec_tv_template[] = { +static const struct cipher_testvec anubis_dec_tv_template[] = { { .key = "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe" "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe", @@ -27142,7 +27142,7 @@ static struct cipher_testvec anubis_dec_tv_template[] = { }, }; -static struct cipher_testvec anubis_cbc_enc_tv_template[] = { +static const struct cipher_testvec anubis_cbc_enc_tv_template[] = { { .key = "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe" "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe", @@ -27177,7 +27177,7 @@ static struct cipher_testvec anubis_cbc_enc_tv_template[] = { }, }; -static struct cipher_testvec anubis_cbc_dec_tv_template[] = { +static const struct cipher_testvec anubis_cbc_dec_tv_template[] = { { .key = "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe" "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe", @@ -27215,7 +27215,7 @@ static struct cipher_testvec anubis_cbc_dec_tv_template[] = { /* * XETA test vectors */ -static struct cipher_testvec xeta_enc_tv_template[] = { +static const struct cipher_testvec xeta_enc_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -27258,7 +27258,7 @@ static struct cipher_testvec xeta_enc_tv_template[] = { } }; -static struct cipher_testvec xeta_dec_tv_template[] = { +static const struct cipher_testvec xeta_dec_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -27304,7 +27304,7 @@ static struct cipher_testvec xeta_dec_tv_template[] = { /* * FCrypt test vectors */ -static struct cipher_testvec fcrypt_pcbc_enc_tv_template[] = { +static const struct cipher_testvec fcrypt_pcbc_enc_tv_template[] = { { /* http://www.openafs.org/pipermail/openafs-devel/2000-December/005320.html */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00", .klen = 8, @@ -27365,7 +27365,7 @@ static struct cipher_testvec fcrypt_pcbc_enc_tv_template[] = { } }; -static struct cipher_testvec fcrypt_pcbc_dec_tv_template[] = { +static const struct cipher_testvec fcrypt_pcbc_dec_tv_template[] = { { /* http://www.openafs.org/pipermail/openafs-devel/2000-December/005320.html */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00", .klen = 8, @@ -27429,7 +27429,7 @@ static struct cipher_testvec fcrypt_pcbc_dec_tv_template[] = { /* * CAMELLIA test vectors. */ -static struct cipher_testvec camellia_enc_tv_template[] = { +static const struct cipher_testvec camellia_enc_tv_template[] = { { .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" "\xfe\xdc\xba\x98\x76\x54\x32\x10", @@ -27729,7 +27729,7 @@ static struct cipher_testvec camellia_enc_tv_template[] = { }, }; -static struct cipher_testvec camellia_dec_tv_template[] = { +static const struct cipher_testvec camellia_dec_tv_template[] = { { .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" "\xfe\xdc\xba\x98\x76\x54\x32\x10", @@ -28029,7 +28029,7 @@ static struct cipher_testvec camellia_dec_tv_template[] = { }, }; -static struct cipher_testvec camellia_cbc_enc_tv_template[] = { +static const struct cipher_testvec camellia_cbc_enc_tv_template[] = { { .key = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b" "\x51\x2e\x03\xd5\x34\x12\x00\x06", @@ -28325,7 +28325,7 @@ static struct cipher_testvec camellia_cbc_enc_tv_template[] = { }, }; -static struct cipher_testvec camellia_cbc_dec_tv_template[] = { +static const struct cipher_testvec camellia_cbc_dec_tv_template[] = { { .key = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b" "\x51\x2e\x03\xd5\x34\x12\x00\x06", @@ -28621,7 +28621,7 @@ static struct cipher_testvec camellia_cbc_dec_tv_template[] = { }, }; -static struct cipher_testvec camellia_ctr_enc_tv_template[] = { +static const struct cipher_testvec camellia_ctr_enc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -29288,7 +29288,7 @@ static struct cipher_testvec camellia_ctr_enc_tv_template[] = { }, }; -static struct cipher_testvec camellia_ctr_dec_tv_template[] = { +static const struct cipher_testvec camellia_ctr_dec_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -29955,7 +29955,7 @@ static struct cipher_testvec camellia_ctr_dec_tv_template[] = { }, }; -static struct cipher_testvec camellia_lrw_enc_tv_template[] = { +static const struct cipher_testvec camellia_lrw_enc_tv_template[] = { /* Generated from AES-LRW test vectors */ { .key = "\x45\x62\xac\x25\xf8\x28\x17\x6d" @@ -30207,7 +30207,7 @@ static struct cipher_testvec camellia_lrw_enc_tv_template[] = { }, }; -static struct cipher_testvec camellia_lrw_dec_tv_template[] = { +static const struct cipher_testvec camellia_lrw_dec_tv_template[] = { /* Generated from AES-LRW test vectors */ /* same as enc vectors with input and result reversed */ { @@ -30460,7 +30460,7 @@ static struct cipher_testvec camellia_lrw_dec_tv_template[] = { }, }; -static struct cipher_testvec camellia_xts_enc_tv_template[] = { +static const struct cipher_testvec camellia_xts_enc_tv_template[] = { /* Generated from AES-XTS test vectors */ { .key = "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -30802,7 +30802,7 @@ static struct cipher_testvec camellia_xts_enc_tv_template[] = { }, }; -static struct cipher_testvec camellia_xts_dec_tv_template[] = { +static const struct cipher_testvec camellia_xts_dec_tv_template[] = { /* Generated from AES-XTS test vectors */ /* same as enc vectors with input and result reversed */ { @@ -31148,7 +31148,7 @@ static struct cipher_testvec camellia_xts_dec_tv_template[] = { /* * SEED test vectors */ -static struct cipher_testvec seed_enc_tv_template[] = { +static const struct cipher_testvec seed_enc_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -31190,7 +31190,7 @@ static struct cipher_testvec seed_enc_tv_template[] = { } }; -static struct cipher_testvec seed_dec_tv_template[] = { +static const struct cipher_testvec seed_dec_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -31232,7 +31232,7 @@ static struct cipher_testvec seed_dec_tv_template[] = { } }; -static struct cipher_testvec salsa20_stream_enc_tv_template[] = { +static const struct cipher_testvec salsa20_stream_enc_tv_template[] = { /* * Testvectors from verified.test-vectors submitted to ECRYPT. * They are truncated to size 39, 64, 111, 129 to test a variety @@ -32401,7 +32401,7 @@ static struct cipher_testvec salsa20_stream_enc_tv_template[] = { }, }; -static struct cipher_testvec chacha20_enc_tv_template[] = { +static const struct cipher_testvec chacha20_enc_tv_template[] = { { /* RFC7539 A.2. Test Vector #1 */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -32912,7 +32912,7 @@ static struct cipher_testvec chacha20_enc_tv_template[] = { /* * CTS (Cipher Text Stealing) mode tests */ -static struct cipher_testvec cts_mode_enc_tv_template[] = { +static const struct cipher_testvec cts_mode_enc_tv_template[] = { { /* from rfc3962 */ .klen = 16, .key = "\x63\x68\x69\x63\x6b\x65\x6e\x20" @@ -33014,7 +33014,7 @@ static struct cipher_testvec cts_mode_enc_tv_template[] = { } }; -static struct cipher_testvec cts_mode_dec_tv_template[] = { +static const struct cipher_testvec cts_mode_dec_tv_template[] = { { /* from rfc3962 */ .klen = 16, .key = "\x63\x68\x69\x63\x6b\x65\x6e\x20" @@ -33132,7 +33132,7 @@ struct comp_testvec { * Params: winbits=-11, Z_DEFAULT_COMPRESSION, MAX_MEM_LEVEL. */ -static struct comp_testvec deflate_comp_tv_template[] = { +static const struct comp_testvec deflate_comp_tv_template[] = { { .inlen = 70, .outlen = 38, @@ -33168,7 +33168,7 @@ static struct comp_testvec deflate_comp_tv_template[] = { }, }; -static struct comp_testvec deflate_decomp_tv_template[] = { +static const struct comp_testvec deflate_decomp_tv_template[] = { { .inlen = 122, .outlen = 191, @@ -33207,7 +33207,7 @@ static struct comp_testvec deflate_decomp_tv_template[] = { /* * LZO test vectors (null-terminated strings). */ -static struct comp_testvec lzo_comp_tv_template[] = { +static const struct comp_testvec lzo_comp_tv_template[] = { { .inlen = 70, .outlen = 57, @@ -33247,7 +33247,7 @@ static struct comp_testvec lzo_comp_tv_template[] = { }, }; -static struct comp_testvec lzo_decomp_tv_template[] = { +static const struct comp_testvec lzo_decomp_tv_template[] = { { .inlen = 133, .outlen = 159, @@ -33290,7 +33290,7 @@ static struct comp_testvec lzo_decomp_tv_template[] = { */ #define MICHAEL_MIC_TEST_VECTORS 6 -static struct hash_testvec michael_mic_tv_template[] = { +static const struct hash_testvec michael_mic_tv_template[] = { { .key = "\x00\x00\x00\x00\x00\x00\x00\x00", .ksize = 8, @@ -33338,7 +33338,7 @@ static struct hash_testvec michael_mic_tv_template[] = { /* * CRC32 test vectors */ -static struct hash_testvec crc32_tv_template[] = { +static const struct hash_testvec crc32_tv_template[] = { { .key = "\x87\xa9\xcb\xed", .ksize = 4, @@ -33770,7 +33770,7 @@ static struct hash_testvec crc32_tv_template[] = { /* * CRC32C test vectors */ -static struct hash_testvec crc32c_tv_template[] = { +static const struct hash_testvec crc32c_tv_template[] = { { .psize = 0, .digest = "\x00\x00\x00\x00", @@ -34206,7 +34206,7 @@ static struct hash_testvec crc32c_tv_template[] = { /* * Blakcifn CRC test vectors */ -static struct hash_testvec bfin_crc_tv_template[] = { +static const struct hash_testvec bfin_crc_tv_template[] = { { .psize = 0, .digest = "\x00\x00\x00\x00", @@ -34291,7 +34291,7 @@ static struct hash_testvec bfin_crc_tv_template[] = { }; -static struct comp_testvec lz4_comp_tv_template[] = { +static const struct comp_testvec lz4_comp_tv_template[] = { { .inlen = 255, .outlen = 218, @@ -34322,7 +34322,7 @@ static struct comp_testvec lz4_comp_tv_template[] = { }, }; -static struct comp_testvec lz4_decomp_tv_template[] = { +static const struct comp_testvec lz4_decomp_tv_template[] = { { .inlen = 218, .outlen = 255, @@ -34352,7 +34352,7 @@ static struct comp_testvec lz4_decomp_tv_template[] = { }, }; -static struct comp_testvec lz4hc_comp_tv_template[] = { +static const struct comp_testvec lz4hc_comp_tv_template[] = { { .inlen = 255, .outlen = 216, @@ -34383,7 +34383,7 @@ static struct comp_testvec lz4hc_comp_tv_template[] = { }, }; -static struct comp_testvec lz4hc_decomp_tv_template[] = { +static const struct comp_testvec lz4hc_decomp_tv_template[] = { { .inlen = 216, .outlen = 255, From e6c2e65c70a6f606ea764f301e4024c85e0cd7a8 Mon Sep 17 00:00:00 2001 From: Marcelo Cerri Date: Mon, 27 Feb 2017 09:38:25 -0300 Subject: [PATCH 013/101] crypto: cbc - Propagate NEED_FALLBACK bit When requesting a fallback algorithm, we should propagate the NEED_FALLBACK bit when search for the underlying algorithm. This will prevents drivers from allocating unnecessary fallbacks that are never called. For instance, currently the vmx-crypto driver will use the following chain of calls when calling the fallback implementation: p8_aes_cbc -> cbc(p8_aes) -> aes-generic However p8_aes will always delegate its calls to aes-generic. With this patch, p8_aes_cbc will be able to use cbc(aes-generic) directly as its fallback. The same applies to aes_s390. Signed-off-by: Marcelo Henrique Cerri Signed-off-by: Herbert Xu --- crypto/cbc.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/crypto/cbc.c b/crypto/cbc.c index bc160a3186dc..b761b1f9c6ca 100644 --- a/crypto/cbc.c +++ b/crypto/cbc.c @@ -10,6 +10,7 @@ * */ +#include #include #include #include @@ -108,8 +109,10 @@ static void crypto_cbc_free(struct skcipher_instance *inst) static int crypto_cbc_create(struct crypto_template *tmpl, struct rtattr **tb) { struct skcipher_instance *inst; + struct crypto_attr_type *algt; struct crypto_spawn *spawn; struct crypto_alg *alg; + u32 mask; int err; err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SKCIPHER); @@ -120,8 +123,16 @@ static int crypto_cbc_create(struct crypto_template *tmpl, struct rtattr **tb) if (!inst) return -ENOMEM; - alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, - CRYPTO_ALG_TYPE_MASK); + algt = crypto_get_attr_type(tb); + err = PTR_ERR(algt); + if (IS_ERR(algt)) + goto err_free_inst; + + mask = CRYPTO_ALG_TYPE_MASK | + crypto_requires_off(algt->type, algt->mask, + CRYPTO_ALG_NEED_FALLBACK); + + alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, mask); err = PTR_ERR(alg); if (IS_ERR(alg)) goto err_free_inst; From d2c2a85cfe829f9d0736dba567edc86ba8524fb2 Mon Sep 17 00:00:00 2001 From: Marcelo Cerri Date: Mon, 27 Feb 2017 09:38:26 -0300 Subject: [PATCH 014/101] crypto: ctr - Propagate NEED_FALLBACK bit When requesting a fallback algorithm, we should propagate the NEED_FALLBACK bit when search for the underlying algorithm. This will prevents drivers from allocating unnecessary fallbacks that are never called. For instance, currently the vmx-crypto driver will use the following chain of calls when calling the fallback implementation: p8_aes_ctr -> ctr(p8_aes) -> aes-generic However p8_aes will always delegate its calls to aes-generic. With this patch, p8_aes_ctr will be able to use ctr(aes-generic) directly as its fallback. The same applies to aes_s390. Signed-off-by: Marcelo Henrique Cerri Signed-off-by: Herbert Xu --- crypto/ctr.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/crypto/ctr.c b/crypto/ctr.c index a4f4a8983169..477d9226ccaa 100644 --- a/crypto/ctr.c +++ b/crypto/ctr.c @@ -181,15 +181,24 @@ static void crypto_ctr_exit_tfm(struct crypto_tfm *tfm) static struct crypto_instance *crypto_ctr_alloc(struct rtattr **tb) { struct crypto_instance *inst; + struct crypto_attr_type *algt; struct crypto_alg *alg; + u32 mask; int err; err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER); if (err) return ERR_PTR(err); - alg = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_CIPHER, - CRYPTO_ALG_TYPE_MASK); + algt = crypto_get_attr_type(tb); + if (IS_ERR(algt)) + return ERR_CAST(algt); + + mask = CRYPTO_ALG_TYPE_MASK | + crypto_requires_off(algt->type, algt->mask, + CRYPTO_ALG_NEED_FALLBACK); + + alg = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_CIPHER, mask); if (IS_ERR(alg)) return ERR_CAST(alg); @@ -350,6 +359,8 @@ static int crypto_rfc3686_create(struct crypto_template *tmpl, struct skcipher_alg *alg; struct crypto_skcipher_spawn *spawn; const char *cipher_name; + u32 mask; + int err; algt = crypto_get_attr_type(tb); @@ -367,12 +378,14 @@ static int crypto_rfc3686_create(struct crypto_template *tmpl, if (!inst) return -ENOMEM; + mask = crypto_requires_sync(algt->type, algt->mask) | + crypto_requires_off(algt->type, algt->mask, + CRYPTO_ALG_NEED_FALLBACK); + spawn = skcipher_instance_ctx(inst); crypto_set_skcipher_spawn(spawn, skcipher_crypto_instance(inst)); - err = crypto_grab_skcipher(spawn, cipher_name, 0, - crypto_requires_sync(algt->type, - algt->mask)); + err = crypto_grab_skcipher(spawn, cipher_name, 0, mask); if (err) goto err_free_inst; From eff041553d2e4d7978e47f99e7cffb866c71d98f Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Wed, 22 Feb 2017 07:55:24 +0100 Subject: [PATCH 015/101] clk: meson-gxbb: expose clock CLKID_RNG0 Expose clock CLKID_RNG0 which is needed for the HW random number generator. Signed-off-by: Heiner Kallweit Reviewed-by: Neil Armstrong Signed-off-by: Herbert Xu --- drivers/clk/meson/gxbb.h | 2 +- include/dt-bindings/clock/gxbb-clkc.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/clk/meson/gxbb.h b/drivers/clk/meson/gxbb.h index 8ee2022ce5d5..cbd62e46bb5b 100644 --- a/drivers/clk/meson/gxbb.h +++ b/drivers/clk/meson/gxbb.h @@ -193,7 +193,7 @@ /* CLKID_I2C */ /* #define CLKID_SAR_ADC */ #define CLKID_SMART_CARD 24 -#define CLKID_RNG0 25 +/* CLKID_RNG0 */ #define CLKID_UART0 26 #define CLKID_SDHC 27 #define CLKID_STREAM 28 diff --git a/include/dt-bindings/clock/gxbb-clkc.h b/include/dt-bindings/clock/gxbb-clkc.h index 692846c7941b..63f4c2c44a1f 100644 --- a/include/dt-bindings/clock/gxbb-clkc.h +++ b/include/dt-bindings/clock/gxbb-clkc.h @@ -12,9 +12,10 @@ #define CLKID_FCLK_DIV4 6 #define CLKID_CLK81 12 #define CLKID_MPLL2 15 -#define CLKID_SPI 34 #define CLKID_I2C 22 #define CLKID_SAR_ADC 23 +#define CLKID_RNG0 25 +#define CLKID_SPI 34 #define CLKID_ETH 36 #define CLKID_USB0 50 #define CLKID_USB1 51 From e56e12acfe6f5895299f420174e111b49ef69380 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Wed, 22 Feb 2017 07:59:08 +0100 Subject: [PATCH 016/101] dt-bindings: rng: add clock to DT binding documentation for hwrng Add clock to DT binding documentation. Signed-off-by: Heiner Kallweit Acked-by: Rob Herring Signed-off-by: Herbert Xu --- .../devicetree/bindings/rng/amlogic,meson-rng.txt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/rng/amlogic,meson-rng.txt b/Documentation/devicetree/bindings/rng/amlogic,meson-rng.txt index 202f2d09a23f..4d403645ac9b 100644 --- a/Documentation/devicetree/bindings/rng/amlogic,meson-rng.txt +++ b/Documentation/devicetree/bindings/rng/amlogic,meson-rng.txt @@ -6,9 +6,16 @@ Required properties: - compatible : should be "amlogic,meson-rng" - reg : Specifies base physical address and size of the registers. +Optional properties: + +- clocks : phandle to the following named clocks +- clock-names: Name of core clock, must be "core" + Example: rng { - compatible = "amlogic,meson-rng"; - reg = <0x0 0xc8834000 0x0 0x4>; + compatible = "amlogic,meson-rng"; + reg = <0x0 0xc8834000 0x0 0x4>; + clocks = <&clkc CLKID_RNG0>; + clock-names = "core"; }; From 1b3f6d148692ef3c3c96af1647083ff38d429e46 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Wed, 22 Feb 2017 08:00:55 +0100 Subject: [PATCH 017/101] ARM64: dts: meson-gx: add clock CLKID_RNG0 to hwrng node Add clock CLKID_RNG0 to HW randon number generator node. Signed-off-by: Heiner Kallweit Signed-off-by: Herbert Xu --- arch/arm64/boot/dts/amlogic/meson-gx.dtsi | 2 +- arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/amlogic/meson-gx.dtsi b/arch/arm64/boot/dts/amlogic/meson-gx.dtsi index 5d995f7724af..620495a43363 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gx.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-gx.dtsi @@ -380,7 +380,7 @@ #size-cells = <2>; ranges = <0x0 0x0 0x0 0xc8834000 0x0 0x2000>; - rng { + hwrng: rng { compatible = "amlogic,meson-rng"; reg = <0x0 0x0 0x0 0x4>; }; diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi index 04b3324bc132..a375cb21cc8b 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi @@ -524,3 +524,8 @@ &vpu { compatible = "amlogic,meson-gxbb-vpu", "amlogic,meson-gx-vpu"; }; + +&hwrng { + clocks = <&clkc CLKID_RNG0>; + clock-names = "core"; +}; From 7a794a73d4440cb257f541cc29630c7560266cf7 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Wed, 22 Feb 2017 08:02:31 +0100 Subject: [PATCH 018/101] hwrng: meson - add clock handling to driver Add handling of RNG0 clock to the driver. Signed-off-by: Heiner Kallweit Signed-off-by: Herbert Xu --- drivers/char/hw_random/meson-rng.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/drivers/char/hw_random/meson-rng.c b/drivers/char/hw_random/meson-rng.c index 119d698439ae..2e23be802a62 100644 --- a/drivers/char/hw_random/meson-rng.c +++ b/drivers/char/hw_random/meson-rng.c @@ -62,6 +62,7 @@ #include #include #include +#include #define RNG_DATA 0x00 @@ -69,6 +70,7 @@ struct meson_rng_data { void __iomem *base; struct platform_device *pdev; struct hwrng rng; + struct clk *core_clk; }; static int meson_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) @@ -81,11 +83,17 @@ static int meson_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) return sizeof(u32); } +static void meson_rng_clk_disable(void *data) +{ + clk_disable_unprepare(data); +} + static int meson_rng_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct meson_rng_data *data; struct resource *res; + int ret; data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) @@ -98,6 +106,20 @@ static int meson_rng_probe(struct platform_device *pdev) if (IS_ERR(data->base)) return PTR_ERR(data->base); + data->core_clk = devm_clk_get(dev, "core"); + if (IS_ERR(data->core_clk)) + data->core_clk = NULL; + + if (data->core_clk) { + ret = clk_prepare_enable(data->core_clk); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, meson_rng_clk_disable, + data->core_clk); + if (ret) + return ret; + } + data->rng.name = pdev->name; data->rng.read = meson_rng_read; From 43ec540e6f9b8e795dc9000114636ff72afc5b01 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Tue, 7 Mar 2017 15:14:49 +0100 Subject: [PATCH 019/101] hwrng: omap - move clock related code to omap_rng_probe() Currently, the code that takes a reference to the clock and enables it is located inside of_get_omap_rng_device_details(), called only when probing through the Device Tree. However, there is nothing that makes this clock logic dependent on the Device Tree, so it makes more sense to have it in omap_rng_probe() directly. Moreover, we make sure to bail out if we can't enable the clock. Indeed, while the clock is optional, if a clock is present, we really want to succeed in enabling it. And we fix the error message to fit on one line, so that it is grep-friendly. Signed-off-by: Thomas Petazzoni Signed-off-by: Herbert Xu --- drivers/char/hw_random/omap-rng.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c index b1ad12552b56..74d11ae6abe9 100644 --- a/drivers/char/hw_random/omap-rng.c +++ b/drivers/char/hw_random/omap-rng.c @@ -398,16 +398,6 @@ static int of_get_omap_rng_device_details(struct omap_rng_dev *priv, return err; } - priv->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(priv->clk) && PTR_ERR(priv->clk) == -EPROBE_DEFER) - return -EPROBE_DEFER; - if (!IS_ERR(priv->clk)) { - err = clk_prepare_enable(priv->clk); - if (err) - dev_err(&pdev->dev, "unable to enable the clk, " - "err = %d\n", err); - } - /* * On OMAP4, enabling the shutdown_oflo interrupt is * done in the interrupt mask register. There is no @@ -478,6 +468,18 @@ static int omap_rng_probe(struct platform_device *pdev) goto err_ioremap; } + priv->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(priv->clk) && PTR_ERR(priv->clk) == -EPROBE_DEFER) + return -EPROBE_DEFER; + if (!IS_ERR(priv->clk)) { + ret = clk_prepare_enable(priv->clk); + if (ret) { + dev_err(&pdev->dev, + "Unable to enable the clk: %d\n", ret); + goto err_register; + } + } + ret = (dev->of_node) ? of_get_omap_rng_device_details(priv, pdev) : get_omap_rng_device_details(priv); if (ret) From 132c57caefeddde914350d42bb6e3b5a3688faa5 Mon Sep 17 00:00:00 2001 From: Ryder Lee Date: Thu, 9 Mar 2017 10:11:12 +0800 Subject: [PATCH 020/101] crypto: mediatek - rework interrupt handler This patch removes redundant task that used to handle interrupt from ring manager, so that the same task/handler can be shared. It also uses aes->id and sha-id to distinguish interrupt sources. Signed-off-by: Ryder Lee Signed-off-by: Herbert Xu --- drivers/crypto/mediatek/mtk-aes.c | 73 ++++++++------------------ drivers/crypto/mediatek/mtk-platform.h | 4 ++ drivers/crypto/mediatek/mtk-sha.c | 73 ++++++++------------------ 3 files changed, 49 insertions(+), 101 deletions(-) diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c index 3a47cdb8f0c8..e67881f7430a 100644 --- a/drivers/crypto/mediatek/mtk-aes.c +++ b/drivers/crypto/mediatek/mtk-aes.c @@ -1092,55 +1092,26 @@ static struct aead_alg aes_gcm_alg = { }, }; -static void mtk_aes_enc_task(unsigned long data) +static void mtk_aes_done_task(unsigned long data) { - struct mtk_cryp *cryp = (struct mtk_cryp *)data; - struct mtk_aes_rec *aes = cryp->aes[0]; + struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data; + struct mtk_cryp *cryp = aes->cryp; mtk_aes_unmap(cryp, aes); aes->resume(cryp, aes); } -static void mtk_aes_dec_task(unsigned long data) +static irqreturn_t mtk_aes_irq(int irq, void *dev_id) { - struct mtk_cryp *cryp = (struct mtk_cryp *)data; - struct mtk_aes_rec *aes = cryp->aes[1]; + struct mtk_aes_rec *aes = (struct mtk_aes_rec *)dev_id; + struct mtk_cryp *cryp = aes->cryp; + u32 val = mtk_aes_read(cryp, RDR_STAT(aes->id)); - mtk_aes_unmap(cryp, aes); - aes->resume(cryp, aes); -} - -static irqreturn_t mtk_aes_enc_irq(int irq, void *dev_id) -{ - struct mtk_cryp *cryp = (struct mtk_cryp *)dev_id; - struct mtk_aes_rec *aes = cryp->aes[0]; - u32 val = mtk_aes_read(cryp, RDR_STAT(RING0)); - - mtk_aes_write(cryp, RDR_STAT(RING0), val); + mtk_aes_write(cryp, RDR_STAT(aes->id), val); if (likely(AES_FLAGS_BUSY & aes->flags)) { - mtk_aes_write(cryp, RDR_PROC_COUNT(RING0), MTK_CNT_RST); - mtk_aes_write(cryp, RDR_THRESH(RING0), - MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); - - tasklet_schedule(&aes->task); - } else { - dev_warn(cryp->dev, "AES interrupt when no active requests.\n"); - } - return IRQ_HANDLED; -} - -static irqreturn_t mtk_aes_dec_irq(int irq, void *dev_id) -{ - struct mtk_cryp *cryp = (struct mtk_cryp *)dev_id; - struct mtk_aes_rec *aes = cryp->aes[1]; - u32 val = mtk_aes_read(cryp, RDR_STAT(RING1)); - - mtk_aes_write(cryp, RDR_STAT(RING1), val); - - if (likely(AES_FLAGS_BUSY & aes->flags)) { - mtk_aes_write(cryp, RDR_PROC_COUNT(RING1), MTK_CNT_RST); - mtk_aes_write(cryp, RDR_THRESH(RING1), + mtk_aes_write(cryp, RDR_PROC_COUNT(aes->id), MTK_CNT_RST); + mtk_aes_write(cryp, RDR_THRESH(aes->id), MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); tasklet_schedule(&aes->task); @@ -1171,14 +1142,18 @@ static int mtk_aes_record_init(struct mtk_cryp *cryp) if (!aes[i]->buf) goto err_cleanup; - aes[i]->id = i; + aes[i]->cryp = cryp; spin_lock_init(&aes[i]->lock); crypto_init_queue(&aes[i]->queue, AES_QUEUE_SIZE); + + tasklet_init(&aes[i]->task, mtk_aes_done_task, + (unsigned long)aes[i]); } - tasklet_init(&aes[0]->task, mtk_aes_enc_task, (unsigned long)cryp); - tasklet_init(&aes[1]->task, mtk_aes_dec_task, (unsigned long)cryp); + /* Link to ring0 and ring1 respectively */ + aes[0]->id = RING0; + aes[1]->id = RING1; return 0; @@ -1246,19 +1221,17 @@ int mtk_cipher_alg_register(struct mtk_cryp *cryp) if (ret) goto err_record; - /* Ring0 is use by encryption record */ - ret = devm_request_irq(cryp->dev, cryp->irq[RING0], mtk_aes_enc_irq, - IRQF_TRIGGER_LOW, "mtk-aes", cryp); + ret = devm_request_irq(cryp->dev, cryp->irq[RING0], mtk_aes_irq, + 0, "mtk-aes", cryp->aes[0]); if (ret) { - dev_err(cryp->dev, "unable to request AES encryption irq.\n"); + dev_err(cryp->dev, "unable to request AES irq.\n"); goto err_res; } - /* Ring1 is use by decryption record */ - ret = devm_request_irq(cryp->dev, cryp->irq[RING1], mtk_aes_dec_irq, - IRQF_TRIGGER_LOW, "mtk-aes", cryp); + ret = devm_request_irq(cryp->dev, cryp->irq[RING1], mtk_aes_irq, + 0, "mtk-aes", cryp->aes[1]); if (ret) { - dev_err(cryp->dev, "unable to request AES decryption irq.\n"); + dev_err(cryp->dev, "unable to request AES irq.\n"); goto err_res; } diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h index ed6d8717f7f4..d2a1db6bddc6 100644 --- a/drivers/crypto/mediatek/mtk-platform.h +++ b/drivers/crypto/mediatek/mtk-platform.h @@ -125,6 +125,7 @@ typedef int (*mtk_aes_fn)(struct mtk_cryp *cryp, struct mtk_aes_rec *aes); /** * struct mtk_aes_rec - AES operation record + * @cryp: pointer to Cryptographic device * @queue: crypto request queue * @areq: pointer to async request * @task: the tasklet is use in AES interrupt @@ -143,6 +144,7 @@ typedef int (*mtk_aes_fn)(struct mtk_cryp *cryp, struct mtk_aes_rec *aes); * Structure used to record AES execution state. */ struct mtk_aes_rec { + struct mtk_cryp *cryp; struct crypto_queue queue; struct crypto_async_request *areq; struct tasklet_struct task; @@ -166,6 +168,7 @@ struct mtk_aes_rec { /** * struct mtk_sha_rec - SHA operation record + * @cryp: pointer to Cryptographic device * @queue: crypto request queue * @req: pointer to ahash request * @task: the tasklet is use in SHA interrupt @@ -176,6 +179,7 @@ struct mtk_aes_rec { * Structure used to record SHA execution state. */ struct mtk_sha_rec { + struct mtk_cryp *cryp; struct crypto_queue queue; struct ahash_request *req; struct tasklet_struct task; diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index 55e3805fba07..1fb98f6a4ca3 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -1216,60 +1216,31 @@ static struct ahash_alg algs_sha384_sha512[] = { }, }; -static void mtk_sha_task0(unsigned long data) +static void mtk_sha_done_task(unsigned long data) { - struct mtk_cryp *cryp = (struct mtk_cryp *)data; - struct mtk_sha_rec *sha = cryp->sha[0]; + struct mtk_sha_rec *sha = (struct mtk_sha_rec *)data; + struct mtk_cryp *cryp = sha->cryp; mtk_sha_unmap(cryp, sha); mtk_sha_complete(cryp, sha); } -static void mtk_sha_task1(unsigned long data) +static irqreturn_t mtk_sha_irq(int irq, void *dev_id) { - struct mtk_cryp *cryp = (struct mtk_cryp *)data; - struct mtk_sha_rec *sha = cryp->sha[1]; + struct mtk_sha_rec *sha = (struct mtk_sha_rec *)dev_id; + struct mtk_cryp *cryp = sha->cryp; + u32 val = mtk_sha_read(cryp, RDR_STAT(sha->id)); - mtk_sha_unmap(cryp, sha); - mtk_sha_complete(cryp, sha); -} - -static irqreturn_t mtk_sha_ring2_irq(int irq, void *dev_id) -{ - struct mtk_cryp *cryp = (struct mtk_cryp *)dev_id; - struct mtk_sha_rec *sha = cryp->sha[0]; - u32 val = mtk_sha_read(cryp, RDR_STAT(RING2)); - - mtk_sha_write(cryp, RDR_STAT(RING2), val); + mtk_sha_write(cryp, RDR_STAT(sha->id), val); if (likely((SHA_FLAGS_BUSY & sha->flags))) { - mtk_sha_write(cryp, RDR_PROC_COUNT(RING2), MTK_CNT_RST); - mtk_sha_write(cryp, RDR_THRESH(RING2), + mtk_sha_write(cryp, RDR_PROC_COUNT(sha->id), MTK_CNT_RST); + mtk_sha_write(cryp, RDR_THRESH(sha->id), MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); tasklet_schedule(&sha->task); } else { - dev_warn(cryp->dev, "AES interrupt when no active requests.\n"); - } - return IRQ_HANDLED; -} - -static irqreturn_t mtk_sha_ring3_irq(int irq, void *dev_id) -{ - struct mtk_cryp *cryp = (struct mtk_cryp *)dev_id; - struct mtk_sha_rec *sha = cryp->sha[1]; - u32 val = mtk_sha_read(cryp, RDR_STAT(RING3)); - - mtk_sha_write(cryp, RDR_STAT(RING3), val); - - if (likely((SHA_FLAGS_BUSY & sha->flags))) { - mtk_sha_write(cryp, RDR_PROC_COUNT(RING3), MTK_CNT_RST); - mtk_sha_write(cryp, RDR_THRESH(RING3), - MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); - - tasklet_schedule(&sha->task); - } else { - dev_warn(cryp->dev, "AES interrupt when no active requests.\n"); + dev_warn(cryp->dev, "SHA interrupt when no active requests.\n"); } return IRQ_HANDLED; } @@ -1288,14 +1259,18 @@ static int mtk_sha_record_init(struct mtk_cryp *cryp) if (!sha[i]) goto err_cleanup; - sha[i]->id = i + RING2; + sha[i]->cryp = cryp; spin_lock_init(&sha[i]->lock); crypto_init_queue(&sha[i]->queue, SHA_QUEUE_SIZE); + + tasklet_init(&sha[i]->task, mtk_sha_done_task, + (unsigned long)sha[i]); } - tasklet_init(&sha[0]->task, mtk_sha_task0, (unsigned long)cryp); - tasklet_init(&sha[1]->task, mtk_sha_task1, (unsigned long)cryp); + /* Link to ring2 and ring3 respectively */ + sha[0]->id = RING2; + sha[1]->id = RING3; cryp->rec = 1; @@ -1368,19 +1343,15 @@ int mtk_hash_alg_register(struct mtk_cryp *cryp) if (err) goto err_record; - /* Ring2 is use by SHA record0 */ - err = devm_request_irq(cryp->dev, cryp->irq[RING2], - mtk_sha_ring2_irq, IRQF_TRIGGER_LOW, - "mtk-sha", cryp); + err = devm_request_irq(cryp->dev, cryp->irq[RING2], mtk_sha_irq, + 0, "mtk-sha", cryp->sha[0]); if (err) { dev_err(cryp->dev, "unable to request sha irq0.\n"); goto err_res; } - /* Ring3 is use by SHA record1 */ - err = devm_request_irq(cryp->dev, cryp->irq[RING3], - mtk_sha_ring3_irq, IRQF_TRIGGER_LOW, - "mtk-sha", cryp); + err = devm_request_irq(cryp->dev, cryp->irq[RING3], mtk_sha_irq, + 0, "mtk-sha", cryp->sha[1]); if (err) { dev_err(cryp->dev, "unable to request sha irq1.\n"); goto err_res; From b7a2be388b5574428170e1a22f800f1d71cbfd64 Mon Sep 17 00:00:00 2001 From: Ryder Lee Date: Thu, 9 Mar 2017 10:11:13 +0800 Subject: [PATCH 021/101] crypto: mediatek - add MTK_* prefix and correct annotations. Dummy patch to add MTK_* prefix to ring enum and fix incorrect annotations. Signed-off-by: Ryder Lee Signed-off-by: Herbert Xu --- drivers/crypto/mediatek/mtk-aes.c | 12 ++++++------ drivers/crypto/mediatek/mtk-platform.c | 12 ++++++------ drivers/crypto/mediatek/mtk-platform.h | 26 +++++++++++++------------- drivers/crypto/mediatek/mtk-sha.c | 14 +++++++------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c index e67881f7430a..b57b68f2ac93 100644 --- a/drivers/crypto/mediatek/mtk-aes.c +++ b/drivers/crypto/mediatek/mtk-aes.c @@ -1152,8 +1152,8 @@ static int mtk_aes_record_init(struct mtk_cryp *cryp) } /* Link to ring0 and ring1 respectively */ - aes[0]->id = RING0; - aes[1]->id = RING1; + aes[0]->id = MTK_RING0; + aes[1]->id = MTK_RING1; return 0; @@ -1221,14 +1221,14 @@ int mtk_cipher_alg_register(struct mtk_cryp *cryp) if (ret) goto err_record; - ret = devm_request_irq(cryp->dev, cryp->irq[RING0], mtk_aes_irq, + ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING0], mtk_aes_irq, 0, "mtk-aes", cryp->aes[0]); if (ret) { dev_err(cryp->dev, "unable to request AES irq.\n"); goto err_res; } - ret = devm_request_irq(cryp->dev, cryp->irq[RING1], mtk_aes_irq, + ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING1], mtk_aes_irq, 0, "mtk-aes", cryp->aes[1]); if (ret) { dev_err(cryp->dev, "unable to request AES irq.\n"); @@ -1236,8 +1236,8 @@ int mtk_cipher_alg_register(struct mtk_cryp *cryp) } /* Enable ring0 and ring1 interrupt */ - mtk_aes_write(cryp, AIC_ENABLE_SET(RING0), MTK_IRQ_RDR0); - mtk_aes_write(cryp, AIC_ENABLE_SET(RING1), MTK_IRQ_RDR1); + mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING0), MTK_IRQ_RDR0); + mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING1), MTK_IRQ_RDR1); spin_lock(&mtk_aes.lock); list_add_tail(&cryp->aes_list, &mtk_aes.dev_list); diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c index a9c713d4c733..50de335ed332 100644 --- a/drivers/crypto/mediatek/mtk-platform.c +++ b/drivers/crypto/mediatek/mtk-platform.c @@ -334,7 +334,7 @@ static int mtk_packet_engine_setup(struct mtk_cryp *cryp) /* Enable the 4 rings for the packet engines. */ mtk_desc_ring_link(cryp, 0xf); - for (i = 0; i < RING_MAX; i++) { + for (i = 0; i < MTK_RING_MAX; i++) { mtk_cmd_desc_ring_setup(cryp, i, &cap); mtk_res_desc_ring_setup(cryp, i, &cap); } @@ -359,7 +359,7 @@ static int mtk_aic_cap_check(struct mtk_cryp *cryp, int hw) { u32 val; - if (hw == RING_MAX) + if (hw == MTK_RING_MAX) val = readl(cryp->base + AIC_G_VERSION); else val = readl(cryp->base + AIC_VERSION(hw)); @@ -368,7 +368,7 @@ static int mtk_aic_cap_check(struct mtk_cryp *cryp, int hw) if (val != MTK_AIC_VER11 && val != MTK_AIC_VER12) return -ENXIO; - if (hw == RING_MAX) + if (hw == MTK_RING_MAX) val = readl(cryp->base + AIC_G_OPTIONS); else val = readl(cryp->base + AIC_OPTIONS(hw)); @@ -389,7 +389,7 @@ static int mtk_aic_init(struct mtk_cryp *cryp, int hw) return err; /* Disable all interrupts and set initial configuration */ - if (hw == RING_MAX) { + if (hw == MTK_RING_MAX) { writel(0, cryp->base + AIC_G_ENABLE_CTRL); writel(0, cryp->base + AIC_G_POL_CTRL); writel(0, cryp->base + AIC_G_TYPE_CTRL); @@ -431,7 +431,7 @@ static void mtk_desc_dma_free(struct mtk_cryp *cryp) { int i; - for (i = 0; i < RING_MAX; i++) { + for (i = 0; i < MTK_RING_MAX; i++) { dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ, cryp->ring[i]->res_base, cryp->ring[i]->res_dma); @@ -447,7 +447,7 @@ static int mtk_desc_ring_alloc(struct mtk_cryp *cryp) struct mtk_ring **ring = cryp->ring; int i, err = ENOMEM; - for (i = 0; i < RING_MAX; i++) { + for (i = 0; i < MTK_RING_MAX; i++) { ring[i] = kzalloc(sizeof(**ring), GFP_KERNEL); if (!ring[i]) goto err_cleanup; diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h index d2a1db6bddc6..3bbe9b0072cc 100644 --- a/drivers/crypto/mediatek/mtk-platform.h +++ b/drivers/crypto/mediatek/mtk-platform.h @@ -38,14 +38,14 @@ * Ring 2/3 are used by SHA. */ enum { - RING0 = 0, - RING1, - RING2, - RING3, - RING_MAX, + MTK_RING0, + MTK_RING1, + MTK_RING2, + MTK_RING3, + MTK_RING_MAX }; -#define MTK_REC_NUM (RING_MAX / 2) +#define MTK_REC_NUM (MTK_RING_MAX / 2) #define MTK_IRQ_NUM 5 /** @@ -137,7 +137,7 @@ typedef int (*mtk_aes_fn)(struct mtk_cryp *cryp, struct mtk_aes_rec *aes); * @resume: pointer to resume function * @total: request buffer length * @buf: pointer to page buffer - * @id: record identification + * @id: the current use of ring * @flags: it's describing AES operation state * @lock: the async queue lock * @@ -172,9 +172,9 @@ struct mtk_aes_rec { * @queue: crypto request queue * @req: pointer to ahash request * @task: the tasklet is use in SHA interrupt - * @id: record identification + * @id: the current use of ring * @flags: it's describing SHA operation state - * @lock: the ablkcipher queue lock + * @lock: the async queue lock * * Structure used to record SHA execution state. */ @@ -197,9 +197,9 @@ struct mtk_sha_rec { * @clk_ethif: pointer to ethif clock * @clk_cryp: pointer to crypto clock * @irq: global system and rings IRQ - * @ring: pointer to execution state of AES - * @aes: pointer to execution state of SHA - * @sha: each execution record map to a ring + * @ring: pointer to descriptor rings + * @aes: pointer to operation record of AES + * @sha: pointer to operation record of SHA * @aes_list: device list of AES * @sha_list: device list of SHA * @tmp: pointer to temporary buffer for internal use @@ -215,7 +215,7 @@ struct mtk_cryp { struct clk *clk_cryp; int irq[MTK_IRQ_NUM]; - struct mtk_ring *ring[RING_MAX]; + struct mtk_ring *ring[MTK_RING_MAX]; struct mtk_aes_rec *aes[MTK_REC_NUM]; struct mtk_sha_rec *sha[MTK_REC_NUM]; diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index 1fb98f6a4ca3..b4f0c62a9152 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -694,7 +694,7 @@ static void mtk_sha_finish_req(struct mtk_cryp *cryp, sha->req->base.complete(&sha->req->base, err); /* Handle new request */ - mtk_sha_handle_queue(cryp, sha->id - RING2, NULL); + mtk_sha_handle_queue(cryp, sha->id - MTK_RING2, NULL); } static int mtk_sha_handle_queue(struct mtk_cryp *cryp, u8 id, @@ -1269,8 +1269,8 @@ static int mtk_sha_record_init(struct mtk_cryp *cryp) } /* Link to ring2 and ring3 respectively */ - sha[0]->id = RING2; - sha[1]->id = RING3; + sha[0]->id = MTK_RING2; + sha[1]->id = MTK_RING3; cryp->rec = 1; @@ -1343,14 +1343,14 @@ int mtk_hash_alg_register(struct mtk_cryp *cryp) if (err) goto err_record; - err = devm_request_irq(cryp->dev, cryp->irq[RING2], mtk_sha_irq, + err = devm_request_irq(cryp->dev, cryp->irq[MTK_RING2], mtk_sha_irq, 0, "mtk-sha", cryp->sha[0]); if (err) { dev_err(cryp->dev, "unable to request sha irq0.\n"); goto err_res; } - err = devm_request_irq(cryp->dev, cryp->irq[RING3], mtk_sha_irq, + err = devm_request_irq(cryp->dev, cryp->irq[MTK_RING3], mtk_sha_irq, 0, "mtk-sha", cryp->sha[1]); if (err) { dev_err(cryp->dev, "unable to request sha irq1.\n"); @@ -1358,8 +1358,8 @@ int mtk_hash_alg_register(struct mtk_cryp *cryp) } /* Enable ring2 and ring3 interrupt for hash */ - mtk_sha_write(cryp, AIC_ENABLE_SET(RING2), MTK_IRQ_RDR2); - mtk_sha_write(cryp, AIC_ENABLE_SET(RING3), MTK_IRQ_RDR3); + mtk_sha_write(cryp, AIC_ENABLE_SET(MTK_RING2), MTK_IRQ_RDR2); + mtk_sha_write(cryp, AIC_ENABLE_SET(MTK_RING3), MTK_IRQ_RDR3); cryp->tmp = dma_alloc_coherent(cryp->dev, SHA_TMP_BUF_SIZE, &cryp->tmp_dma, GFP_KERNEL); From 82445fe995f4b1edf131157a7328796d5aec1efb Mon Sep 17 00:00:00 2001 From: Ryder Lee Date: Thu, 9 Mar 2017 10:11:14 +0800 Subject: [PATCH 022/101] crypto: mediatek - make mtk_sha_xmit() more generic This is a transitional patch. It merges mtk_sha_xmit() and mtk_sha_xmit2() to make transmit function more generic. In addition, res->buf and cryp->tmp_dma in mtk_sha_xmit() are useless, since crypto engine writes the result digests into ctx->tfm.digest instead of res->buf. It's better to remove it. Signed-off-by: Ryder Lee Signed-off-by: Herbert Xu --- drivers/crypto/mediatek/mtk-platform.h | 4 - drivers/crypto/mediatek/mtk-sha.c | 116 ++++++++----------------- 2 files changed, 34 insertions(+), 86 deletions(-) diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h index 3bbe9b0072cc..78ce54e136c5 100644 --- a/drivers/crypto/mediatek/mtk-platform.h +++ b/drivers/crypto/mediatek/mtk-platform.h @@ -202,8 +202,6 @@ struct mtk_sha_rec { * @sha: pointer to operation record of SHA * @aes_list: device list of AES * @sha_list: device list of SHA - * @tmp: pointer to temporary buffer for internal use - * @tmp_dma: DMA address of temporary buffer * @rec: it's used to select SHA record for tfm * * Structure storing cryptographic device information. @@ -222,8 +220,6 @@ struct mtk_cryp { struct list_head aes_list; struct list_head sha_list; - void *tmp; - dma_addr_t tmp_dma; bool rec; }; diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index b4f0c62a9152..ef6fb201f70d 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -17,7 +17,6 @@ #define SHA_ALIGN_MSK (sizeof(u32) - 1) #define SHA_QUEUE_SIZE 512 -#define SHA_TMP_BUF_SIZE 512 #define SHA_BUF_SIZE ((u32)PAGE_SIZE) #define SHA_OP_UPDATE 1 @@ -319,7 +318,7 @@ static void mtk_sha_info_init(struct mtk_sha_reqctx *ctx) */ static int mtk_sha_info_update(struct mtk_cryp *cryp, struct mtk_sha_rec *sha, - size_t len) + size_t len1, size_t len2) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); struct mtk_sha_info *info = &ctx->info; @@ -331,11 +330,11 @@ static int mtk_sha_info_update(struct mtk_cryp *cryp, ct->ctrl[0] &= ~SHA_TFM_START; ctx->ct_hdr &= ~SHA_DATA_LEN_MSK; - ctx->ct_hdr |= cpu_to_le32(len); + ctx->ct_hdr |= cpu_to_le32(len1 + len2); ct->cmd[0] &= ~SHA_DATA_LEN_MSK; - ct->cmd[0] |= cpu_to_le32(len); + ct->cmd[0] |= cpu_to_le32(len1 + len2); - ctx->digcnt += len; + ctx->digcnt += len1; ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info), DMA_BIDIRECTIONAL); @@ -422,26 +421,24 @@ static int mtk_sha_init(struct ahash_request *req) } static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha, - dma_addr_t addr, size_t len) + dma_addr_t addr1, size_t len1, + dma_addr_t addr2, size_t len2) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); struct mtk_ring *ring = cryp->ring[sha->id]; struct mtk_desc *cmd = ring->cmd_base + ring->cmd_pos; struct mtk_desc *res = ring->res_base + ring->res_pos; - int err; + int err, count = 0; - err = mtk_sha_info_update(cryp, sha, len); + err = mtk_sha_info_update(cryp, sha, len1, len2); if (err) return err; /* Fill in the command/result descriptors */ - res->hdr = MTK_DESC_FIRST | MTK_DESC_LAST | MTK_DESC_BUF_LEN(len); - res->buf = cpu_to_le32(cryp->tmp_dma); - - cmd->hdr = MTK_DESC_FIRST | MTK_DESC_LAST | MTK_DESC_BUF_LEN(len) | + res->hdr = MTK_DESC_FIRST | MTK_DESC_BUF_LEN(len1); + cmd->hdr = MTK_DESC_FIRST | MTK_DESC_BUF_LEN(len1) | MTK_DESC_CT_LEN(ctx->ct_size); - - cmd->buf = cpu_to_le32(addr); + cmd->buf = cpu_to_le32(addr1); cmd->ct = cpu_to_le32(ctx->ct_dma); cmd->ct_hdr = ctx->ct_hdr; cmd->tfm = cpu_to_le32(ctx->tfm_dma); @@ -450,61 +447,25 @@ static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha, ring->cmd_pos = 0; ring->res_pos = ring->cmd_pos; - /* - * Make sure that all changes to the DMA ring are done before we - * start engine. - */ - wmb(); - /* Start DMA transfer */ - mtk_sha_write(cryp, RDR_PREP_COUNT(sha->id), MTK_DESC_CNT(1)); - mtk_sha_write(cryp, CDR_PREP_COUNT(sha->id), MTK_DESC_CNT(1)); + count++; - return -EINPROGRESS; -} + if (len2) { + cmd = ring->cmd_base + ring->cmd_pos; + res = ring->res_base + ring->res_pos; -static int mtk_sha_xmit2(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha, - struct mtk_sha_reqctx *ctx, - size_t len1, size_t len2) -{ - struct mtk_ring *ring = cryp->ring[sha->id]; - struct mtk_desc *cmd = ring->cmd_base + ring->cmd_pos; - struct mtk_desc *res = ring->res_base + ring->res_pos; - int err; + res->hdr = MTK_DESC_BUF_LEN(len2); + cmd->hdr = MTK_DESC_BUF_LEN(len2); + cmd->buf = cpu_to_le32(addr2); - err = mtk_sha_info_update(cryp, sha, len1 + len2); - if (err) - return err; + if (++ring->cmd_pos == MTK_DESC_NUM) + ring->cmd_pos = 0; - /* Fill in the command/result descriptors */ - res->hdr = MTK_DESC_BUF_LEN(len1) | MTK_DESC_FIRST; - res->buf = cpu_to_le32(cryp->tmp_dma); + ring->res_pos = ring->cmd_pos; + count++; + } - cmd->hdr = MTK_DESC_BUF_LEN(len1) | MTK_DESC_FIRST | - MTK_DESC_CT_LEN(ctx->ct_size); - cmd->buf = cpu_to_le32(sg_dma_address(ctx->sg)); - cmd->ct = cpu_to_le32(ctx->ct_dma); - cmd->ct_hdr = ctx->ct_hdr; - cmd->tfm = cpu_to_le32(ctx->tfm_dma); - - if (++ring->cmd_pos == MTK_DESC_NUM) - ring->cmd_pos = 0; - - ring->res_pos = ring->cmd_pos; - - cmd = ring->cmd_base + ring->cmd_pos; - res = ring->res_base + ring->res_pos; - - res->hdr = MTK_DESC_BUF_LEN(len2) | MTK_DESC_LAST; - res->buf = cpu_to_le32(cryp->tmp_dma); - - cmd->hdr = MTK_DESC_BUF_LEN(len2) | MTK_DESC_LAST; - cmd->buf = cpu_to_le32(ctx->dma_addr); - - if (++ring->cmd_pos == MTK_DESC_NUM) - ring->cmd_pos = 0; - - ring->res_pos = ring->cmd_pos; + cmd->hdr |= MTK_DESC_LAST; + res->hdr |= MTK_DESC_LAST; /* * Make sure that all changes to the DMA ring are done before we @@ -512,8 +473,8 @@ static int mtk_sha_xmit2(struct mtk_cryp *cryp, */ wmb(); /* Start DMA transfer */ - mtk_sha_write(cryp, RDR_PREP_COUNT(sha->id), MTK_DESC_CNT(2)); - mtk_sha_write(cryp, CDR_PREP_COUNT(sha->id), MTK_DESC_CNT(2)); + mtk_sha_write(cryp, RDR_PREP_COUNT(sha->id), MTK_DESC_CNT(count)); + mtk_sha_write(cryp, CDR_PREP_COUNT(sha->id), MTK_DESC_CNT(count)); return -EINPROGRESS; } @@ -532,7 +493,7 @@ static int mtk_sha_dma_map(struct mtk_cryp *cryp, ctx->flags &= ~SHA_FLAGS_SG; - return mtk_sha_xmit(cryp, sha, ctx->dma_addr, count); + return mtk_sha_xmit(cryp, sha, ctx->dma_addr, count, 0, 0); } static int mtk_sha_update_slow(struct mtk_cryp *cryp, @@ -625,7 +586,8 @@ static int mtk_sha_update_start(struct mtk_cryp *cryp, if (len == 0) { ctx->flags &= ~SHA_FLAGS_SG; - return mtk_sha_xmit(cryp, sha, ctx->dma_addr, count); + return mtk_sha_xmit(cryp, sha, ctx->dma_addr, + count, 0, 0); } else { ctx->sg = sg; @@ -635,7 +597,8 @@ static int mtk_sha_update_start(struct mtk_cryp *cryp, } ctx->flags |= SHA_FLAGS_SG; - return mtk_sha_xmit2(cryp, sha, ctx, len, count); + return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg), + len, ctx->dma_addr, count); } } @@ -646,7 +609,8 @@ static int mtk_sha_update_start(struct mtk_cryp *cryp, ctx->flags |= SHA_FLAGS_SG; - return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg), len); + return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg), + len, 0, 0); } static int mtk_sha_final_req(struct mtk_cryp *cryp, @@ -1361,14 +1325,6 @@ int mtk_hash_alg_register(struct mtk_cryp *cryp) mtk_sha_write(cryp, AIC_ENABLE_SET(MTK_RING2), MTK_IRQ_RDR2); mtk_sha_write(cryp, AIC_ENABLE_SET(MTK_RING3), MTK_IRQ_RDR3); - cryp->tmp = dma_alloc_coherent(cryp->dev, SHA_TMP_BUF_SIZE, - &cryp->tmp_dma, GFP_KERNEL); - if (!cryp->tmp) { - dev_err(cryp->dev, "unable to allocate tmp buffer.\n"); - err = -EINVAL; - goto err_res; - } - spin_lock(&mtk_sha.lock); list_add_tail(&cryp->sha_list, &mtk_sha.dev_list); spin_unlock(&mtk_sha.lock); @@ -1383,8 +1339,6 @@ err_algs: spin_lock(&mtk_sha.lock); list_del(&cryp->sha_list); spin_unlock(&mtk_sha.lock); - dma_free_coherent(cryp->dev, SHA_TMP_BUF_SIZE, - cryp->tmp, cryp->tmp_dma); err_res: mtk_sha_record_free(cryp); err_record: @@ -1400,7 +1354,5 @@ void mtk_hash_alg_release(struct mtk_cryp *cryp) spin_unlock(&mtk_sha.lock); mtk_sha_unregister_algs(); - dma_free_coherent(cryp->dev, SHA_TMP_BUF_SIZE, - cryp->tmp, cryp->tmp_dma); mtk_sha_record_free(cryp); } From 3d21c41f7e9b51c70d37b316f11cf5cce9dd3311 Mon Sep 17 00:00:00 2001 From: Ryder Lee Date: Thu, 9 Mar 2017 10:11:15 +0800 Subject: [PATCH 023/101] crypto: mediatek - simplify descriptor ring management This patch replaces cmd_pos/res_pos with pointer cmd_next/res_next. In old code, we must to add one to shift ring to the next segment, and then use this value to caculate current offset from ring base for each DMA operation. Now these pointers helps us to simplify flow, so we just need to move pointers and check the boundaries of ring. Signed-off-by: Ryder Lee Signed-off-by: Herbert Xu --- drivers/crypto/mediatek/mtk-aes.c | 14 ++++++----- drivers/crypto/mediatek/mtk-platform.c | 3 +++ drivers/crypto/mediatek/mtk-platform.h | 8 +++--- drivers/crypto/mediatek/mtk-sha.c | 35 ++++++++++++++------------ 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c index b57b68f2ac93..21f3e594804b 100644 --- a/drivers/crypto/mediatek/mtk-aes.c +++ b/drivers/crypto/mediatek/mtk-aes.c @@ -262,7 +262,7 @@ static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) /* Write command descriptors */ for (nents = 0; nents < slen; ++nents, ssg = sg_next(ssg)) { - cmd = ring->cmd_base + ring->cmd_pos; + cmd = ring->cmd_next; cmd->hdr = MTK_DESC_BUF_LEN(ssg->length); cmd->buf = cpu_to_le32(sg_dma_address(ssg)); @@ -274,22 +274,24 @@ static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) cmd->tfm = cpu_to_le32(aes->ctx->tfm_dma); } - if (++ring->cmd_pos == MTK_DESC_NUM) - ring->cmd_pos = 0; + /* Shift ring buffer and check boundary */ + if (++ring->cmd_next == ring->cmd_base + MTK_DESC_NUM) + ring->cmd_next = ring->cmd_base; } cmd->hdr |= MTK_DESC_LAST; /* Prepare result descriptors */ for (nents = 0; nents < dlen; ++nents, dsg = sg_next(dsg)) { - res = ring->res_base + ring->res_pos; + res = ring->res_next; res->hdr = MTK_DESC_BUF_LEN(dsg->length); res->buf = cpu_to_le32(sg_dma_address(dsg)); if (nents == 0) res->hdr |= MTK_DESC_FIRST; - if (++ring->res_pos == MTK_DESC_NUM) - ring->res_pos = 0; + /* Shift ring buffer and check boundary */ + if (++ring->res_next == ring->res_base + MTK_DESC_NUM) + ring->res_next = ring->res_base; } res->hdr |= MTK_DESC_LAST; diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c index 50de335ed332..b6ecc288b540 100644 --- a/drivers/crypto/mediatek/mtk-platform.c +++ b/drivers/crypto/mediatek/mtk-platform.c @@ -465,6 +465,9 @@ static int mtk_desc_ring_alloc(struct mtk_cryp *cryp) GFP_KERNEL); if (!ring[i]->res_base) goto err_cleanup; + + ring[i]->cmd_next = ring[i]->cmd_base; + ring[i]->res_next = ring[i]->res_base; } return 0; diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h index 78ce54e136c5..218e30d1828a 100644 --- a/drivers/crypto/mediatek/mtk-platform.h +++ b/drivers/crypto/mediatek/mtk-platform.h @@ -84,11 +84,11 @@ struct mtk_desc { /** * struct mtk_ring - Descriptor ring * @cmd_base: pointer to command descriptor ring base + * @cmd_next: pointer to the next command descriptor * @cmd_dma: DMA address of command descriptor ring - * @cmd_pos: current position in the command descriptor ring * @res_base: pointer to result descriptor ring base + * @res_next: pointer to the next result descriptor * @res_dma: DMA address of result descriptor ring - * @res_pos: current position in the result descriptor ring * * A descriptor ring is a circular buffer that is used to manage * one or more descriptors. There are two type of descriptor rings; @@ -96,11 +96,11 @@ struct mtk_desc { */ struct mtk_ring { struct mtk_desc *cmd_base; + struct mtk_desc *cmd_next; dma_addr_t cmd_dma; - u32 cmd_pos; struct mtk_desc *res_base; + struct mtk_desc *res_next; dma_addr_t res_dma; - u32 res_pos; }; /** diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index ef6fb201f70d..0884d6243cbc 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -152,6 +152,21 @@ static inline void mtk_sha_write(struct mtk_cryp *cryp, writel_relaxed(value, cryp->base + offset); } +static inline void mtk_sha_ring_shift(struct mtk_ring *ring, + struct mtk_desc **cmd_curr, + struct mtk_desc **res_curr, + int *count) +{ + *cmd_curr = ring->cmd_next++; + *res_curr = ring->res_next++; + (*count)++; + + if (ring->cmd_next == ring->cmd_base + MTK_DESC_NUM) { + ring->cmd_next = ring->cmd_base; + ring->res_next = ring->res_base; + } +} + static struct mtk_cryp *mtk_sha_find_dev(struct mtk_sha_ctx *tctx) { struct mtk_cryp *cryp = NULL; @@ -426,8 +441,7 @@ static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha, { struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); struct mtk_ring *ring = cryp->ring[sha->id]; - struct mtk_desc *cmd = ring->cmd_base + ring->cmd_pos; - struct mtk_desc *res = ring->res_base + ring->res_pos; + struct mtk_desc *cmd, *res; int err, count = 0; err = mtk_sha_info_update(cryp, sha, len1, len2); @@ -435,6 +449,8 @@ static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha, return err; /* Fill in the command/result descriptors */ + mtk_sha_ring_shift(ring, &cmd, &res, &count); + res->hdr = MTK_DESC_FIRST | MTK_DESC_BUF_LEN(len1); cmd->hdr = MTK_DESC_FIRST | MTK_DESC_BUF_LEN(len1) | MTK_DESC_CT_LEN(ctx->ct_size); @@ -443,25 +459,12 @@ static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha, cmd->ct_hdr = ctx->ct_hdr; cmd->tfm = cpu_to_le32(ctx->tfm_dma); - if (++ring->cmd_pos == MTK_DESC_NUM) - ring->cmd_pos = 0; - - ring->res_pos = ring->cmd_pos; - count++; - if (len2) { - cmd = ring->cmd_base + ring->cmd_pos; - res = ring->res_base + ring->res_pos; + mtk_sha_ring_shift(ring, &cmd, &res, &count); res->hdr = MTK_DESC_BUF_LEN(len2); cmd->hdr = MTK_DESC_BUF_LEN(len2); cmd->buf = cpu_to_le32(addr2); - - if (++ring->cmd_pos == MTK_DESC_NUM) - ring->cmd_pos = 0; - - ring->res_pos = ring->cmd_pos; - count++; } cmd->hdr |= MTK_DESC_LAST; From 0d4a826611676198aea8ae531b84dd980a4b5f61 Mon Sep 17 00:00:00 2001 From: Ryder Lee Date: Thu, 9 Mar 2017 10:11:16 +0800 Subject: [PATCH 024/101] crypto: mediatek - add queue_task tasklet This patch adds 'queue_task' to dequeue crypto requset. This will help to avoid directly calling mtk_aes_handle_queue() / mtk_sha_handle_queue() from done tasklet or error handler. In order to avoid confusion, the new code properly renames DMA completion "task" to "done_task". Signed-off-by: Ryder Lee Signed-off-by: Herbert Xu --- drivers/crypto/mediatek/mtk-aes.c | 20 ++++++++++++++++---- drivers/crypto/mediatek/mtk-platform.h | 12 ++++++++---- drivers/crypto/mediatek/mtk-sha.c | 19 +++++++++++++++---- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c index 21f3e594804b..140e9a3ba177 100644 --- a/drivers/crypto/mediatek/mtk-aes.c +++ b/drivers/crypto/mediatek/mtk-aes.c @@ -533,7 +533,8 @@ static int mtk_aes_complete(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) aes->areq->complete(aes->areq, 0); /* Handle new request */ - return mtk_aes_handle_queue(cryp, aes->id, NULL); + tasklet_schedule(&aes->queue_task); + return 0; } static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) @@ -1094,6 +1095,13 @@ static struct aead_alg aes_gcm_alg = { }, }; +static void mtk_aes_queue_task(unsigned long data) +{ + struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data; + + mtk_aes_handle_queue(aes->cryp, aes->id, NULL); +} + static void mtk_aes_done_task(unsigned long data) { struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data; @@ -1116,7 +1124,7 @@ static irqreturn_t mtk_aes_irq(int irq, void *dev_id) mtk_aes_write(cryp, RDR_THRESH(aes->id), MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); - tasklet_schedule(&aes->task); + tasklet_schedule(&aes->done_task); } else { dev_warn(cryp->dev, "AES interrupt when no active requests.\n"); } @@ -1149,7 +1157,9 @@ static int mtk_aes_record_init(struct mtk_cryp *cryp) spin_lock_init(&aes[i]->lock); crypto_init_queue(&aes[i]->queue, AES_QUEUE_SIZE); - tasklet_init(&aes[i]->task, mtk_aes_done_task, + tasklet_init(&aes[i]->queue_task, mtk_aes_queue_task, + (unsigned long)aes[i]); + tasklet_init(&aes[i]->done_task, mtk_aes_done_task, (unsigned long)aes[i]); } @@ -1173,7 +1183,9 @@ static void mtk_aes_record_free(struct mtk_cryp *cryp) int i; for (i = 0; i < MTK_REC_NUM; i++) { - tasklet_kill(&cryp->aes[i]->task); + tasklet_kill(&cryp->aes[i]->done_task); + tasklet_kill(&cryp->aes[i]->queue_task); + free_page((unsigned long)cryp->aes[i]->buf); kfree(cryp->aes[i]); } diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h index 218e30d1828a..cc98c2cdb339 100644 --- a/drivers/crypto/mediatek/mtk-platform.h +++ b/drivers/crypto/mediatek/mtk-platform.h @@ -128,7 +128,8 @@ typedef int (*mtk_aes_fn)(struct mtk_cryp *cryp, struct mtk_aes_rec *aes); * @cryp: pointer to Cryptographic device * @queue: crypto request queue * @areq: pointer to async request - * @task: the tasklet is use in AES interrupt + * @done_task: the tasklet is use in AES interrupt + * @queue_task: the tasklet is used to dequeue request * @ctx: pointer to current context * @src: the structure that holds source sg list info * @dst: the structure that holds destination sg list info @@ -147,7 +148,8 @@ struct mtk_aes_rec { struct mtk_cryp *cryp; struct crypto_queue queue; struct crypto_async_request *areq; - struct tasklet_struct task; + struct tasklet_struct done_task; + struct tasklet_struct queue_task; struct mtk_aes_base_ctx *ctx; struct mtk_aes_dma src; struct mtk_aes_dma dst; @@ -171,7 +173,8 @@ struct mtk_aes_rec { * @cryp: pointer to Cryptographic device * @queue: crypto request queue * @req: pointer to ahash request - * @task: the tasklet is use in SHA interrupt + * @done_task: the tasklet is use in SHA interrupt + * @queue_task: the tasklet is used to dequeue request * @id: the current use of ring * @flags: it's describing SHA operation state * @lock: the async queue lock @@ -182,7 +185,8 @@ struct mtk_sha_rec { struct mtk_cryp *cryp; struct crypto_queue queue; struct ahash_request *req; - struct tasklet_struct task; + struct tasklet_struct done_task; + struct tasklet_struct queue_task; u8 id; unsigned long flags; diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index 0884d6243cbc..dd3582b6a1a5 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -661,7 +661,7 @@ static void mtk_sha_finish_req(struct mtk_cryp *cryp, sha->req->base.complete(&sha->req->base, err); /* Handle new request */ - mtk_sha_handle_queue(cryp, sha->id - MTK_RING2, NULL); + tasklet_schedule(&sha->queue_task); } static int mtk_sha_handle_queue(struct mtk_cryp *cryp, u8 id, @@ -1183,6 +1183,13 @@ static struct ahash_alg algs_sha384_sha512[] = { }, }; +static void mtk_sha_queue_task(unsigned long data) +{ + struct mtk_sha_rec *sha = (struct mtk_sha_rec *)data; + + mtk_sha_handle_queue(sha->cryp, sha->id - MTK_RING2, NULL); +} + static void mtk_sha_done_task(unsigned long data) { struct mtk_sha_rec *sha = (struct mtk_sha_rec *)data; @@ -1205,7 +1212,7 @@ static irqreturn_t mtk_sha_irq(int irq, void *dev_id) mtk_sha_write(cryp, RDR_THRESH(sha->id), MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); - tasklet_schedule(&sha->task); + tasklet_schedule(&sha->done_task); } else { dev_warn(cryp->dev, "SHA interrupt when no active requests.\n"); } @@ -1231,7 +1238,9 @@ static int mtk_sha_record_init(struct mtk_cryp *cryp) spin_lock_init(&sha[i]->lock); crypto_init_queue(&sha[i]->queue, SHA_QUEUE_SIZE); - tasklet_init(&sha[i]->task, mtk_sha_done_task, + tasklet_init(&sha[i]->queue_task, mtk_sha_queue_task, + (unsigned long)sha[i]); + tasklet_init(&sha[i]->done_task, mtk_sha_done_task, (unsigned long)sha[i]); } @@ -1254,7 +1263,9 @@ static void mtk_sha_record_free(struct mtk_cryp *cryp) int i; for (i = 0; i < MTK_REC_NUM; i++) { - tasklet_kill(&cryp->sha[i]->task); + tasklet_kill(&cryp->sha[i]->done_task); + tasklet_kill(&cryp->sha[i]->queue_task); + kfree(cryp->sha[i]); } } From 5041d714fc3e32a4cb0408312ce71d80114413b5 Mon Sep 17 00:00:00 2001 From: Ryder Lee Date: Thu, 9 Mar 2017 10:11:17 +0800 Subject: [PATCH 025/101] crypto: mediatek - fix error handling in mtk_aes_complete() This patch fixes how errors should be handled by mtk_aes_complete(). Signed-off-by: Ryder Lee Signed-off-by: Herbert Xu --- drivers/crypto/mediatek/mtk-aes.c | 39 ++++++++++++++++++------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c index 140e9a3ba177..6a0180d00de0 100644 --- a/drivers/crypto/mediatek/mtk-aes.c +++ b/drivers/crypto/mediatek/mtk-aes.c @@ -248,6 +248,17 @@ static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma) sg->length += dma->remainder; } +static inline int mtk_aes_complete(struct mtk_cryp *cryp, + struct mtk_aes_rec *aes, + int err) +{ + aes->flags &= ~AES_FLAGS_BUSY; + aes->areq->complete(aes->areq, err); + /* Handle new request */ + tasklet_schedule(&aes->queue_task); + return err; +} + /* * Write descriptors for processing. This will configure the engine, load * the transform information and then start the packet processing. @@ -352,7 +363,7 @@ static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) ctx->ct_dma = dma_map_single(cryp->dev, &ctx->ct, sizeof(ctx->ct), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma))) - return -EINVAL; + goto exit; ctx->tfm_dma = dma_map_single(cryp->dev, &ctx->tfm, sizeof(ctx->tfm), DMA_TO_DEVICE); @@ -389,8 +400,8 @@ sg_map_err: tfm_map_err: dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->ct), DMA_TO_DEVICE); - - return -EINVAL; +exit: + return mtk_aes_complete(cryp, aes, -EINVAL); } /* Initialize transform information of CBC/ECB/CTR mode */ @@ -467,7 +478,7 @@ static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, padlen = mtk_aes_padlen(len); if (len + padlen > AES_BUF_SIZE) - return -ENOMEM; + return mtk_aes_complete(cryp, aes, -ENOMEM); if (!src_aligned) { sg_copy_to_buffer(src, sg_nents(src), aes->buf, len); @@ -527,14 +538,10 @@ static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id, return ctx->start(cryp, aes); } -static int mtk_aes_complete(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) +static int mtk_aes_transfer_complete(struct mtk_cryp *cryp, + struct mtk_aes_rec *aes) { - aes->flags &= ~AES_FLAGS_BUSY; - aes->areq->complete(aes->areq, 0); - - /* Handle new request */ - tasklet_schedule(&aes->queue_task); - return 0; + return mtk_aes_complete(cryp, aes, 0); } static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) @@ -543,7 +550,7 @@ static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) struct mtk_aes_reqctx *rctx = ablkcipher_request_ctx(req); mtk_aes_set_mode(aes, rctx); - aes->resume = mtk_aes_complete; + aes->resume = mtk_aes_transfer_complete; return mtk_aes_dma(cryp, aes, req->src, req->dst, req->nbytes); } @@ -568,7 +575,7 @@ static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) /* Check for transfer completion. */ cctx->offset += aes->total; if (cctx->offset >= req->nbytes) - return mtk_aes_complete(cryp, aes); + return mtk_aes_transfer_complete(cryp, aes); /* Compute data length. */ datalen = req->nbytes - cctx->offset; @@ -602,7 +609,6 @@ static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) cctx->iv[3] = cpu_to_be32(ctr); crypto_inc((u8 *)cctx->iv, AES_BLOCK_SIZE); } - aes->resume = mtk_aes_ctr_transfer; return mtk_aes_dma(cryp, aes, src, dst, datalen); } @@ -618,6 +624,7 @@ static int mtk_aes_ctr_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) memcpy(cctx->iv, req->info, AES_BLOCK_SIZE); cctx->offset = 0; aes->total = 0; + aes->resume = mtk_aes_ctr_transfer; return mtk_aes_ctr_transfer(cryp, aes); } @@ -859,7 +866,7 @@ static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, if (!src_aligned || !dst_aligned) { if (aes->total > AES_BUF_SIZE) - return -ENOMEM; + return mtk_aes_complete(cryp, aes, -ENOMEM); if (!src_aligned) { sg_copy_to_buffer(src, sg_nents(src), aes->buf, len); @@ -905,7 +912,7 @@ static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) aes->total = len; gctx->textlen = req->cryptlen - gctx->authsize; } - aes->resume = mtk_aes_complete; + aes->resume = mtk_aes_transfer_complete; return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len); } From 98b10235f348ddb88be7afc1da167a85f88fcb56 Mon Sep 17 00:00:00 2001 From: Ryder Lee Date: Thu, 9 Mar 2017 10:11:18 +0800 Subject: [PATCH 026/101] crypto: mediatek - add mtk_aes_gcm_tag_verify() This patch adds mtk_aes_gcm_tag_verify() which is used to compare authenticated tag. Signed-off-by: Ryder Lee Signed-off-by: Herbert Xu --- drivers/crypto/mediatek/mtk-aes.c | 24 ++++++++++++++++++++++-- drivers/crypto/mediatek/mtk-platform.h | 2 ++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c index 6a0180d00de0..8f3efa581294 100644 --- a/drivers/crypto/mediatek/mtk-aes.c +++ b/drivers/crypto/mediatek/mtk-aes.c @@ -70,6 +70,8 @@ #define AES_FLAGS_ENCRYPT BIT(4) #define AES_FLAGS_BUSY BIT(5) +#define AES_AUTH_TAG_ERR cpu_to_le32(BIT(26)) + /** * Command token(CT) is a set of hardware instructions that * are used to control engine's processing flow of AES. @@ -306,6 +308,9 @@ static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) } res->hdr |= MTK_DESC_LAST; + /* Pointer to current result descriptor */ + ring->res_prev = res; + /* Prepare enough space for authenticated tag */ if (aes->flags & AES_FLAGS_GCM) res->hdr += AES_BLOCK_SIZE; @@ -799,6 +804,19 @@ mtk_aes_gcm_ctx_cast(struct mtk_aes_base_ctx *ctx) return container_of(ctx, struct mtk_aes_gcm_ctx, base); } +/* + * Engine will verify and compare tag automatically, so we just need + * to check returned status which stored in the result descriptor. + */ +static int mtk_aes_gcm_tag_verify(struct mtk_cryp *cryp, + struct mtk_aes_rec *aes) +{ + u32 status = cryp->ring[aes->id]->res_prev->ct; + + return mtk_aes_complete(cryp, aes, (status & AES_AUTH_TAG_ERR) ? + -EBADMSG : 0); +} + /* Initialize transform information of GCM mode */ static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, @@ -902,6 +920,8 @@ static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) if (aes->flags & AES_FLAGS_ENCRYPT) { u32 tag[4]; + + aes->resume = mtk_aes_transfer_complete; /* Compute total process length. */ aes->total = len + gctx->authsize; /* Compute text length. */ @@ -909,10 +929,10 @@ static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) /* Hardware will append authenticated tag to output buffer */ scatterwalk_map_and_copy(tag, req->dst, len, gctx->authsize, 1); } else { + aes->resume = mtk_aes_gcm_tag_verify; aes->total = len; gctx->textlen = req->cryptlen - gctx->authsize; } - aes->resume = mtk_aes_transfer_complete; return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len); } @@ -925,7 +945,7 @@ static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode) rctx->mode = AES_FLAGS_GCM | mode; return mtk_aes_handle_queue(ctx->cryp, !!(mode & AES_FLAGS_ENCRYPT), - &req->base); + &req->base); } static void mtk_gcm_setkey_done(struct crypto_async_request *req, int err) diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h index cc98c2cdb339..303c152dc931 100644 --- a/drivers/crypto/mediatek/mtk-platform.h +++ b/drivers/crypto/mediatek/mtk-platform.h @@ -88,6 +88,7 @@ struct mtk_desc { * @cmd_dma: DMA address of command descriptor ring * @res_base: pointer to result descriptor ring base * @res_next: pointer to the next result descriptor + * @res_prev: pointer to the previous result descriptor * @res_dma: DMA address of result descriptor ring * * A descriptor ring is a circular buffer that is used to manage @@ -100,6 +101,7 @@ struct mtk_ring { dma_addr_t cmd_dma; struct mtk_desc *res_base; struct mtk_desc *res_next; + struct mtk_desc *res_prev; dma_addr_t res_dma; }; From 9aa2fcb8cf8b1030db2ed5afaa8737e94437a27c Mon Sep 17 00:00:00 2001 From: Ryder Lee Date: Thu, 9 Mar 2017 10:11:19 +0800 Subject: [PATCH 027/101] crypto: mediatek - make hardware operation flow more efficient This patch refines data structures, which are used to control engine's data path, to make it more efficient. Hence current change are: - gathers the broken pieces of structures 'mtk_aes_ct''mtk_aes_tfm' into struct mtk_aes_info hence avoiding additional DMA-mapping. - adds 'keymode' in struct mtk_aes_base_ctx. When .setkey() callback is called, we store keybit setting in keymode. Doing so, there is no need to check keylen second time in mtk_aes_info_init() / mtk_aes_gcm_info_init(). Besides, this patch also removes unused macro definitions and adds helper inline function to write security information(key, IV,...) to info->state. Signed-off-by: Ryder Lee Signed-off-by: Herbert Xu --- drivers/crypto/mediatek/mtk-aes.c | 261 +++++++++++++++--------------- drivers/crypto/mediatek/mtk-sha.c | 92 +++++------ 2 files changed, 165 insertions(+), 188 deletions(-) diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c index 8f3efa581294..9e845e866dec 100644 --- a/drivers/crypto/mediatek/mtk-aes.c +++ b/drivers/crypto/mediatek/mtk-aes.c @@ -19,13 +19,10 @@ #define AES_BUF_ORDER 2 #define AES_BUF_SIZE ((PAGE_SIZE << AES_BUF_ORDER) \ & ~(AES_BLOCK_SIZE - 1)) +#define AES_MAX_STATE_BUF_SIZE SIZE_IN_WORDS(AES_KEYSIZE_256 + \ + AES_BLOCK_SIZE * 2) +#define AES_MAX_CT_SIZE 6 -/* AES command token size */ -#define AES_CT_SIZE_ECB 2 -#define AES_CT_SIZE_CBC 3 -#define AES_CT_SIZE_CTR 3 -#define AES_CT_SIZE_GCM_OUT 5 -#define AES_CT_SIZE_GCM_IN 6 #define AES_CT_CTRL_HDR cpu_to_le32(0x00220000) /* AES-CBC/ECB/CTR command token */ @@ -50,6 +47,8 @@ #define AES_TFM_128BITS cpu_to_le32(0xb << 16) #define AES_TFM_192BITS cpu_to_le32(0xd << 16) #define AES_TFM_256BITS cpu_to_le32(0xf << 16) +#define AES_TFM_GHASH_DIGEST cpu_to_le32(0x2 << 21) +#define AES_TFM_GHASH cpu_to_le32(0x4 << 23) /* AES transform information word 1 fields */ #define AES_TFM_ECB cpu_to_le32(0x0 << 0) #define AES_TFM_CBC cpu_to_le32(0x1 << 0) @@ -59,10 +58,9 @@ #define AES_TFM_FULL_IV cpu_to_le32(0xf << 5) /* using IV 0-3 */ #define AES_TFM_IV_CTR_MODE cpu_to_le32(0x1 << 10) #define AES_TFM_ENC_HASH cpu_to_le32(0x1 << 17) -#define AES_TFM_GHASH_DIG cpu_to_le32(0x2 << 21) -#define AES_TFM_GHASH cpu_to_le32(0x4 << 23) /* AES flags */ +#define AES_FLAGS_CIPHER_MSK GENMASK(2, 0) #define AES_FLAGS_ECB BIT(0) #define AES_FLAGS_CBC BIT(1) #define AES_FLAGS_CTR BIT(2) @@ -73,18 +71,12 @@ #define AES_AUTH_TAG_ERR cpu_to_le32(BIT(26)) /** - * Command token(CT) is a set of hardware instructions that - * are used to control engine's processing flow of AES. + * mtk_aes_info - hardware information of AES + * @cmd: command token, hardware instruction + * @tfm: transform state of cipher algorithm. + * @state: contains keys and initial vectors. * - * Transform information(TFM) is used to define AES state and - * contains all keys and initial vectors. - * - * The engine requires CT and TFM to do: - * - Commands decoding and control of the engine's data path. - * - Coordinating hardware data fetch and store operations. - * - Result token construction and output. - * - * Memory map of GCM's TFM: + * Memory layout of GCM buffer: * /-----------\ * | AES KEY | 128/196/256 bits * |-----------| @@ -92,14 +84,16 @@ * |-----------| * | IVs | 4 * 4 bytes * \-----------/ + * + * The engine requires all these info to do: + * - Commands decoding and control of the engine's data path. + * - Coordinating hardware data fetch and store operations. + * - Result token construction and output. */ -struct mtk_aes_ct { - __le32 cmd[AES_CT_SIZE_GCM_IN]; -}; - -struct mtk_aes_tfm { - __le32 ctrl[2]; - __le32 state[SIZE_IN_WORDS(AES_KEYSIZE_256 + AES_BLOCK_SIZE * 2)]; +struct mtk_aes_info { + __le32 cmd[AES_MAX_CT_SIZE]; + __le32 tfm[2]; + __le32 state[AES_MAX_STATE_BUF_SIZE]; }; struct mtk_aes_reqctx { @@ -109,11 +103,12 @@ struct mtk_aes_reqctx { struct mtk_aes_base_ctx { struct mtk_cryp *cryp; u32 keylen; + __le32 keymode; + mtk_aes_fn start; - struct mtk_aes_ct ct; + struct mtk_aes_info info; dma_addr_t ct_dma; - struct mtk_aes_tfm tfm; dma_addr_t tfm_dma; __le32 ct_hdr; @@ -250,6 +245,22 @@ static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma) sg->length += dma->remainder; } +static inline void mtk_aes_write_state_le(__le32 *dst, const u32 *src, u32 size) +{ + int i; + + for (i = 0; i < SIZE_IN_WORDS(size); i++) + dst[i] = cpu_to_le32(src[i]); +} + +static inline void mtk_aes_write_state_be(__be32 *dst, const u32 *src, u32 size) +{ + int i; + + for (i = 0; i < SIZE_IN_WORDS(size); i++) + dst[i] = cpu_to_be32(src[i]); +} + static inline int mtk_aes_complete(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, int err) @@ -331,9 +342,7 @@ static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) { struct mtk_aes_base_ctx *ctx = aes->ctx; - dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->ct), - DMA_TO_DEVICE); - dma_unmap_single(cryp->dev, ctx->tfm_dma, sizeof(ctx->tfm), + dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info), DMA_TO_DEVICE); if (aes->src.sg == aes->dst.sg) { @@ -364,16 +373,14 @@ static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) { struct mtk_aes_base_ctx *ctx = aes->ctx; + struct mtk_aes_info *info = &ctx->info; - ctx->ct_dma = dma_map_single(cryp->dev, &ctx->ct, sizeof(ctx->ct), + ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma))) goto exit; - ctx->tfm_dma = dma_map_single(cryp->dev, &ctx->tfm, sizeof(ctx->tfm), - DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(cryp->dev, ctx->tfm_dma))) - goto tfm_map_err; + ctx->tfm_dma = ctx->ct_dma + sizeof(info->cmd); if (aes->src.sg == aes->dst.sg) { aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg, @@ -400,11 +407,7 @@ static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) return mtk_aes_xmit(cryp, aes); sg_map_err: - dma_unmap_single(cryp->dev, ctx->tfm_dma, sizeof(ctx->tfm), - DMA_TO_DEVICE); -tfm_map_err: - dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->ct), - DMA_TO_DEVICE); + dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(*info), DMA_TO_DEVICE); exit: return mtk_aes_complete(cryp, aes, -EINVAL); } @@ -415,50 +418,43 @@ static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, { struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq); struct mtk_aes_base_ctx *ctx = aes->ctx; + struct mtk_aes_info *info = &ctx->info; + u32 cnt = 0; ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len); - ctx->ct.cmd[0] = AES_CMD0 | cpu_to_le32(len); - ctx->ct.cmd[1] = AES_CMD1; + info->cmd[cnt++] = AES_CMD0 | cpu_to_le32(len); + info->cmd[cnt++] = AES_CMD1; + info->tfm[0] = AES_TFM_SIZE(ctx->keylen) | ctx->keymode; if (aes->flags & AES_FLAGS_ENCRYPT) - ctx->tfm.ctrl[0] = AES_TFM_BASIC_OUT; + info->tfm[0] |= AES_TFM_BASIC_OUT; else - ctx->tfm.ctrl[0] = AES_TFM_BASIC_IN; + info->tfm[0] |= AES_TFM_BASIC_IN; - if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_128)) - ctx->tfm.ctrl[0] |= AES_TFM_128BITS; - else if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_256)) - ctx->tfm.ctrl[0] |= AES_TFM_256BITS; - else - ctx->tfm.ctrl[0] |= AES_TFM_192BITS; + switch (aes->flags & AES_FLAGS_CIPHER_MSK) { + case AES_FLAGS_CBC: + info->tfm[1] = AES_TFM_CBC; + break; + case AES_FLAGS_ECB: + info->tfm[1] = AES_TFM_ECB; + goto ecb; + case AES_FLAGS_CTR: + info->tfm[1] = AES_TFM_CTR_LOAD; + goto ctr; - if (aes->flags & AES_FLAGS_CBC) { - const u32 *iv = (const u32 *)req->info; - u32 *iv_state = ctx->tfm.state + ctx->keylen; - int i; - - ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen + - SIZE_IN_WORDS(AES_BLOCK_SIZE)); - ctx->tfm.ctrl[1] = AES_TFM_CBC | AES_TFM_FULL_IV; - - for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++) - iv_state[i] = cpu_to_le32(iv[i]); - - ctx->ct.cmd[2] = AES_CMD2; - ctx->ct_size = AES_CT_SIZE_CBC; - } else if (aes->flags & AES_FLAGS_ECB) { - ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen); - ctx->tfm.ctrl[1] = AES_TFM_ECB; - - ctx->ct_size = AES_CT_SIZE_ECB; - } else if (aes->flags & AES_FLAGS_CTR) { - ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen + - SIZE_IN_WORDS(AES_BLOCK_SIZE)); - ctx->tfm.ctrl[1] = AES_TFM_CTR_LOAD | AES_TFM_FULL_IV; - - ctx->ct.cmd[2] = AES_CMD2; - ctx->ct_size = AES_CT_SIZE_CTR; + default: + /* Should not happen... */ + return; } + + mtk_aes_write_state_le(info->state + ctx->keylen, req->info, + AES_BLOCK_SIZE); +ctr: + info->tfm[0] += AES_TFM_SIZE(SIZE_IN_WORDS(AES_BLOCK_SIZE)); + info->tfm[1] |= AES_TFM_FULL_IV; + info->cmd[cnt++] = AES_CMD2; +ecb: + ctx->ct_size = cnt; } static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, @@ -572,8 +568,7 @@ static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(ctx); struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq); struct scatterlist *src, *dst; - int i; - u32 start, end, ctr, blocks, *iv_state; + u32 start, end, ctr, blocks; size_t datalen; bool fragmented = false; @@ -602,9 +597,8 @@ static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) scatterwalk_ffwd(cctx->dst, req->dst, cctx->offset)); /* Write IVs into transform state buffer. */ - iv_state = ctx->tfm.state + ctx->keylen; - for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++) - iv_state[i] = cpu_to_le32(cctx->iv[i]); + mtk_aes_write_state_le(ctx->info.state + ctx->keylen, cctx->iv, + AES_BLOCK_SIZE); if (unlikely(fragmented)) { /* @@ -639,21 +633,25 @@ static int mtk_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, u32 keylen) { struct mtk_aes_base_ctx *ctx = crypto_ablkcipher_ctx(tfm); - const u32 *aes_key = (const u32 *)key; - u32 *key_state = ctx->tfm.state; - int i; - if (keylen != AES_KEYSIZE_128 && - keylen != AES_KEYSIZE_192 && - keylen != AES_KEYSIZE_256) { + switch (keylen) { + case AES_KEYSIZE_128: + ctx->keymode = AES_TFM_128BITS; + break; + case AES_KEYSIZE_192: + ctx->keymode = AES_TFM_192BITS; + break; + case AES_KEYSIZE_256: + ctx->keymode = AES_TFM_256BITS; + break; + + default: crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } ctx->keylen = SIZE_IN_WORDS(keylen); - - for (i = 0; i < ctx->keylen; i++) - key_state[i] = cpu_to_le32(aes_key[i]); + mtk_aes_write_state_le(ctx->info.state, (const u32 *)key, keylen); return 0; } @@ -825,45 +823,35 @@ static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp, struct aead_request *req = aead_request_cast(aes->areq); struct mtk_aes_base_ctx *ctx = aes->ctx; struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx); - const u32 *iv = (const u32 *)req->iv; - u32 *iv_state = ctx->tfm.state + ctx->keylen + - SIZE_IN_WORDS(AES_BLOCK_SIZE); + struct mtk_aes_info *info = &ctx->info; u32 ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req)); - int i; + u32 cnt = 0; ctx->ct_hdr = AES_CT_CTRL_HDR | len; - ctx->ct.cmd[0] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen); - ctx->ct.cmd[1] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen); - ctx->ct.cmd[2] = AES_GCM_CMD2; - ctx->ct.cmd[3] = AES_GCM_CMD3 | cpu_to_le32(gctx->textlen); + info->cmd[cnt++] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen); + info->cmd[cnt++] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen); + info->cmd[cnt++] = AES_GCM_CMD2; + info->cmd[cnt++] = AES_GCM_CMD3 | cpu_to_le32(gctx->textlen); if (aes->flags & AES_FLAGS_ENCRYPT) { - ctx->ct.cmd[4] = AES_GCM_CMD4 | cpu_to_le32(gctx->authsize); - ctx->ct_size = AES_CT_SIZE_GCM_OUT; - ctx->tfm.ctrl[0] = AES_TFM_GCM_OUT; + info->cmd[cnt++] = AES_GCM_CMD4 | cpu_to_le32(gctx->authsize); + info->tfm[0] = AES_TFM_GCM_OUT; } else { - ctx->ct.cmd[4] = AES_GCM_CMD5 | cpu_to_le32(gctx->authsize); - ctx->ct.cmd[5] = AES_GCM_CMD6 | cpu_to_le32(gctx->authsize); - ctx->ct_size = AES_CT_SIZE_GCM_IN; - ctx->tfm.ctrl[0] = AES_TFM_GCM_IN; + info->cmd[cnt++] = AES_GCM_CMD5 | cpu_to_le32(gctx->authsize); + info->cmd[cnt++] = AES_GCM_CMD6 | cpu_to_le32(gctx->authsize); + info->tfm[0] = AES_TFM_GCM_IN; } + ctx->ct_size = cnt; - if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_128)) - ctx->tfm.ctrl[0] |= AES_TFM_128BITS; - else if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_256)) - ctx->tfm.ctrl[0] |= AES_TFM_256BITS; - else - ctx->tfm.ctrl[0] |= AES_TFM_192BITS; + info->tfm[0] |= AES_TFM_GHASH_DIGEST | AES_TFM_GHASH | AES_TFM_SIZE( + ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE + ivsize)) | + ctx->keymode; + info->tfm[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE | AES_TFM_3IV | + AES_TFM_ENC_HASH; - ctx->tfm.ctrl[0] |= AES_TFM_GHASH_DIG | AES_TFM_GHASH | - AES_TFM_SIZE(ctx->keylen + SIZE_IN_WORDS( - AES_BLOCK_SIZE + ivsize)); - ctx->tfm.ctrl[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE | - AES_TFM_3IV | AES_TFM_ENC_HASH; - - for (i = 0; i < SIZE_IN_WORDS(ivsize); i++) - iv_state[i] = cpu_to_le32(iv[i]); + mtk_aes_write_state_le(info->state + ctx->keylen + SIZE_IN_WORDS( + AES_BLOCK_SIZE), (const u32 *)req->iv, ivsize); } static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, @@ -979,24 +967,26 @@ static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, struct scatterlist sg[1]; struct skcipher_request req; } *data; - const u32 *aes_key; - u32 *key_state, *hash_state; - int err, i; + int err; - if (keylen != AES_KEYSIZE_256 && - keylen != AES_KEYSIZE_192 && - keylen != AES_KEYSIZE_128) { + switch (keylen) { + case AES_KEYSIZE_128: + ctx->keymode = AES_TFM_128BITS; + break; + case AES_KEYSIZE_192: + ctx->keymode = AES_TFM_192BITS; + break; + case AES_KEYSIZE_256: + ctx->keymode = AES_TFM_256BITS; + break; + + default: crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } - key_state = ctx->tfm.state; - aes_key = (u32 *)key; ctx->keylen = SIZE_IN_WORDS(keylen); - for (i = 0; i < ctx->keylen; i++) - ctx->tfm.state[i] = cpu_to_le32(aes_key[i]); - /* Same as crypto_gcm_setkey() from crypto/gcm.c */ crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) & @@ -1031,10 +1021,11 @@ static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, if (err) goto out; - hash_state = key_state + ctx->keylen; - - for (i = 0; i < 4; i++) - hash_state[i] = cpu_to_be32(data->hash[i]); + /* Write key into state buffer */ + mtk_aes_write_state_le(ctx->info.state, (const u32 *)key, keylen); + /* Write key(H) into state buffer */ + mtk_aes_write_state_be(ctx->info.state + ctx->keylen, data->hash, + AES_BLOCK_SIZE); out: kzfree(data); return err; diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index dd3582b6a1a5..2226f12d1c7a 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -23,6 +23,7 @@ #define SHA_OP_FINAL 2 #define SHA_DATA_LEN_MSK cpu_to_le32(GENMASK(16, 0)) +#define SHA_MAX_DIGEST_BUF_SIZE 32 /* SHA command token */ #define SHA_CT_SIZE 5 @@ -33,7 +34,6 @@ /* SHA transform information */ #define SHA_TFM_HASH cpu_to_le32(0x2 << 0) -#define SHA_TFM_INNER_DIG cpu_to_le32(0x1 << 21) #define SHA_TFM_SIZE(x) cpu_to_le32((x) << 8) #define SHA_TFM_START cpu_to_le32(0x1 << 4) #define SHA_TFM_CONTINUE cpu_to_le32(0x1 << 5) @@ -60,31 +60,17 @@ #define SHA_FLAGS_PAD BIT(10) /** - * mtk_sha_ct is a set of hardware instructions(command token) - * that are used to control engine's processing flow of SHA, - * and it contains the first two words of transform state. - */ -struct mtk_sha_ct { - __le32 ctrl[2]; - __le32 cmd[3]; -}; - -/** - * mtk_sha_tfm is used to define SHA transform state - * and store result digest that produced by engine. - */ -struct mtk_sha_tfm { - __le32 ctrl[2]; - __le32 digest[SIZE_IN_WORDS(SHA512_DIGEST_SIZE)]; -}; - -/** - * mtk_sha_info consists of command token and transform state - * of SHA, its role is similar to mtk_aes_info. + * mtk_sha_info - hardware information of AES + * @cmd: command token, hardware instruction + * @tfm: transform state of cipher algorithm. + * @state: contains keys and initial vectors. + * */ struct mtk_sha_info { - struct mtk_sha_ct ct; - struct mtk_sha_tfm tfm; + __le32 ctrl[2]; + __le32 cmd[3]; + __le32 tfm[2]; + __le32 digest[SHA_MAX_DIGEST_BUF_SIZE]; }; struct mtk_sha_reqctx { @@ -93,7 +79,6 @@ struct mtk_sha_reqctx { unsigned long op; u64 digcnt; - bool start; size_t bufcnt; dma_addr_t dma_addr; @@ -265,7 +250,9 @@ static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len) bits[1] = cpu_to_be64(size << 3); bits[0] = cpu_to_be64(size >> 61); - if (ctx->flags & (SHA_FLAGS_SHA384 | SHA_FLAGS_SHA512)) { + switch (ctx->flags & SHA_FLAGS_ALGO_MSK) { + case SHA_FLAGS_SHA384: + case SHA_FLAGS_SHA512: index = ctx->bufcnt & 0x7f; padlen = (index < 112) ? (112 - index) : ((128 + 112) - index); *(ctx->buffer + ctx->bufcnt) = 0x80; @@ -273,7 +260,9 @@ static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len) memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16); ctx->bufcnt += padlen + 16; ctx->flags |= SHA_FLAGS_PAD; - } else { + break; + + default: index = ctx->bufcnt & 0x3f; padlen = (index < 56) ? (56 - index) : ((64 + 56) - index); *(ctx->buffer + ctx->bufcnt) = 0x80; @@ -281,36 +270,35 @@ static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len) memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8); ctx->bufcnt += padlen + 8; ctx->flags |= SHA_FLAGS_PAD; + break; } } /* Initialize basic transform information of SHA */ static void mtk_sha_info_init(struct mtk_sha_reqctx *ctx) { - struct mtk_sha_ct *ct = &ctx->info.ct; - struct mtk_sha_tfm *tfm = &ctx->info.tfm; + struct mtk_sha_info *info = &ctx->info; ctx->ct_hdr = SHA_CT_CTRL_HDR; ctx->ct_size = SHA_CT_SIZE; - tfm->ctrl[0] = SHA_TFM_HASH | SHA_TFM_INNER_DIG | - SHA_TFM_SIZE(SIZE_IN_WORDS(ctx->ds)); + info->tfm[0] = SHA_TFM_HASH | SHA_TFM_SIZE(SIZE_IN_WORDS(ctx->ds)); switch (ctx->flags & SHA_FLAGS_ALGO_MSK) { case SHA_FLAGS_SHA1: - tfm->ctrl[0] |= SHA_TFM_SHA1; + info->tfm[0] |= SHA_TFM_SHA1; break; case SHA_FLAGS_SHA224: - tfm->ctrl[0] |= SHA_TFM_SHA224; + info->tfm[0] |= SHA_TFM_SHA224; break; case SHA_FLAGS_SHA256: - tfm->ctrl[0] |= SHA_TFM_SHA256; + info->tfm[0] |= SHA_TFM_SHA256; break; case SHA_FLAGS_SHA384: - tfm->ctrl[0] |= SHA_TFM_SHA384; + info->tfm[0] |= SHA_TFM_SHA384; break; case SHA_FLAGS_SHA512: - tfm->ctrl[0] |= SHA_TFM_SHA512; + info->tfm[0] |= SHA_TFM_SHA512; break; default: @@ -318,13 +306,13 @@ static void mtk_sha_info_init(struct mtk_sha_reqctx *ctx) return; } - tfm->ctrl[1] = SHA_TFM_HASH_STORE; - ct->ctrl[0] = tfm->ctrl[0] | SHA_TFM_CONTINUE | SHA_TFM_START; - ct->ctrl[1] = tfm->ctrl[1]; + info->tfm[1] = SHA_TFM_HASH_STORE; + info->ctrl[0] = info->tfm[0] | SHA_TFM_CONTINUE | SHA_TFM_START; + info->ctrl[1] = info->tfm[1]; - ct->cmd[0] = SHA_CMD0; - ct->cmd[1] = SHA_CMD1; - ct->cmd[2] = SHA_CMD2 | SHA_TFM_DIGEST(SIZE_IN_WORDS(ctx->ds)); + info->cmd[0] = SHA_CMD0; + info->cmd[1] = SHA_CMD1; + info->cmd[2] = SHA_CMD2 | SHA_TFM_DIGEST(SIZE_IN_WORDS(ctx->ds)); } /* @@ -337,17 +325,15 @@ static int mtk_sha_info_update(struct mtk_cryp *cryp, { struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); struct mtk_sha_info *info = &ctx->info; - struct mtk_sha_ct *ct = &info->ct; - - if (ctx->start) - ctx->start = false; - else - ct->ctrl[0] &= ~SHA_TFM_START; ctx->ct_hdr &= ~SHA_DATA_LEN_MSK; ctx->ct_hdr |= cpu_to_le32(len1 + len2); - ct->cmd[0] &= ~SHA_DATA_LEN_MSK; - ct->cmd[0] |= cpu_to_le32(len1 + len2); + info->cmd[0] &= ~SHA_DATA_LEN_MSK; + info->cmd[0] |= cpu_to_le32(len1 + len2); + + /* Setting SHA_TFM_START only for the first iteration */ + if (ctx->digcnt) + info->ctrl[0] &= ~SHA_TFM_START; ctx->digcnt += len1; @@ -357,7 +343,8 @@ static int mtk_sha_info_update(struct mtk_cryp *cryp, dev_err(cryp->dev, "dma %zu bytes error\n", sizeof(*info)); return -EINVAL; } - ctx->tfm_dma = ctx->ct_dma + sizeof(*ct); + + ctx->tfm_dma = ctx->ct_dma + sizeof(info->ctrl) + sizeof(info->cmd); return 0; } @@ -422,7 +409,6 @@ static int mtk_sha_init(struct ahash_request *req) ctx->bufcnt = 0; ctx->digcnt = 0; ctx->buffer = tctx->buf; - ctx->start = true; if (tctx->flags & SHA_FLAGS_HMAC) { struct mtk_sha_hmac_ctx *bctx = tctx->base; @@ -635,7 +621,7 @@ static int mtk_sha_final_req(struct mtk_cryp *cryp, static int mtk_sha_finish(struct ahash_request *req) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - u32 *digest = ctx->info.tfm.digest; + __le32 *digest = ctx->info.digest; u32 *result = (u32 *)req->result; int i; From 986bdd5944b3d630431198759517e79f60bd8fff Mon Sep 17 00:00:00 2001 From: Fabien DESSENNE Date: Thu, 9 Mar 2017 10:40:05 +0100 Subject: [PATCH 028/101] crypto: doc - fix typo (struct sdesc) Add missing " " in api-samples.rst Signed-off-by: Fabien Dessenne Signed-off-by: Herbert Xu --- Documentation/crypto/api-samples.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/crypto/api-samples.rst b/Documentation/crypto/api-samples.rst index 0a10819f6107..d021fd96a76d 100644 --- a/Documentation/crypto/api-samples.rst +++ b/Documentation/crypto/api-samples.rst @@ -155,9 +155,9 @@ Code Example For Use of Operational State Memory With SHASH char ctx[]; }; - static struct sdescinit_sdesc(struct crypto_shash *alg) + static struct sdesc init_sdesc(struct crypto_shash *alg) { - struct sdescsdesc; + struct sdesc sdesc; int size; size = sizeof(struct shash_desc) + crypto_shash_descsize(alg); @@ -172,7 +172,7 @@ Code Example For Use of Operational State Memory With SHASH static int calc_hash(struct crypto_shashalg, const unsigned chardata, unsigned int datalen, unsigned chardigest) { - struct sdescsdesc; + struct sdesc sdesc; int ret; sdesc = init_sdesc(alg); From 28845b079b9611b3082aa53e2dae2ea10a16a1c2 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sat, 11 Mar 2017 08:11:00 +0200 Subject: [PATCH 029/101] MAINTAINERS: Add maintianer entry for crypto/s5p-sss Add Krzysztof Kozlowski and Vladimir Zapolskiy as maintainers of s5p-sss driver for handling reviews, testing and getting bug reports from the users. Cc: Vladimir Zapolskiy Cc: Herbert Xu Signed-off-by: Krzysztof Kozlowski Acked-by: Vladimir Zapolskiy Signed-off-by: Herbert Xu --- MAINTAINERS | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index c265a5fe4848..01565164bcc6 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10931,6 +10931,14 @@ F: Documentation/devicetree/bindings/regulator/samsung,s2m*.txt F: Documentation/devicetree/bindings/regulator/samsung,s5m*.txt F: Documentation/devicetree/bindings/clock/samsung,s2mps11.txt +SAMSUNG S5P Security SubSystem (SSS) DRIVER +M: Krzysztof Kozlowski +M: Vladimir Zapolskiy +L: linux-crypto@vger.kernel.org +L: linux-samsung-soc@vger.kernel.org +S: Maintained +F: drivers/crypto/s5p-sss.c + SAMSUNG S5P/EXYNOS4 SOC SERIES CAMERA SUBSYSTEM DRIVERS M: Kyungmin Park M: Sylwester Nawrocki From de696a26435ae50f06fd913bc84e0bf602caee1f Mon Sep 17 00:00:00 2001 From: Daniel Axtens Date: Wed, 15 Mar 2017 23:37:34 +1100 Subject: [PATCH 030/101] crypto: powerpc - Factor out the core CRC vpmsum algorithm The core nuts and bolts of the crc32c vpmsum algorithm will also work for a number of other CRC algorithms with different polynomials. Factor out the function into a new asm file. To handle multiple users of the function, a user simply provides constants, defines the name of their CRC function, and then #includes the core algorithm file. Cc: Anton Blanchard Signed-off-by: Daniel Axtens Signed-off-by: Herbert Xu --- arch/powerpc/crypto/crc32-vpmsum_core.S | 726 ++++++++++++++++++++++++ arch/powerpc/crypto/crc32c-vpmsum_asm.S | 714 +---------------------- 2 files changed, 729 insertions(+), 711 deletions(-) create mode 100644 arch/powerpc/crypto/crc32-vpmsum_core.S diff --git a/arch/powerpc/crypto/crc32-vpmsum_core.S b/arch/powerpc/crypto/crc32-vpmsum_core.S new file mode 100644 index 000000000000..7c6be6a5c977 --- /dev/null +++ b/arch/powerpc/crypto/crc32-vpmsum_core.S @@ -0,0 +1,726 @@ +/* + * Core of the accelerated CRC algorithm. + * In your file, define the constants and CRC_FUNCTION_NAME + * Then include this file. + * + * Calculate the checksum of data that is 16 byte aligned and a multiple of + * 16 bytes. + * + * The first step is to reduce it to 1024 bits. We do this in 8 parallel + * chunks in order to mask the latency of the vpmsum instructions. If we + * have more than 32 kB of data to checksum we repeat this step multiple + * times, passing in the previous 1024 bits. + * + * The next step is to reduce the 1024 bits to 64 bits. This step adds + * 32 bits of 0s to the end - this matches what a CRC does. We just + * calculate constants that land the data in this 32 bits. + * + * We then use fixed point Barrett reduction to compute a mod n over GF(2) + * for n = CRC using POWER8 instructions. We use x = 32. + * + * http://en.wikipedia.org/wiki/Barrett_reduction + * + * Copyright (C) 2015 Anton Blanchard , IBM + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. +*/ + +#include +#include + +#define MAX_SIZE 32768 + + .text + +#if defined(__BIG_ENDIAN__) +#define BYTESWAP_DATA +#else +#undef BYTESWAP_DATA +#endif + +#define off16 r25 +#define off32 r26 +#define off48 r27 +#define off64 r28 +#define off80 r29 +#define off96 r30 +#define off112 r31 + +#define const1 v24 +#define const2 v25 + +#define byteswap v26 +#define mask_32bit v27 +#define mask_64bit v28 +#define zeroes v29 + +#ifdef BYTESWAP_DATA +#define VPERM(A, B, C, D) vperm A, B, C, D +#else +#define VPERM(A, B, C, D) +#endif + +/* unsigned int CRC_FUNCTION_NAME(unsigned int crc, void *p, unsigned long len) */ +FUNC_START(CRC_FUNCTION_NAME) + std r31,-8(r1) + std r30,-16(r1) + std r29,-24(r1) + std r28,-32(r1) + std r27,-40(r1) + std r26,-48(r1) + std r25,-56(r1) + + li off16,16 + li off32,32 + li off48,48 + li off64,64 + li off80,80 + li off96,96 + li off112,112 + li r0,0 + + /* Enough room for saving 10 non volatile VMX registers */ + subi r6,r1,56+10*16 + subi r7,r1,56+2*16 + + stvx v20,0,r6 + stvx v21,off16,r6 + stvx v22,off32,r6 + stvx v23,off48,r6 + stvx v24,off64,r6 + stvx v25,off80,r6 + stvx v26,off96,r6 + stvx v27,off112,r6 + stvx v28,0,r7 + stvx v29,off16,r7 + + mr r10,r3 + + vxor zeroes,zeroes,zeroes + vspltisw v0,-1 + + vsldoi mask_32bit,zeroes,v0,4 + vsldoi mask_64bit,zeroes,v0,8 + + /* Get the initial value into v8 */ + vxor v8,v8,v8 + MTVRD(v8, R3) + vsldoi v8,zeroes,v8,8 /* shift into bottom 32 bits */ + +#ifdef BYTESWAP_DATA + addis r3,r2,.byteswap_constant@toc@ha + addi r3,r3,.byteswap_constant@toc@l + + lvx byteswap,0,r3 + addi r3,r3,16 +#endif + + cmpdi r5,256 + blt .Lshort + + rldicr r6,r5,0,56 + + /* Checksum in blocks of MAX_SIZE */ +1: lis r7,MAX_SIZE@h + ori r7,r7,MAX_SIZE@l + mr r9,r7 + cmpd r6,r7 + bgt 2f + mr r7,r6 +2: subf r6,r7,r6 + + /* our main loop does 128 bytes at a time */ + srdi r7,r7,7 + + /* + * Work out the offset into the constants table to start at. Each + * constant is 16 bytes, and it is used against 128 bytes of input + * data - 128 / 16 = 8 + */ + sldi r8,r7,4 + srdi r9,r9,3 + subf r8,r8,r9 + + /* We reduce our final 128 bytes in a separate step */ + addi r7,r7,-1 + mtctr r7 + + addis r3,r2,.constants@toc@ha + addi r3,r3,.constants@toc@l + + /* Find the start of our constants */ + add r3,r3,r8 + + /* zero v0-v7 which will contain our checksums */ + vxor v0,v0,v0 + vxor v1,v1,v1 + vxor v2,v2,v2 + vxor v3,v3,v3 + vxor v4,v4,v4 + vxor v5,v5,v5 + vxor v6,v6,v6 + vxor v7,v7,v7 + + lvx const1,0,r3 + + /* + * If we are looping back to consume more data we use the values + * already in v16-v23. + */ + cmpdi r0,1 + beq 2f + + /* First warm up pass */ + lvx v16,0,r4 + lvx v17,off16,r4 + VPERM(v16,v16,v16,byteswap) + VPERM(v17,v17,v17,byteswap) + lvx v18,off32,r4 + lvx v19,off48,r4 + VPERM(v18,v18,v18,byteswap) + VPERM(v19,v19,v19,byteswap) + lvx v20,off64,r4 + lvx v21,off80,r4 + VPERM(v20,v20,v20,byteswap) + VPERM(v21,v21,v21,byteswap) + lvx v22,off96,r4 + lvx v23,off112,r4 + VPERM(v22,v22,v22,byteswap) + VPERM(v23,v23,v23,byteswap) + addi r4,r4,8*16 + + /* xor in initial value */ + vxor v16,v16,v8 + +2: bdz .Lfirst_warm_up_done + + addi r3,r3,16 + lvx const2,0,r3 + + /* Second warm up pass */ + VPMSUMD(v8,v16,const1) + lvx v16,0,r4 + VPERM(v16,v16,v16,byteswap) + ori r2,r2,0 + + VPMSUMD(v9,v17,const1) + lvx v17,off16,r4 + VPERM(v17,v17,v17,byteswap) + ori r2,r2,0 + + VPMSUMD(v10,v18,const1) + lvx v18,off32,r4 + VPERM(v18,v18,v18,byteswap) + ori r2,r2,0 + + VPMSUMD(v11,v19,const1) + lvx v19,off48,r4 + VPERM(v19,v19,v19,byteswap) + ori r2,r2,0 + + VPMSUMD(v12,v20,const1) + lvx v20,off64,r4 + VPERM(v20,v20,v20,byteswap) + ori r2,r2,0 + + VPMSUMD(v13,v21,const1) + lvx v21,off80,r4 + VPERM(v21,v21,v21,byteswap) + ori r2,r2,0 + + VPMSUMD(v14,v22,const1) + lvx v22,off96,r4 + VPERM(v22,v22,v22,byteswap) + ori r2,r2,0 + + VPMSUMD(v15,v23,const1) + lvx v23,off112,r4 + VPERM(v23,v23,v23,byteswap) + + addi r4,r4,8*16 + + bdz .Lfirst_cool_down + + /* + * main loop. We modulo schedule it such that it takes three iterations + * to complete - first iteration load, second iteration vpmsum, third + * iteration xor. + */ + .balign 16 +4: lvx const1,0,r3 + addi r3,r3,16 + ori r2,r2,0 + + vxor v0,v0,v8 + VPMSUMD(v8,v16,const2) + lvx v16,0,r4 + VPERM(v16,v16,v16,byteswap) + ori r2,r2,0 + + vxor v1,v1,v9 + VPMSUMD(v9,v17,const2) + lvx v17,off16,r4 + VPERM(v17,v17,v17,byteswap) + ori r2,r2,0 + + vxor v2,v2,v10 + VPMSUMD(v10,v18,const2) + lvx v18,off32,r4 + VPERM(v18,v18,v18,byteswap) + ori r2,r2,0 + + vxor v3,v3,v11 + VPMSUMD(v11,v19,const2) + lvx v19,off48,r4 + VPERM(v19,v19,v19,byteswap) + lvx const2,0,r3 + ori r2,r2,0 + + vxor v4,v4,v12 + VPMSUMD(v12,v20,const1) + lvx v20,off64,r4 + VPERM(v20,v20,v20,byteswap) + ori r2,r2,0 + + vxor v5,v5,v13 + VPMSUMD(v13,v21,const1) + lvx v21,off80,r4 + VPERM(v21,v21,v21,byteswap) + ori r2,r2,0 + + vxor v6,v6,v14 + VPMSUMD(v14,v22,const1) + lvx v22,off96,r4 + VPERM(v22,v22,v22,byteswap) + ori r2,r2,0 + + vxor v7,v7,v15 + VPMSUMD(v15,v23,const1) + lvx v23,off112,r4 + VPERM(v23,v23,v23,byteswap) + + addi r4,r4,8*16 + + bdnz 4b + +.Lfirst_cool_down: + /* First cool down pass */ + lvx const1,0,r3 + addi r3,r3,16 + + vxor v0,v0,v8 + VPMSUMD(v8,v16,const1) + ori r2,r2,0 + + vxor v1,v1,v9 + VPMSUMD(v9,v17,const1) + ori r2,r2,0 + + vxor v2,v2,v10 + VPMSUMD(v10,v18,const1) + ori r2,r2,0 + + vxor v3,v3,v11 + VPMSUMD(v11,v19,const1) + ori r2,r2,0 + + vxor v4,v4,v12 + VPMSUMD(v12,v20,const1) + ori r2,r2,0 + + vxor v5,v5,v13 + VPMSUMD(v13,v21,const1) + ori r2,r2,0 + + vxor v6,v6,v14 + VPMSUMD(v14,v22,const1) + ori r2,r2,0 + + vxor v7,v7,v15 + VPMSUMD(v15,v23,const1) + ori r2,r2,0 + +.Lsecond_cool_down: + /* Second cool down pass */ + vxor v0,v0,v8 + vxor v1,v1,v9 + vxor v2,v2,v10 + vxor v3,v3,v11 + vxor v4,v4,v12 + vxor v5,v5,v13 + vxor v6,v6,v14 + vxor v7,v7,v15 + + /* + * vpmsumd produces a 96 bit result in the least significant bits + * of the register. Since we are bit reflected we have to shift it + * left 32 bits so it occupies the least significant bits in the + * bit reflected domain. + */ + vsldoi v0,v0,zeroes,4 + vsldoi v1,v1,zeroes,4 + vsldoi v2,v2,zeroes,4 + vsldoi v3,v3,zeroes,4 + vsldoi v4,v4,zeroes,4 + vsldoi v5,v5,zeroes,4 + vsldoi v6,v6,zeroes,4 + vsldoi v7,v7,zeroes,4 + + /* xor with last 1024 bits */ + lvx v8,0,r4 + lvx v9,off16,r4 + VPERM(v8,v8,v8,byteswap) + VPERM(v9,v9,v9,byteswap) + lvx v10,off32,r4 + lvx v11,off48,r4 + VPERM(v10,v10,v10,byteswap) + VPERM(v11,v11,v11,byteswap) + lvx v12,off64,r4 + lvx v13,off80,r4 + VPERM(v12,v12,v12,byteswap) + VPERM(v13,v13,v13,byteswap) + lvx v14,off96,r4 + lvx v15,off112,r4 + VPERM(v14,v14,v14,byteswap) + VPERM(v15,v15,v15,byteswap) + + addi r4,r4,8*16 + + vxor v16,v0,v8 + vxor v17,v1,v9 + vxor v18,v2,v10 + vxor v19,v3,v11 + vxor v20,v4,v12 + vxor v21,v5,v13 + vxor v22,v6,v14 + vxor v23,v7,v15 + + li r0,1 + cmpdi r6,0 + addi r6,r6,128 + bne 1b + + /* Work out how many bytes we have left */ + andi. r5,r5,127 + + /* Calculate where in the constant table we need to start */ + subfic r6,r5,128 + add r3,r3,r6 + + /* How many 16 byte chunks are in the tail */ + srdi r7,r5,4 + mtctr r7 + + /* + * Reduce the previously calculated 1024 bits to 64 bits, shifting + * 32 bits to include the trailing 32 bits of zeros + */ + lvx v0,0,r3 + lvx v1,off16,r3 + lvx v2,off32,r3 + lvx v3,off48,r3 + lvx v4,off64,r3 + lvx v5,off80,r3 + lvx v6,off96,r3 + lvx v7,off112,r3 + addi r3,r3,8*16 + + VPMSUMW(v0,v16,v0) + VPMSUMW(v1,v17,v1) + VPMSUMW(v2,v18,v2) + VPMSUMW(v3,v19,v3) + VPMSUMW(v4,v20,v4) + VPMSUMW(v5,v21,v5) + VPMSUMW(v6,v22,v6) + VPMSUMW(v7,v23,v7) + + /* Now reduce the tail (0 - 112 bytes) */ + cmpdi r7,0 + beq 1f + + lvx v16,0,r4 + lvx v17,0,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off16,r4 + lvx v17,off16,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off32,r4 + lvx v17,off32,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off48,r4 + lvx v17,off48,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off64,r4 + lvx v17,off64,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off80,r4 + lvx v17,off80,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off96,r4 + lvx v17,off96,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + + /* Now xor all the parallel chunks together */ +1: vxor v0,v0,v1 + vxor v2,v2,v3 + vxor v4,v4,v5 + vxor v6,v6,v7 + + vxor v0,v0,v2 + vxor v4,v4,v6 + + vxor v0,v0,v4 + +.Lbarrett_reduction: + /* Barrett constants */ + addis r3,r2,.barrett_constants@toc@ha + addi r3,r3,.barrett_constants@toc@l + + lvx const1,0,r3 + lvx const2,off16,r3 + + vsldoi v1,v0,v0,8 + vxor v0,v0,v1 /* xor two 64 bit results together */ + + /* shift left one bit */ + vspltisb v1,1 + vsl v0,v0,v1 + + vand v0,v0,mask_64bit + + /* + * The reflected version of Barrett reduction. Instead of bit + * reflecting our data (which is expensive to do), we bit reflect our + * constants and our algorithm, which means the intermediate data in + * our vector registers goes from 0-63 instead of 63-0. We can reflect + * the algorithm because we don't carry in mod 2 arithmetic. + */ + vand v1,v0,mask_32bit /* bottom 32 bits of a */ + VPMSUMD(v1,v1,const1) /* ma */ + vand v1,v1,mask_32bit /* bottom 32bits of ma */ + VPMSUMD(v1,v1,const2) /* qn */ + vxor v0,v0,v1 /* a - qn, subtraction is xor in GF(2) */ + + /* + * Since we are bit reflected, the result (ie the low 32 bits) is in + * the high 32 bits. We just need to shift it left 4 bytes + * V0 [ 0 1 X 3 ] + * V0 [ 0 X 2 3 ] + */ + vsldoi v0,v0,zeroes,4 /* shift result into top 64 bits of */ + + /* Get it into r3 */ + MFVRD(R3, v0) + +.Lout: + subi r6,r1,56+10*16 + subi r7,r1,56+2*16 + + lvx v20,0,r6 + lvx v21,off16,r6 + lvx v22,off32,r6 + lvx v23,off48,r6 + lvx v24,off64,r6 + lvx v25,off80,r6 + lvx v26,off96,r6 + lvx v27,off112,r6 + lvx v28,0,r7 + lvx v29,off16,r7 + + ld r31,-8(r1) + ld r30,-16(r1) + ld r29,-24(r1) + ld r28,-32(r1) + ld r27,-40(r1) + ld r26,-48(r1) + ld r25,-56(r1) + + blr + +.Lfirst_warm_up_done: + lvx const1,0,r3 + addi r3,r3,16 + + VPMSUMD(v8,v16,const1) + VPMSUMD(v9,v17,const1) + VPMSUMD(v10,v18,const1) + VPMSUMD(v11,v19,const1) + VPMSUMD(v12,v20,const1) + VPMSUMD(v13,v21,const1) + VPMSUMD(v14,v22,const1) + VPMSUMD(v15,v23,const1) + + b .Lsecond_cool_down + +.Lshort: + cmpdi r5,0 + beq .Lzero + + addis r3,r2,.short_constants@toc@ha + addi r3,r3,.short_constants@toc@l + + /* Calculate where in the constant table we need to start */ + subfic r6,r5,256 + add r3,r3,r6 + + /* How many 16 byte chunks? */ + srdi r7,r5,4 + mtctr r7 + + vxor v19,v19,v19 + vxor v20,v20,v20 + + lvx v0,0,r4 + lvx v16,0,r3 + VPERM(v0,v0,v16,byteswap) + vxor v0,v0,v8 /* xor in initial value */ + VPMSUMW(v0,v0,v16) + bdz .Lv0 + + lvx v1,off16,r4 + lvx v17,off16,r3 + VPERM(v1,v1,v17,byteswap) + VPMSUMW(v1,v1,v17) + bdz .Lv1 + + lvx v2,off32,r4 + lvx v16,off32,r3 + VPERM(v2,v2,v16,byteswap) + VPMSUMW(v2,v2,v16) + bdz .Lv2 + + lvx v3,off48,r4 + lvx v17,off48,r3 + VPERM(v3,v3,v17,byteswap) + VPMSUMW(v3,v3,v17) + bdz .Lv3 + + lvx v4,off64,r4 + lvx v16,off64,r3 + VPERM(v4,v4,v16,byteswap) + VPMSUMW(v4,v4,v16) + bdz .Lv4 + + lvx v5,off80,r4 + lvx v17,off80,r3 + VPERM(v5,v5,v17,byteswap) + VPMSUMW(v5,v5,v17) + bdz .Lv5 + + lvx v6,off96,r4 + lvx v16,off96,r3 + VPERM(v6,v6,v16,byteswap) + VPMSUMW(v6,v6,v16) + bdz .Lv6 + + lvx v7,off112,r4 + lvx v17,off112,r3 + VPERM(v7,v7,v17,byteswap) + VPMSUMW(v7,v7,v17) + bdz .Lv7 + + addi r3,r3,128 + addi r4,r4,128 + + lvx v8,0,r4 + lvx v16,0,r3 + VPERM(v8,v8,v16,byteswap) + VPMSUMW(v8,v8,v16) + bdz .Lv8 + + lvx v9,off16,r4 + lvx v17,off16,r3 + VPERM(v9,v9,v17,byteswap) + VPMSUMW(v9,v9,v17) + bdz .Lv9 + + lvx v10,off32,r4 + lvx v16,off32,r3 + VPERM(v10,v10,v16,byteswap) + VPMSUMW(v10,v10,v16) + bdz .Lv10 + + lvx v11,off48,r4 + lvx v17,off48,r3 + VPERM(v11,v11,v17,byteswap) + VPMSUMW(v11,v11,v17) + bdz .Lv11 + + lvx v12,off64,r4 + lvx v16,off64,r3 + VPERM(v12,v12,v16,byteswap) + VPMSUMW(v12,v12,v16) + bdz .Lv12 + + lvx v13,off80,r4 + lvx v17,off80,r3 + VPERM(v13,v13,v17,byteswap) + VPMSUMW(v13,v13,v17) + bdz .Lv13 + + lvx v14,off96,r4 + lvx v16,off96,r3 + VPERM(v14,v14,v16,byteswap) + VPMSUMW(v14,v14,v16) + bdz .Lv14 + + lvx v15,off112,r4 + lvx v17,off112,r3 + VPERM(v15,v15,v17,byteswap) + VPMSUMW(v15,v15,v17) + +.Lv15: vxor v19,v19,v15 +.Lv14: vxor v20,v20,v14 +.Lv13: vxor v19,v19,v13 +.Lv12: vxor v20,v20,v12 +.Lv11: vxor v19,v19,v11 +.Lv10: vxor v20,v20,v10 +.Lv9: vxor v19,v19,v9 +.Lv8: vxor v20,v20,v8 +.Lv7: vxor v19,v19,v7 +.Lv6: vxor v20,v20,v6 +.Lv5: vxor v19,v19,v5 +.Lv4: vxor v20,v20,v4 +.Lv3: vxor v19,v19,v3 +.Lv2: vxor v20,v20,v2 +.Lv1: vxor v19,v19,v1 +.Lv0: vxor v20,v20,v0 + + vxor v0,v19,v20 + + b .Lbarrett_reduction + +.Lzero: + mr r3,r10 + b .Lout + +FUNC_END(CRC_FUNCTION_NAME) diff --git a/arch/powerpc/crypto/crc32c-vpmsum_asm.S b/arch/powerpc/crypto/crc32c-vpmsum_asm.S index dc640b212299..c0d080caefc1 100644 --- a/arch/powerpc/crypto/crc32c-vpmsum_asm.S +++ b/arch/powerpc/crypto/crc32c-vpmsum_asm.S @@ -1,20 +1,5 @@ /* - * Calculate the checksum of data that is 16 byte aligned and a multiple of - * 16 bytes. - * - * The first step is to reduce it to 1024 bits. We do this in 8 parallel - * chunks in order to mask the latency of the vpmsum instructions. If we - * have more than 32 kB of data to checksum we repeat this step multiple - * times, passing in the previous 1024 bits. - * - * The next step is to reduce the 1024 bits to 64 bits. This step adds - * 32 bits of 0s to the end - this matches what a CRC does. We just - * calculate constants that land the data in this 32 bits. - * - * We then use fixed point Barrett reduction to compute a mod n over GF(2) - * for n = CRC using POWER8 instructions. We use x = 32. - * - * http://en.wikipedia.org/wiki/Barrett_reduction + * Calculate a crc32c with vpmsum acceleration * * Copyright (C) 2015 Anton Blanchard , IBM * @@ -23,9 +8,6 @@ * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. */ -#include -#include - .section .rodata .balign 16 @@ -33,7 +15,6 @@ /* byte reverse permute constant */ .octa 0x0F0E0D0C0B0A09080706050403020100 -#define MAX_SIZE 32768 .constants: /* Reduce 262144 kbits to 1024 bits */ @@ -860,694 +841,5 @@ /* 33 bit reflected Barrett constant n */ .octa 0x00000000000000000000000105ec76f1 - .text - -#if defined(__BIG_ENDIAN__) -#define BYTESWAP_DATA -#else -#undef BYTESWAP_DATA -#endif - -#define off16 r25 -#define off32 r26 -#define off48 r27 -#define off64 r28 -#define off80 r29 -#define off96 r30 -#define off112 r31 - -#define const1 v24 -#define const2 v25 - -#define byteswap v26 -#define mask_32bit v27 -#define mask_64bit v28 -#define zeroes v29 - -#ifdef BYTESWAP_DATA -#define VPERM(A, B, C, D) vperm A, B, C, D -#else -#define VPERM(A, B, C, D) -#endif - -/* unsigned int __crc32c_vpmsum(unsigned int crc, void *p, unsigned long len) */ -FUNC_START(__crc32c_vpmsum) - std r31,-8(r1) - std r30,-16(r1) - std r29,-24(r1) - std r28,-32(r1) - std r27,-40(r1) - std r26,-48(r1) - std r25,-56(r1) - - li off16,16 - li off32,32 - li off48,48 - li off64,64 - li off80,80 - li off96,96 - li off112,112 - li r0,0 - - /* Enough room for saving 10 non volatile VMX registers */ - subi r6,r1,56+10*16 - subi r7,r1,56+2*16 - - stvx v20,0,r6 - stvx v21,off16,r6 - stvx v22,off32,r6 - stvx v23,off48,r6 - stvx v24,off64,r6 - stvx v25,off80,r6 - stvx v26,off96,r6 - stvx v27,off112,r6 - stvx v28,0,r7 - stvx v29,off16,r7 - - mr r10,r3 - - vxor zeroes,zeroes,zeroes - vspltisw v0,-1 - - vsldoi mask_32bit,zeroes,v0,4 - vsldoi mask_64bit,zeroes,v0,8 - - /* Get the initial value into v8 */ - vxor v8,v8,v8 - MTVRD(v8, R3) - vsldoi v8,zeroes,v8,8 /* shift into bottom 32 bits */ - -#ifdef BYTESWAP_DATA - addis r3,r2,.byteswap_constant@toc@ha - addi r3,r3,.byteswap_constant@toc@l - - lvx byteswap,0,r3 - addi r3,r3,16 -#endif - - cmpdi r5,256 - blt .Lshort - - rldicr r6,r5,0,56 - - /* Checksum in blocks of MAX_SIZE */ -1: lis r7,MAX_SIZE@h - ori r7,r7,MAX_SIZE@l - mr r9,r7 - cmpd r6,r7 - bgt 2f - mr r7,r6 -2: subf r6,r7,r6 - - /* our main loop does 128 bytes at a time */ - srdi r7,r7,7 - - /* - * Work out the offset into the constants table to start at. Each - * constant is 16 bytes, and it is used against 128 bytes of input - * data - 128 / 16 = 8 - */ - sldi r8,r7,4 - srdi r9,r9,3 - subf r8,r8,r9 - - /* We reduce our final 128 bytes in a separate step */ - addi r7,r7,-1 - mtctr r7 - - addis r3,r2,.constants@toc@ha - addi r3,r3,.constants@toc@l - - /* Find the start of our constants */ - add r3,r3,r8 - - /* zero v0-v7 which will contain our checksums */ - vxor v0,v0,v0 - vxor v1,v1,v1 - vxor v2,v2,v2 - vxor v3,v3,v3 - vxor v4,v4,v4 - vxor v5,v5,v5 - vxor v6,v6,v6 - vxor v7,v7,v7 - - lvx const1,0,r3 - - /* - * If we are looping back to consume more data we use the values - * already in v16-v23. - */ - cmpdi r0,1 - beq 2f - - /* First warm up pass */ - lvx v16,0,r4 - lvx v17,off16,r4 - VPERM(v16,v16,v16,byteswap) - VPERM(v17,v17,v17,byteswap) - lvx v18,off32,r4 - lvx v19,off48,r4 - VPERM(v18,v18,v18,byteswap) - VPERM(v19,v19,v19,byteswap) - lvx v20,off64,r4 - lvx v21,off80,r4 - VPERM(v20,v20,v20,byteswap) - VPERM(v21,v21,v21,byteswap) - lvx v22,off96,r4 - lvx v23,off112,r4 - VPERM(v22,v22,v22,byteswap) - VPERM(v23,v23,v23,byteswap) - addi r4,r4,8*16 - - /* xor in initial value */ - vxor v16,v16,v8 - -2: bdz .Lfirst_warm_up_done - - addi r3,r3,16 - lvx const2,0,r3 - - /* Second warm up pass */ - VPMSUMD(v8,v16,const1) - lvx v16,0,r4 - VPERM(v16,v16,v16,byteswap) - ori r2,r2,0 - - VPMSUMD(v9,v17,const1) - lvx v17,off16,r4 - VPERM(v17,v17,v17,byteswap) - ori r2,r2,0 - - VPMSUMD(v10,v18,const1) - lvx v18,off32,r4 - VPERM(v18,v18,v18,byteswap) - ori r2,r2,0 - - VPMSUMD(v11,v19,const1) - lvx v19,off48,r4 - VPERM(v19,v19,v19,byteswap) - ori r2,r2,0 - - VPMSUMD(v12,v20,const1) - lvx v20,off64,r4 - VPERM(v20,v20,v20,byteswap) - ori r2,r2,0 - - VPMSUMD(v13,v21,const1) - lvx v21,off80,r4 - VPERM(v21,v21,v21,byteswap) - ori r2,r2,0 - - VPMSUMD(v14,v22,const1) - lvx v22,off96,r4 - VPERM(v22,v22,v22,byteswap) - ori r2,r2,0 - - VPMSUMD(v15,v23,const1) - lvx v23,off112,r4 - VPERM(v23,v23,v23,byteswap) - - addi r4,r4,8*16 - - bdz .Lfirst_cool_down - - /* - * main loop. We modulo schedule it such that it takes three iterations - * to complete - first iteration load, second iteration vpmsum, third - * iteration xor. - */ - .balign 16 -4: lvx const1,0,r3 - addi r3,r3,16 - ori r2,r2,0 - - vxor v0,v0,v8 - VPMSUMD(v8,v16,const2) - lvx v16,0,r4 - VPERM(v16,v16,v16,byteswap) - ori r2,r2,0 - - vxor v1,v1,v9 - VPMSUMD(v9,v17,const2) - lvx v17,off16,r4 - VPERM(v17,v17,v17,byteswap) - ori r2,r2,0 - - vxor v2,v2,v10 - VPMSUMD(v10,v18,const2) - lvx v18,off32,r4 - VPERM(v18,v18,v18,byteswap) - ori r2,r2,0 - - vxor v3,v3,v11 - VPMSUMD(v11,v19,const2) - lvx v19,off48,r4 - VPERM(v19,v19,v19,byteswap) - lvx const2,0,r3 - ori r2,r2,0 - - vxor v4,v4,v12 - VPMSUMD(v12,v20,const1) - lvx v20,off64,r4 - VPERM(v20,v20,v20,byteswap) - ori r2,r2,0 - - vxor v5,v5,v13 - VPMSUMD(v13,v21,const1) - lvx v21,off80,r4 - VPERM(v21,v21,v21,byteswap) - ori r2,r2,0 - - vxor v6,v6,v14 - VPMSUMD(v14,v22,const1) - lvx v22,off96,r4 - VPERM(v22,v22,v22,byteswap) - ori r2,r2,0 - - vxor v7,v7,v15 - VPMSUMD(v15,v23,const1) - lvx v23,off112,r4 - VPERM(v23,v23,v23,byteswap) - - addi r4,r4,8*16 - - bdnz 4b - -.Lfirst_cool_down: - /* First cool down pass */ - lvx const1,0,r3 - addi r3,r3,16 - - vxor v0,v0,v8 - VPMSUMD(v8,v16,const1) - ori r2,r2,0 - - vxor v1,v1,v9 - VPMSUMD(v9,v17,const1) - ori r2,r2,0 - - vxor v2,v2,v10 - VPMSUMD(v10,v18,const1) - ori r2,r2,0 - - vxor v3,v3,v11 - VPMSUMD(v11,v19,const1) - ori r2,r2,0 - - vxor v4,v4,v12 - VPMSUMD(v12,v20,const1) - ori r2,r2,0 - - vxor v5,v5,v13 - VPMSUMD(v13,v21,const1) - ori r2,r2,0 - - vxor v6,v6,v14 - VPMSUMD(v14,v22,const1) - ori r2,r2,0 - - vxor v7,v7,v15 - VPMSUMD(v15,v23,const1) - ori r2,r2,0 - -.Lsecond_cool_down: - /* Second cool down pass */ - vxor v0,v0,v8 - vxor v1,v1,v9 - vxor v2,v2,v10 - vxor v3,v3,v11 - vxor v4,v4,v12 - vxor v5,v5,v13 - vxor v6,v6,v14 - vxor v7,v7,v15 - - /* - * vpmsumd produces a 96 bit result in the least significant bits - * of the register. Since we are bit reflected we have to shift it - * left 32 bits so it occupies the least significant bits in the - * bit reflected domain. - */ - vsldoi v0,v0,zeroes,4 - vsldoi v1,v1,zeroes,4 - vsldoi v2,v2,zeroes,4 - vsldoi v3,v3,zeroes,4 - vsldoi v4,v4,zeroes,4 - vsldoi v5,v5,zeroes,4 - vsldoi v6,v6,zeroes,4 - vsldoi v7,v7,zeroes,4 - - /* xor with last 1024 bits */ - lvx v8,0,r4 - lvx v9,off16,r4 - VPERM(v8,v8,v8,byteswap) - VPERM(v9,v9,v9,byteswap) - lvx v10,off32,r4 - lvx v11,off48,r4 - VPERM(v10,v10,v10,byteswap) - VPERM(v11,v11,v11,byteswap) - lvx v12,off64,r4 - lvx v13,off80,r4 - VPERM(v12,v12,v12,byteswap) - VPERM(v13,v13,v13,byteswap) - lvx v14,off96,r4 - lvx v15,off112,r4 - VPERM(v14,v14,v14,byteswap) - VPERM(v15,v15,v15,byteswap) - - addi r4,r4,8*16 - - vxor v16,v0,v8 - vxor v17,v1,v9 - vxor v18,v2,v10 - vxor v19,v3,v11 - vxor v20,v4,v12 - vxor v21,v5,v13 - vxor v22,v6,v14 - vxor v23,v7,v15 - - li r0,1 - cmpdi r6,0 - addi r6,r6,128 - bne 1b - - /* Work out how many bytes we have left */ - andi. r5,r5,127 - - /* Calculate where in the constant table we need to start */ - subfic r6,r5,128 - add r3,r3,r6 - - /* How many 16 byte chunks are in the tail */ - srdi r7,r5,4 - mtctr r7 - - /* - * Reduce the previously calculated 1024 bits to 64 bits, shifting - * 32 bits to include the trailing 32 bits of zeros - */ - lvx v0,0,r3 - lvx v1,off16,r3 - lvx v2,off32,r3 - lvx v3,off48,r3 - lvx v4,off64,r3 - lvx v5,off80,r3 - lvx v6,off96,r3 - lvx v7,off112,r3 - addi r3,r3,8*16 - - VPMSUMW(v0,v16,v0) - VPMSUMW(v1,v17,v1) - VPMSUMW(v2,v18,v2) - VPMSUMW(v3,v19,v3) - VPMSUMW(v4,v20,v4) - VPMSUMW(v5,v21,v5) - VPMSUMW(v6,v22,v6) - VPMSUMW(v7,v23,v7) - - /* Now reduce the tail (0 - 112 bytes) */ - cmpdi r7,0 - beq 1f - - lvx v16,0,r4 - lvx v17,0,r3 - VPERM(v16,v16,v16,byteswap) - VPMSUMW(v16,v16,v17) - vxor v0,v0,v16 - bdz 1f - - lvx v16,off16,r4 - lvx v17,off16,r3 - VPERM(v16,v16,v16,byteswap) - VPMSUMW(v16,v16,v17) - vxor v0,v0,v16 - bdz 1f - - lvx v16,off32,r4 - lvx v17,off32,r3 - VPERM(v16,v16,v16,byteswap) - VPMSUMW(v16,v16,v17) - vxor v0,v0,v16 - bdz 1f - - lvx v16,off48,r4 - lvx v17,off48,r3 - VPERM(v16,v16,v16,byteswap) - VPMSUMW(v16,v16,v17) - vxor v0,v0,v16 - bdz 1f - - lvx v16,off64,r4 - lvx v17,off64,r3 - VPERM(v16,v16,v16,byteswap) - VPMSUMW(v16,v16,v17) - vxor v0,v0,v16 - bdz 1f - - lvx v16,off80,r4 - lvx v17,off80,r3 - VPERM(v16,v16,v16,byteswap) - VPMSUMW(v16,v16,v17) - vxor v0,v0,v16 - bdz 1f - - lvx v16,off96,r4 - lvx v17,off96,r3 - VPERM(v16,v16,v16,byteswap) - VPMSUMW(v16,v16,v17) - vxor v0,v0,v16 - - /* Now xor all the parallel chunks together */ -1: vxor v0,v0,v1 - vxor v2,v2,v3 - vxor v4,v4,v5 - vxor v6,v6,v7 - - vxor v0,v0,v2 - vxor v4,v4,v6 - - vxor v0,v0,v4 - -.Lbarrett_reduction: - /* Barrett constants */ - addis r3,r2,.barrett_constants@toc@ha - addi r3,r3,.barrett_constants@toc@l - - lvx const1,0,r3 - lvx const2,off16,r3 - - vsldoi v1,v0,v0,8 - vxor v0,v0,v1 /* xor two 64 bit results together */ - - /* shift left one bit */ - vspltisb v1,1 - vsl v0,v0,v1 - - vand v0,v0,mask_64bit - - /* - * The reflected version of Barrett reduction. Instead of bit - * reflecting our data (which is expensive to do), we bit reflect our - * constants and our algorithm, which means the intermediate data in - * our vector registers goes from 0-63 instead of 63-0. We can reflect - * the algorithm because we don't carry in mod 2 arithmetic. - */ - vand v1,v0,mask_32bit /* bottom 32 bits of a */ - VPMSUMD(v1,v1,const1) /* ma */ - vand v1,v1,mask_32bit /* bottom 32bits of ma */ - VPMSUMD(v1,v1,const2) /* qn */ - vxor v0,v0,v1 /* a - qn, subtraction is xor in GF(2) */ - - /* - * Since we are bit reflected, the result (ie the low 32 bits) is in - * the high 32 bits. We just need to shift it left 4 bytes - * V0 [ 0 1 X 3 ] - * V0 [ 0 X 2 3 ] - */ - vsldoi v0,v0,zeroes,4 /* shift result into top 64 bits of */ - - /* Get it into r3 */ - MFVRD(R3, v0) - -.Lout: - subi r6,r1,56+10*16 - subi r7,r1,56+2*16 - - lvx v20,0,r6 - lvx v21,off16,r6 - lvx v22,off32,r6 - lvx v23,off48,r6 - lvx v24,off64,r6 - lvx v25,off80,r6 - lvx v26,off96,r6 - lvx v27,off112,r6 - lvx v28,0,r7 - lvx v29,off16,r7 - - ld r31,-8(r1) - ld r30,-16(r1) - ld r29,-24(r1) - ld r28,-32(r1) - ld r27,-40(r1) - ld r26,-48(r1) - ld r25,-56(r1) - - blr - -.Lfirst_warm_up_done: - lvx const1,0,r3 - addi r3,r3,16 - - VPMSUMD(v8,v16,const1) - VPMSUMD(v9,v17,const1) - VPMSUMD(v10,v18,const1) - VPMSUMD(v11,v19,const1) - VPMSUMD(v12,v20,const1) - VPMSUMD(v13,v21,const1) - VPMSUMD(v14,v22,const1) - VPMSUMD(v15,v23,const1) - - b .Lsecond_cool_down - -.Lshort: - cmpdi r5,0 - beq .Lzero - - addis r3,r2,.short_constants@toc@ha - addi r3,r3,.short_constants@toc@l - - /* Calculate where in the constant table we need to start */ - subfic r6,r5,256 - add r3,r3,r6 - - /* How many 16 byte chunks? */ - srdi r7,r5,4 - mtctr r7 - - vxor v19,v19,v19 - vxor v20,v20,v20 - - lvx v0,0,r4 - lvx v16,0,r3 - VPERM(v0,v0,v16,byteswap) - vxor v0,v0,v8 /* xor in initial value */ - VPMSUMW(v0,v0,v16) - bdz .Lv0 - - lvx v1,off16,r4 - lvx v17,off16,r3 - VPERM(v1,v1,v17,byteswap) - VPMSUMW(v1,v1,v17) - bdz .Lv1 - - lvx v2,off32,r4 - lvx v16,off32,r3 - VPERM(v2,v2,v16,byteswap) - VPMSUMW(v2,v2,v16) - bdz .Lv2 - - lvx v3,off48,r4 - lvx v17,off48,r3 - VPERM(v3,v3,v17,byteswap) - VPMSUMW(v3,v3,v17) - bdz .Lv3 - - lvx v4,off64,r4 - lvx v16,off64,r3 - VPERM(v4,v4,v16,byteswap) - VPMSUMW(v4,v4,v16) - bdz .Lv4 - - lvx v5,off80,r4 - lvx v17,off80,r3 - VPERM(v5,v5,v17,byteswap) - VPMSUMW(v5,v5,v17) - bdz .Lv5 - - lvx v6,off96,r4 - lvx v16,off96,r3 - VPERM(v6,v6,v16,byteswap) - VPMSUMW(v6,v6,v16) - bdz .Lv6 - - lvx v7,off112,r4 - lvx v17,off112,r3 - VPERM(v7,v7,v17,byteswap) - VPMSUMW(v7,v7,v17) - bdz .Lv7 - - addi r3,r3,128 - addi r4,r4,128 - - lvx v8,0,r4 - lvx v16,0,r3 - VPERM(v8,v8,v16,byteswap) - VPMSUMW(v8,v8,v16) - bdz .Lv8 - - lvx v9,off16,r4 - lvx v17,off16,r3 - VPERM(v9,v9,v17,byteswap) - VPMSUMW(v9,v9,v17) - bdz .Lv9 - - lvx v10,off32,r4 - lvx v16,off32,r3 - VPERM(v10,v10,v16,byteswap) - VPMSUMW(v10,v10,v16) - bdz .Lv10 - - lvx v11,off48,r4 - lvx v17,off48,r3 - VPERM(v11,v11,v17,byteswap) - VPMSUMW(v11,v11,v17) - bdz .Lv11 - - lvx v12,off64,r4 - lvx v16,off64,r3 - VPERM(v12,v12,v16,byteswap) - VPMSUMW(v12,v12,v16) - bdz .Lv12 - - lvx v13,off80,r4 - lvx v17,off80,r3 - VPERM(v13,v13,v17,byteswap) - VPMSUMW(v13,v13,v17) - bdz .Lv13 - - lvx v14,off96,r4 - lvx v16,off96,r3 - VPERM(v14,v14,v16,byteswap) - VPMSUMW(v14,v14,v16) - bdz .Lv14 - - lvx v15,off112,r4 - lvx v17,off112,r3 - VPERM(v15,v15,v17,byteswap) - VPMSUMW(v15,v15,v17) - -.Lv15: vxor v19,v19,v15 -.Lv14: vxor v20,v20,v14 -.Lv13: vxor v19,v19,v13 -.Lv12: vxor v20,v20,v12 -.Lv11: vxor v19,v19,v11 -.Lv10: vxor v20,v20,v10 -.Lv9: vxor v19,v19,v9 -.Lv8: vxor v20,v20,v8 -.Lv7: vxor v19,v19,v7 -.Lv6: vxor v20,v20,v6 -.Lv5: vxor v19,v19,v5 -.Lv4: vxor v20,v20,v4 -.Lv3: vxor v19,v19,v3 -.Lv2: vxor v20,v20,v2 -.Lv1: vxor v19,v19,v1 -.Lv0: vxor v20,v20,v0 - - vxor v0,v19,v20 - - b .Lbarrett_reduction - -.Lzero: - mr r3,r10 - b .Lout - -FUNC_END(__crc32_vpmsum) +#define CRC_FUNCTION_NAME __crc32c_vpmsum +#include "crc32-vpmsum_core.S" From 08c7dd1bd4b3a59c0d57b3ce4fe5ba1a90123685 Mon Sep 17 00:00:00 2001 From: Daniel Axtens Date: Wed, 15 Mar 2017 23:37:35 +1100 Subject: [PATCH 031/101] crypto: powerpc - Re-enable non-REFLECTed CRCs When CRC32c was included in the kernel, Anton ripped out the #ifdefs around reflected polynomials, because CRC32c is always reflected. However, not all CRCs use reflection so we'd like to make it optional. Restore the REFLECT parts from Anton's original CRC32 implementation (https://github.com/antonblanchard/crc32-vpmsum) That implementation is available under GPLv2+, so we're OK from a licensing point of view: https://github.com/antonblanchard/crc32-vpmsum/blob/master/LICENSE.TXT As CRC32c requires REFLECT, add that #define. Cc: Anton Blanchard Signed-off-by: Daniel Axtens Signed-off-by: Herbert Xu --- arch/powerpc/crypto/crc32-vpmsum_core.S | 31 ++++++++++++++++++++++++- arch/powerpc/crypto/crc32c-vpmsum_asm.S | 1 + 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/arch/powerpc/crypto/crc32-vpmsum_core.S b/arch/powerpc/crypto/crc32-vpmsum_core.S index 7c6be6a5c977..aadb59c96a27 100644 --- a/arch/powerpc/crypto/crc32-vpmsum_core.S +++ b/arch/powerpc/crypto/crc32-vpmsum_core.S @@ -35,7 +35,9 @@ .text -#if defined(__BIG_ENDIAN__) +#if defined(__BIG_ENDIAN__) && defined(REFLECT) +#define BYTESWAP_DATA +#elif defined(__LITTLE_ENDIAN__) && !defined(REFLECT) #define BYTESWAP_DATA #else #undef BYTESWAP_DATA @@ -108,7 +110,11 @@ FUNC_START(CRC_FUNCTION_NAME) /* Get the initial value into v8 */ vxor v8,v8,v8 MTVRD(v8, R3) +#ifdef REFLECT vsldoi v8,zeroes,v8,8 /* shift into bottom 32 bits */ +#else + vsldoi v8,v8,zeroes,4 /* shift into top 32 bits */ +#endif #ifdef BYTESWAP_DATA addis r3,r2,.byteswap_constant@toc@ha @@ -354,6 +360,7 @@ FUNC_START(CRC_FUNCTION_NAME) vxor v6,v6,v14 vxor v7,v7,v15 +#ifdef REFLECT /* * vpmsumd produces a 96 bit result in the least significant bits * of the register. Since we are bit reflected we have to shift it @@ -368,6 +375,7 @@ FUNC_START(CRC_FUNCTION_NAME) vsldoi v5,v5,zeroes,4 vsldoi v6,v6,zeroes,4 vsldoi v7,v7,zeroes,4 +#endif /* xor with last 1024 bits */ lvx v8,0,r4 @@ -511,12 +519,32 @@ FUNC_START(CRC_FUNCTION_NAME) vsldoi v1,v0,v0,8 vxor v0,v0,v1 /* xor two 64 bit results together */ +#ifdef REFLECT /* shift left one bit */ vspltisb v1,1 vsl v0,v0,v1 +#endif vand v0,v0,mask_64bit +#ifndef REFLECT + /* + * Now for the Barrett reduction algorithm. The idea is to calculate q, + * the multiple of our polynomial that we need to subtract. By + * doing the computation 2x bits higher (ie 64 bits) and shifting the + * result back down 2x bits, we round down to the nearest multiple. + */ + VPMSUMD(v1,v0,const1) /* ma */ + vsldoi v1,zeroes,v1,8 /* q = floor(ma/(2^64)) */ + VPMSUMD(v1,v1,const2) /* qn */ + vxor v0,v0,v1 /* a - qn, subtraction is xor in GF(2) */ + /* + * Get the result into r3. We need to shift it left 8 bytes: + * V0 [ 0 1 2 X ] + * V0 [ 0 X 2 3 ] + */ + vsldoi v0,v0,zeroes,8 /* shift result into top 64 bits */ +#else /* * The reflected version of Barrett reduction. Instead of bit * reflecting our data (which is expensive to do), we bit reflect our @@ -537,6 +565,7 @@ FUNC_START(CRC_FUNCTION_NAME) * V0 [ 0 X 2 3 ] */ vsldoi v0,v0,zeroes,4 /* shift result into top 64 bits of */ +#endif /* Get it into r3 */ MFVRD(R3, v0) diff --git a/arch/powerpc/crypto/crc32c-vpmsum_asm.S b/arch/powerpc/crypto/crc32c-vpmsum_asm.S index c0d080caefc1..d2bea48051a0 100644 --- a/arch/powerpc/crypto/crc32c-vpmsum_asm.S +++ b/arch/powerpc/crypto/crc32c-vpmsum_asm.S @@ -842,4 +842,5 @@ .octa 0x00000000000000000000000105ec76f1 #define CRC_FUNCTION_NAME __crc32c_vpmsum +#define REFLECT #include "crc32-vpmsum_core.S" From b01df1c16c9a6f7a14f843d3ac6b9eef5a7bb17e Mon Sep 17 00:00:00 2001 From: Daniel Axtens Date: Wed, 15 Mar 2017 23:37:36 +1100 Subject: [PATCH 032/101] crypto: powerpc - Add CRC-T10DIF acceleration T10DIF is a CRC16 used heavily in NVMe. It turns out we can accelerate it with a CRC32 library and a few little tricks. Provide the accelerator based the refactored CRC32 code. Cc: Anton Blanchard Thanks-to: Hong Bo Peng Signed-off-by: Daniel Axtens Signed-off-by: Herbert Xu --- arch/powerpc/crypto/Makefile | 2 + arch/powerpc/crypto/crct10dif-vpmsum_asm.S | 850 ++++++++++++++++++++ arch/powerpc/crypto/crct10dif-vpmsum_glue.c | 125 +++ crypto/Kconfig | 9 + 4 files changed, 986 insertions(+) create mode 100644 arch/powerpc/crypto/crct10dif-vpmsum_asm.S create mode 100644 arch/powerpc/crypto/crct10dif-vpmsum_glue.c diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile index 87f40454bad3..e66aaf19764d 100644 --- a/arch/powerpc/crypto/Makefile +++ b/arch/powerpc/crypto/Makefile @@ -10,6 +10,7 @@ obj-$(CONFIG_CRYPTO_SHA1_PPC) += sha1-powerpc.o obj-$(CONFIG_CRYPTO_SHA1_PPC_SPE) += sha1-ppc-spe.o obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o obj-$(CONFIG_CRYPTO_CRC32C_VPMSUM) += crc32c-vpmsum.o +obj-$(CONFIG_CRYPTO_CRCT10DIF_VPMSUM) += crct10dif-vpmsum.o aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes-spe-glue.o md5-ppc-y := md5-asm.o md5-glue.o @@ -17,3 +18,4 @@ sha1-powerpc-y := sha1-powerpc-asm.o sha1.o sha1-ppc-spe-y := sha1-spe-asm.o sha1-spe-glue.o sha256-ppc-spe-y := sha256-spe-asm.o sha256-spe-glue.o crc32c-vpmsum-y := crc32c-vpmsum_asm.o crc32c-vpmsum_glue.o +crct10dif-vpmsum-y := crct10dif-vpmsum_asm.o crct10dif-vpmsum_glue.o diff --git a/arch/powerpc/crypto/crct10dif-vpmsum_asm.S b/arch/powerpc/crypto/crct10dif-vpmsum_asm.S new file mode 100644 index 000000000000..5e3d81a0af1b --- /dev/null +++ b/arch/powerpc/crypto/crct10dif-vpmsum_asm.S @@ -0,0 +1,850 @@ +/* + * Calculate a CRC T10DIF with vpmsum acceleration + * + * Constants generated by crc32-vpmsum, available at + * https://github.com/antonblanchard/crc32-vpmsum + * + * crc32-vpmsum is + * Copyright (C) 2015 Anton Blanchard , IBM + * and is available under the GPL v2 or later. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ + .section .rodata +.balign 16 + +.byteswap_constant: + /* byte reverse permute constant */ + .octa 0x0F0E0D0C0B0A09080706050403020100 + +.constants: + + /* Reduce 262144 kbits to 1024 bits */ + /* x^261184 mod p(x), x^261120 mod p(x) */ + .octa 0x0000000056d300000000000052550000 + + /* x^260160 mod p(x), x^260096 mod p(x) */ + .octa 0x00000000ee67000000000000a1e40000 + + /* x^259136 mod p(x), x^259072 mod p(x) */ + .octa 0x0000000060830000000000004ad10000 + + /* x^258112 mod p(x), x^258048 mod p(x) */ + .octa 0x000000008cfe0000000000009ab40000 + + /* x^257088 mod p(x), x^257024 mod p(x) */ + .octa 0x000000003e93000000000000fdb50000 + + /* x^256064 mod p(x), x^256000 mod p(x) */ + .octa 0x000000003c2000000000000045480000 + + /* x^255040 mod p(x), x^254976 mod p(x) */ + .octa 0x00000000b1fc0000000000008d690000 + + /* x^254016 mod p(x), x^253952 mod p(x) */ + .octa 0x00000000f82b00000000000024ad0000 + + /* x^252992 mod p(x), x^252928 mod p(x) */ + .octa 0x0000000044420000000000009f1a0000 + + /* x^251968 mod p(x), x^251904 mod p(x) */ + .octa 0x00000000e88c00000000000066ec0000 + + /* x^250944 mod p(x), x^250880 mod p(x) */ + .octa 0x00000000385c000000000000c87d0000 + + /* x^249920 mod p(x), x^249856 mod p(x) */ + .octa 0x000000003227000000000000c8ff0000 + + /* x^248896 mod p(x), x^248832 mod p(x) */ + .octa 0x00000000a9a900000000000033440000 + + /* x^247872 mod p(x), x^247808 mod p(x) */ + .octa 0x00000000abaa00000000000066eb0000 + + /* x^246848 mod p(x), x^246784 mod p(x) */ + .octa 0x000000001ac3000000000000c4ef0000 + + /* x^245824 mod p(x), x^245760 mod p(x) */ + .octa 0x0000000063f000000000000056f30000 + + /* x^244800 mod p(x), x^244736 mod p(x) */ + .octa 0x0000000032cc00000000000002050000 + + /* x^243776 mod p(x), x^243712 mod p(x) */ + .octa 0x00000000f8b5000000000000568e0000 + + /* x^242752 mod p(x), x^242688 mod p(x) */ + .octa 0x000000008db100000000000064290000 + + /* x^241728 mod p(x), x^241664 mod p(x) */ + .octa 0x0000000059ca0000000000006b660000 + + /* x^240704 mod p(x), x^240640 mod p(x) */ + .octa 0x000000005f5c00000000000018f80000 + + /* x^239680 mod p(x), x^239616 mod p(x) */ + .octa 0x0000000061af000000000000b6090000 + + /* x^238656 mod p(x), x^238592 mod p(x) */ + .octa 0x00000000e29e000000000000099a0000 + + /* x^237632 mod p(x), x^237568 mod p(x) */ + .octa 0x000000000975000000000000a8360000 + + /* x^236608 mod p(x), x^236544 mod p(x) */ + .octa 0x0000000043900000000000004f570000 + + /* x^235584 mod p(x), x^235520 mod p(x) */ + .octa 0x00000000f9cd000000000000134c0000 + + /* x^234560 mod p(x), x^234496 mod p(x) */ + .octa 0x000000007c29000000000000ec380000 + + /* x^233536 mod p(x), x^233472 mod p(x) */ + .octa 0x000000004c6a000000000000b0d10000 + + /* x^232512 mod p(x), x^232448 mod p(x) */ + .octa 0x00000000e7290000000000007d3e0000 + + /* x^231488 mod p(x), x^231424 mod p(x) */ + .octa 0x00000000f1ab000000000000f0b20000 + + /* x^230464 mod p(x), x^230400 mod p(x) */ + .octa 0x0000000039db0000000000009c270000 + + /* x^229440 mod p(x), x^229376 mod p(x) */ + .octa 0x000000005e2800000000000092890000 + + /* x^228416 mod p(x), x^228352 mod p(x) */ + .octa 0x00000000d44e000000000000d5ee0000 + + /* x^227392 mod p(x), x^227328 mod p(x) */ + .octa 0x00000000cd0a00000000000041f50000 + + /* x^226368 mod p(x), x^226304 mod p(x) */ + .octa 0x00000000c5b400000000000010520000 + + /* x^225344 mod p(x), x^225280 mod p(x) */ + .octa 0x00000000fd2100000000000042170000 + + /* x^224320 mod p(x), x^224256 mod p(x) */ + .octa 0x000000002f2500000000000095c20000 + + /* x^223296 mod p(x), x^223232 mod p(x) */ + .octa 0x000000001b0100000000000001ce0000 + + /* x^222272 mod p(x), x^222208 mod p(x) */ + .octa 0x000000000d430000000000002aca0000 + + /* x^221248 mod p(x), x^221184 mod p(x) */ + .octa 0x0000000030a6000000000000385e0000 + + /* x^220224 mod p(x), x^220160 mod p(x) */ + .octa 0x00000000e37b0000000000006f7a0000 + + /* x^219200 mod p(x), x^219136 mod p(x) */ + .octa 0x00000000873600000000000024320000 + + /* x^218176 mod p(x), x^218112 mod p(x) */ + .octa 0x00000000e9fb000000000000bd9c0000 + + /* x^217152 mod p(x), x^217088 mod p(x) */ + .octa 0x000000003b9500000000000054bc0000 + + /* x^216128 mod p(x), x^216064 mod p(x) */ + .octa 0x00000000133e000000000000a4660000 + + /* x^215104 mod p(x), x^215040 mod p(x) */ + .octa 0x00000000784500000000000079930000 + + /* x^214080 mod p(x), x^214016 mod p(x) */ + .octa 0x00000000b9800000000000001bb80000 + + /* x^213056 mod p(x), x^212992 mod p(x) */ + .octa 0x00000000687600000000000024400000 + + /* x^212032 mod p(x), x^211968 mod p(x) */ + .octa 0x00000000aff300000000000029e10000 + + /* x^211008 mod p(x), x^210944 mod p(x) */ + .octa 0x0000000024b50000000000005ded0000 + + /* x^209984 mod p(x), x^209920 mod p(x) */ + .octa 0x0000000017e8000000000000b12e0000 + + /* x^208960 mod p(x), x^208896 mod p(x) */ + .octa 0x00000000128400000000000026d20000 + + /* x^207936 mod p(x), x^207872 mod p(x) */ + .octa 0x000000002115000000000000a32a0000 + + /* x^206912 mod p(x), x^206848 mod p(x) */ + .octa 0x000000009595000000000000a1210000 + + /* x^205888 mod p(x), x^205824 mod p(x) */ + .octa 0x00000000281e000000000000ee8b0000 + + /* x^204864 mod p(x), x^204800 mod p(x) */ + .octa 0x0000000006010000000000003d0d0000 + + /* x^203840 mod p(x), x^203776 mod p(x) */ + .octa 0x00000000e2b600000000000034e90000 + + /* x^202816 mod p(x), x^202752 mod p(x) */ + .octa 0x000000001bd40000000000004cdb0000 + + /* x^201792 mod p(x), x^201728 mod p(x) */ + .octa 0x00000000df2800000000000030e90000 + + /* x^200768 mod p(x), x^200704 mod p(x) */ + .octa 0x0000000049c200000000000042590000 + + /* x^199744 mod p(x), x^199680 mod p(x) */ + .octa 0x000000009b97000000000000df950000 + + /* x^198720 mod p(x), x^198656 mod p(x) */ + .octa 0x000000006184000000000000da7b0000 + + /* x^197696 mod p(x), x^197632 mod p(x) */ + .octa 0x00000000461700000000000012510000 + + /* x^196672 mod p(x), x^196608 mod p(x) */ + .octa 0x000000009b40000000000000f37e0000 + + /* x^195648 mod p(x), x^195584 mod p(x) */ + .octa 0x00000000eeb2000000000000ecf10000 + + /* x^194624 mod p(x), x^194560 mod p(x) */ + .octa 0x00000000b2e800000000000050f20000 + + /* x^193600 mod p(x), x^193536 mod p(x) */ + .octa 0x00000000f59a000000000000e0b30000 + + /* x^192576 mod p(x), x^192512 mod p(x) */ + .octa 0x00000000467f0000000000004d5a0000 + + /* x^191552 mod p(x), x^191488 mod p(x) */ + .octa 0x00000000da92000000000000bb010000 + + /* x^190528 mod p(x), x^190464 mod p(x) */ + .octa 0x000000001e1000000000000022a40000 + + /* x^189504 mod p(x), x^189440 mod p(x) */ + .octa 0x0000000058fe000000000000836f0000 + + /* x^188480 mod p(x), x^188416 mod p(x) */ + .octa 0x00000000b9ce000000000000d78d0000 + + /* x^187456 mod p(x), x^187392 mod p(x) */ + .octa 0x0000000022210000000000004f8d0000 + + /* x^186432 mod p(x), x^186368 mod p(x) */ + .octa 0x00000000744600000000000033760000 + + /* x^185408 mod p(x), x^185344 mod p(x) */ + .octa 0x000000001c2e000000000000a1e50000 + + /* x^184384 mod p(x), x^184320 mod p(x) */ + .octa 0x00000000dcc8000000000000a1a40000 + + /* x^183360 mod p(x), x^183296 mod p(x) */ + .octa 0x00000000910f00000000000019a20000 + + /* x^182336 mod p(x), x^182272 mod p(x) */ + .octa 0x0000000055d5000000000000f6ae0000 + + /* x^181312 mod p(x), x^181248 mod p(x) */ + .octa 0x00000000c8ba000000000000a7ac0000 + + /* x^180288 mod p(x), x^180224 mod p(x) */ + .octa 0x0000000031f8000000000000eea20000 + + /* x^179264 mod p(x), x^179200 mod p(x) */ + .octa 0x000000001966000000000000c4d90000 + + /* x^178240 mod p(x), x^178176 mod p(x) */ + .octa 0x00000000b9810000000000002b470000 + + /* x^177216 mod p(x), x^177152 mod p(x) */ + .octa 0x000000008303000000000000f7cf0000 + + /* x^176192 mod p(x), x^176128 mod p(x) */ + .octa 0x000000002ce500000000000035b30000 + + /* x^175168 mod p(x), x^175104 mod p(x) */ + .octa 0x000000002fae0000000000000c7c0000 + + /* x^174144 mod p(x), x^174080 mod p(x) */ + .octa 0x00000000f50c0000000000009edf0000 + + /* x^173120 mod p(x), x^173056 mod p(x) */ + .octa 0x00000000714f00000000000004cd0000 + + /* x^172096 mod p(x), x^172032 mod p(x) */ + .octa 0x00000000c161000000000000541b0000 + + /* x^171072 mod p(x), x^171008 mod p(x) */ + .octa 0x0000000021c8000000000000e2700000 + + /* x^170048 mod p(x), x^169984 mod p(x) */ + .octa 0x00000000b93d00000000000009a60000 + + /* x^169024 mod p(x), x^168960 mod p(x) */ + .octa 0x00000000fbcf000000000000761c0000 + + /* x^168000 mod p(x), x^167936 mod p(x) */ + .octa 0x0000000026350000000000009db30000 + + /* x^166976 mod p(x), x^166912 mod p(x) */ + .octa 0x00000000b64f0000000000003e9f0000 + + /* x^165952 mod p(x), x^165888 mod p(x) */ + .octa 0x00000000bd0e00000000000078590000 + + /* x^164928 mod p(x), x^164864 mod p(x) */ + .octa 0x00000000d9360000000000008bc80000 + + /* x^163904 mod p(x), x^163840 mod p(x) */ + .octa 0x000000002f140000000000008c9f0000 + + /* x^162880 mod p(x), x^162816 mod p(x) */ + .octa 0x000000006a270000000000006af70000 + + /* x^161856 mod p(x), x^161792 mod p(x) */ + .octa 0x000000006685000000000000e5210000 + + /* x^160832 mod p(x), x^160768 mod p(x) */ + .octa 0x0000000062da00000000000008290000 + + /* x^159808 mod p(x), x^159744 mod p(x) */ + .octa 0x00000000bb4b000000000000e4d00000 + + /* x^158784 mod p(x), x^158720 mod p(x) */ + .octa 0x00000000d2490000000000004ae10000 + + /* x^157760 mod p(x), x^157696 mod p(x) */ + .octa 0x00000000c85b00000000000000e70000 + + /* x^156736 mod p(x), x^156672 mod p(x) */ + .octa 0x00000000c37a00000000000015650000 + + /* x^155712 mod p(x), x^155648 mod p(x) */ + .octa 0x0000000018530000000000001c2f0000 + + /* x^154688 mod p(x), x^154624 mod p(x) */ + .octa 0x00000000b46600000000000037bd0000 + + /* x^153664 mod p(x), x^153600 mod p(x) */ + .octa 0x00000000439b00000000000012190000 + + /* x^152640 mod p(x), x^152576 mod p(x) */ + .octa 0x00000000b1260000000000005ece0000 + + /* x^151616 mod p(x), x^151552 mod p(x) */ + .octa 0x00000000d8110000000000002a5e0000 + + /* x^150592 mod p(x), x^150528 mod p(x) */ + .octa 0x00000000099f00000000000052330000 + + /* x^149568 mod p(x), x^149504 mod p(x) */ + .octa 0x00000000f9f9000000000000f9120000 + + /* x^148544 mod p(x), x^148480 mod p(x) */ + .octa 0x000000005cc00000000000000ddc0000 + + /* x^147520 mod p(x), x^147456 mod p(x) */ + .octa 0x00000000343b00000000000012200000 + + /* x^146496 mod p(x), x^146432 mod p(x) */ + .octa 0x000000009222000000000000d12b0000 + + /* x^145472 mod p(x), x^145408 mod p(x) */ + .octa 0x00000000d781000000000000eb2d0000 + + /* x^144448 mod p(x), x^144384 mod p(x) */ + .octa 0x000000000bf400000000000058970000 + + /* x^143424 mod p(x), x^143360 mod p(x) */ + .octa 0x00000000094200000000000013690000 + + /* x^142400 mod p(x), x^142336 mod p(x) */ + .octa 0x00000000d55100000000000051950000 + + /* x^141376 mod p(x), x^141312 mod p(x) */ + .octa 0x000000008f11000000000000954b0000 + + /* x^140352 mod p(x), x^140288 mod p(x) */ + .octa 0x00000000140f000000000000b29e0000 + + /* x^139328 mod p(x), x^139264 mod p(x) */ + .octa 0x00000000c6db000000000000db5d0000 + + /* x^138304 mod p(x), x^138240 mod p(x) */ + .octa 0x00000000715b000000000000dfaf0000 + + /* x^137280 mod p(x), x^137216 mod p(x) */ + .octa 0x000000000dea000000000000e3b60000 + + /* x^136256 mod p(x), x^136192 mod p(x) */ + .octa 0x000000006f94000000000000ddaf0000 + + /* x^135232 mod p(x), x^135168 mod p(x) */ + .octa 0x0000000024e1000000000000e4f70000 + + /* x^134208 mod p(x), x^134144 mod p(x) */ + .octa 0x000000008810000000000000aa110000 + + /* x^133184 mod p(x), x^133120 mod p(x) */ + .octa 0x0000000030c2000000000000a8e60000 + + /* x^132160 mod p(x), x^132096 mod p(x) */ + .octa 0x00000000e6d0000000000000ccf30000 + + /* x^131136 mod p(x), x^131072 mod p(x) */ + .octa 0x000000004da000000000000079bf0000 + + /* x^130112 mod p(x), x^130048 mod p(x) */ + .octa 0x000000007759000000000000b3a30000 + + /* x^129088 mod p(x), x^129024 mod p(x) */ + .octa 0x00000000597400000000000028790000 + + /* x^128064 mod p(x), x^128000 mod p(x) */ + .octa 0x000000007acd000000000000b5820000 + + /* x^127040 mod p(x), x^126976 mod p(x) */ + .octa 0x00000000e6e400000000000026ad0000 + + /* x^126016 mod p(x), x^125952 mod p(x) */ + .octa 0x000000006d49000000000000985b0000 + + /* x^124992 mod p(x), x^124928 mod p(x) */ + .octa 0x000000000f0800000000000011520000 + + /* x^123968 mod p(x), x^123904 mod p(x) */ + .octa 0x000000002c7f000000000000846c0000 + + /* x^122944 mod p(x), x^122880 mod p(x) */ + .octa 0x000000005ce7000000000000ae1d0000 + + /* x^121920 mod p(x), x^121856 mod p(x) */ + .octa 0x00000000d4cb000000000000e21d0000 + + /* x^120896 mod p(x), x^120832 mod p(x) */ + .octa 0x000000003a2300000000000019bb0000 + + /* x^119872 mod p(x), x^119808 mod p(x) */ + .octa 0x000000000e1700000000000095290000 + + /* x^118848 mod p(x), x^118784 mod p(x) */ + .octa 0x000000006e6400000000000050d20000 + + /* x^117824 mod p(x), x^117760 mod p(x) */ + .octa 0x000000008d5c0000000000000cd10000 + + /* x^116800 mod p(x), x^116736 mod p(x) */ + .octa 0x00000000ef310000000000007b570000 + + /* x^115776 mod p(x), x^115712 mod p(x) */ + .octa 0x00000000645d00000000000053d60000 + + /* x^114752 mod p(x), x^114688 mod p(x) */ + .octa 0x0000000018fc00000000000077510000 + + /* x^113728 mod p(x), x^113664 mod p(x) */ + .octa 0x000000000cb3000000000000a7b70000 + + /* x^112704 mod p(x), x^112640 mod p(x) */ + .octa 0x00000000991b000000000000d0780000 + + /* x^111680 mod p(x), x^111616 mod p(x) */ + .octa 0x00000000845a000000000000be3c0000 + + /* x^110656 mod p(x), x^110592 mod p(x) */ + .octa 0x00000000d3a9000000000000df020000 + + /* x^109632 mod p(x), x^109568 mod p(x) */ + .octa 0x0000000017d7000000000000063e0000 + + /* x^108608 mod p(x), x^108544 mod p(x) */ + .octa 0x000000007a860000000000008ab40000 + + /* x^107584 mod p(x), x^107520 mod p(x) */ + .octa 0x00000000fd7c000000000000c7bd0000 + + /* x^106560 mod p(x), x^106496 mod p(x) */ + .octa 0x00000000a56b000000000000efd60000 + + /* x^105536 mod p(x), x^105472 mod p(x) */ + .octa 0x0000000010e400000000000071380000 + + /* x^104512 mod p(x), x^104448 mod p(x) */ + .octa 0x00000000994500000000000004d30000 + + /* x^103488 mod p(x), x^103424 mod p(x) */ + .octa 0x00000000b83c0000000000003b0e0000 + + /* x^102464 mod p(x), x^102400 mod p(x) */ + .octa 0x00000000d6c10000000000008b020000 + + /* x^101440 mod p(x), x^101376 mod p(x) */ + .octa 0x000000009efc000000000000da940000 + + /* x^100416 mod p(x), x^100352 mod p(x) */ + .octa 0x000000005e87000000000000f9f70000 + + /* x^99392 mod p(x), x^99328 mod p(x) */ + .octa 0x000000006c9b00000000000045e40000 + + /* x^98368 mod p(x), x^98304 mod p(x) */ + .octa 0x00000000178a00000000000083940000 + + /* x^97344 mod p(x), x^97280 mod p(x) */ + .octa 0x00000000f0c8000000000000f0a00000 + + /* x^96320 mod p(x), x^96256 mod p(x) */ + .octa 0x00000000f699000000000000b74b0000 + + /* x^95296 mod p(x), x^95232 mod p(x) */ + .octa 0x00000000316d000000000000c1cf0000 + + /* x^94272 mod p(x), x^94208 mod p(x) */ + .octa 0x00000000987e00000000000072680000 + + /* x^93248 mod p(x), x^93184 mod p(x) */ + .octa 0x00000000acff000000000000e0ab0000 + + /* x^92224 mod p(x), x^92160 mod p(x) */ + .octa 0x00000000a1f6000000000000c5a80000 + + /* x^91200 mod p(x), x^91136 mod p(x) */ + .octa 0x0000000061bd000000000000cf690000 + + /* x^90176 mod p(x), x^90112 mod p(x) */ + .octa 0x00000000c9f2000000000000cbcc0000 + + /* x^89152 mod p(x), x^89088 mod p(x) */ + .octa 0x000000005a33000000000000de050000 + + /* x^88128 mod p(x), x^88064 mod p(x) */ + .octa 0x00000000e416000000000000ccd70000 + + /* x^87104 mod p(x), x^87040 mod p(x) */ + .octa 0x0000000058930000000000002f670000 + + /* x^86080 mod p(x), x^86016 mod p(x) */ + .octa 0x00000000a9d3000000000000152f0000 + + /* x^85056 mod p(x), x^84992 mod p(x) */ + .octa 0x00000000c114000000000000ecc20000 + + /* x^84032 mod p(x), x^83968 mod p(x) */ + .octa 0x00000000b9270000000000007c890000 + + /* x^83008 mod p(x), x^82944 mod p(x) */ + .octa 0x000000002e6000000000000006ee0000 + + /* x^81984 mod p(x), x^81920 mod p(x) */ + .octa 0x00000000dfc600000000000009100000 + + /* x^80960 mod p(x), x^80896 mod p(x) */ + .octa 0x000000004911000000000000ad4e0000 + + /* x^79936 mod p(x), x^79872 mod p(x) */ + .octa 0x00000000ae1b000000000000b04d0000 + + /* x^78912 mod p(x), x^78848 mod p(x) */ + .octa 0x0000000005fa000000000000e9900000 + + /* x^77888 mod p(x), x^77824 mod p(x) */ + .octa 0x0000000004a1000000000000cc6f0000 + + /* x^76864 mod p(x), x^76800 mod p(x) */ + .octa 0x00000000af73000000000000ed110000 + + /* x^75840 mod p(x), x^75776 mod p(x) */ + .octa 0x0000000082530000000000008f7e0000 + + /* x^74816 mod p(x), x^74752 mod p(x) */ + .octa 0x00000000cfdc000000000000594f0000 + + /* x^73792 mod p(x), x^73728 mod p(x) */ + .octa 0x00000000a6b6000000000000a8750000 + + /* x^72768 mod p(x), x^72704 mod p(x) */ + .octa 0x00000000fd76000000000000aa0c0000 + + /* x^71744 mod p(x), x^71680 mod p(x) */ + .octa 0x0000000006f500000000000071db0000 + + /* x^70720 mod p(x), x^70656 mod p(x) */ + .octa 0x0000000037ca000000000000ab0c0000 + + /* x^69696 mod p(x), x^69632 mod p(x) */ + .octa 0x00000000d7ab000000000000b7a00000 + + /* x^68672 mod p(x), x^68608 mod p(x) */ + .octa 0x00000000440800000000000090d30000 + + /* x^67648 mod p(x), x^67584 mod p(x) */ + .octa 0x00000000186100000000000054730000 + + /* x^66624 mod p(x), x^66560 mod p(x) */ + .octa 0x000000007368000000000000a3a20000 + + /* x^65600 mod p(x), x^65536 mod p(x) */ + .octa 0x0000000026d0000000000000f9040000 + + /* x^64576 mod p(x), x^64512 mod p(x) */ + .octa 0x00000000fe770000000000009c0a0000 + + /* x^63552 mod p(x), x^63488 mod p(x) */ + .octa 0x000000002cba000000000000d1e70000 + + /* x^62528 mod p(x), x^62464 mod p(x) */ + .octa 0x00000000f8bd0000000000005ac10000 + + /* x^61504 mod p(x), x^61440 mod p(x) */ + .octa 0x000000007372000000000000d68d0000 + + /* x^60480 mod p(x), x^60416 mod p(x) */ + .octa 0x00000000f37f00000000000089f60000 + + /* x^59456 mod p(x), x^59392 mod p(x) */ + .octa 0x00000000078400000000000008a90000 + + /* x^58432 mod p(x), x^58368 mod p(x) */ + .octa 0x00000000d3e400000000000042360000 + + /* x^57408 mod p(x), x^57344 mod p(x) */ + .octa 0x00000000eba800000000000092d50000 + + /* x^56384 mod p(x), x^56320 mod p(x) */ + .octa 0x00000000afbe000000000000b4d50000 + + /* x^55360 mod p(x), x^55296 mod p(x) */ + .octa 0x00000000d8ca000000000000c9060000 + + /* x^54336 mod p(x), x^54272 mod p(x) */ + .octa 0x00000000c2d00000000000008f4f0000 + + /* x^53312 mod p(x), x^53248 mod p(x) */ + .octa 0x00000000373200000000000028690000 + + /* x^52288 mod p(x), x^52224 mod p(x) */ + .octa 0x0000000046ae000000000000c3b30000 + + /* x^51264 mod p(x), x^51200 mod p(x) */ + .octa 0x00000000b243000000000000f8700000 + + /* x^50240 mod p(x), x^50176 mod p(x) */ + .octa 0x00000000f7f500000000000029eb0000 + + /* x^49216 mod p(x), x^49152 mod p(x) */ + .octa 0x000000000c7e000000000000fe730000 + + /* x^48192 mod p(x), x^48128 mod p(x) */ + .octa 0x00000000c38200000000000096000000 + + /* x^47168 mod p(x), x^47104 mod p(x) */ + .octa 0x000000008956000000000000683c0000 + + /* x^46144 mod p(x), x^46080 mod p(x) */ + .octa 0x00000000422d0000000000005f1e0000 + + /* x^45120 mod p(x), x^45056 mod p(x) */ + .octa 0x00000000ac0f0000000000006f810000 + + /* x^44096 mod p(x), x^44032 mod p(x) */ + .octa 0x00000000ce30000000000000031f0000 + + /* x^43072 mod p(x), x^43008 mod p(x) */ + .octa 0x000000003d43000000000000455a0000 + + /* x^42048 mod p(x), x^41984 mod p(x) */ + .octa 0x000000007ebe000000000000a6050000 + + /* x^41024 mod p(x), x^40960 mod p(x) */ + .octa 0x00000000976e00000000000077eb0000 + + /* x^40000 mod p(x), x^39936 mod p(x) */ + .octa 0x000000000872000000000000389c0000 + + /* x^38976 mod p(x), x^38912 mod p(x) */ + .octa 0x000000008979000000000000c7b20000 + + /* x^37952 mod p(x), x^37888 mod p(x) */ + .octa 0x000000005c1e0000000000001d870000 + + /* x^36928 mod p(x), x^36864 mod p(x) */ + .octa 0x00000000aebb00000000000045810000 + + /* x^35904 mod p(x), x^35840 mod p(x) */ + .octa 0x000000004f7e0000000000006d4a0000 + + /* x^34880 mod p(x), x^34816 mod p(x) */ + .octa 0x00000000ea98000000000000b9200000 + + /* x^33856 mod p(x), x^33792 mod p(x) */ + .octa 0x00000000f39600000000000022f20000 + + /* x^32832 mod p(x), x^32768 mod p(x) */ + .octa 0x000000000bc500000000000041ca0000 + + /* x^31808 mod p(x), x^31744 mod p(x) */ + .octa 0x00000000786400000000000078500000 + + /* x^30784 mod p(x), x^30720 mod p(x) */ + .octa 0x00000000be970000000000009e7e0000 + + /* x^29760 mod p(x), x^29696 mod p(x) */ + .octa 0x00000000dd6d000000000000a53c0000 + + /* x^28736 mod p(x), x^28672 mod p(x) */ + .octa 0x000000004c3f00000000000039340000 + + /* x^27712 mod p(x), x^27648 mod p(x) */ + .octa 0x0000000093a4000000000000b58e0000 + + /* x^26688 mod p(x), x^26624 mod p(x) */ + .octa 0x0000000050fb00000000000062d40000 + + /* x^25664 mod p(x), x^25600 mod p(x) */ + .octa 0x00000000f505000000000000a26f0000 + + /* x^24640 mod p(x), x^24576 mod p(x) */ + .octa 0x0000000064f900000000000065e60000 + + /* x^23616 mod p(x), x^23552 mod p(x) */ + .octa 0x00000000e8c2000000000000aad90000 + + /* x^22592 mod p(x), x^22528 mod p(x) */ + .octa 0x00000000720b000000000000a3b00000 + + /* x^21568 mod p(x), x^21504 mod p(x) */ + .octa 0x00000000e992000000000000d2680000 + + /* x^20544 mod p(x), x^20480 mod p(x) */ + .octa 0x000000009132000000000000cf4c0000 + + /* x^19520 mod p(x), x^19456 mod p(x) */ + .octa 0x00000000608a00000000000076610000 + + /* x^18496 mod p(x), x^18432 mod p(x) */ + .octa 0x000000009948000000000000fb9f0000 + + /* x^17472 mod p(x), x^17408 mod p(x) */ + .octa 0x00000000173000000000000003770000 + + /* x^16448 mod p(x), x^16384 mod p(x) */ + .octa 0x000000006fe300000000000004880000 + + /* x^15424 mod p(x), x^15360 mod p(x) */ + .octa 0x00000000e15300000000000056a70000 + + /* x^14400 mod p(x), x^14336 mod p(x) */ + .octa 0x0000000092d60000000000009dfd0000 + + /* x^13376 mod p(x), x^13312 mod p(x) */ + .octa 0x0000000002fd00000000000074c80000 + + /* x^12352 mod p(x), x^12288 mod p(x) */ + .octa 0x00000000c78b000000000000a3ec0000 + + /* x^11328 mod p(x), x^11264 mod p(x) */ + .octa 0x000000009262000000000000b3530000 + + /* x^10304 mod p(x), x^10240 mod p(x) */ + .octa 0x0000000084f200000000000047bf0000 + + /* x^9280 mod p(x), x^9216 mod p(x) */ + .octa 0x0000000067ee000000000000e97c0000 + + /* x^8256 mod p(x), x^8192 mod p(x) */ + .octa 0x00000000535b00000000000091e10000 + + /* x^7232 mod p(x), x^7168 mod p(x) */ + .octa 0x000000007ebb00000000000055060000 + + /* x^6208 mod p(x), x^6144 mod p(x) */ + .octa 0x00000000c6a1000000000000fd360000 + + /* x^5184 mod p(x), x^5120 mod p(x) */ + .octa 0x000000001be500000000000055860000 + + /* x^4160 mod p(x), x^4096 mod p(x) */ + .octa 0x00000000ae0e0000000000005bd00000 + + /* x^3136 mod p(x), x^3072 mod p(x) */ + .octa 0x0000000022040000000000008db20000 + + /* x^2112 mod p(x), x^2048 mod p(x) */ + .octa 0x00000000c9eb000000000000efe20000 + + /* x^1088 mod p(x), x^1024 mod p(x) */ + .octa 0x0000000039b400000000000051d10000 + +.short_constants: + + /* Reduce final 1024-2048 bits to 64 bits, shifting 32 bits to include the trailing 32 bits of zeros */ + /* x^2048 mod p(x), x^2016 mod p(x), x^1984 mod p(x), x^1952 mod p(x) */ + .octa 0xefe20000dccf00009440000033590000 + + /* x^1920 mod p(x), x^1888 mod p(x), x^1856 mod p(x), x^1824 mod p(x) */ + .octa 0xee6300002f3f000062180000e0ed0000 + + /* x^1792 mod p(x), x^1760 mod p(x), x^1728 mod p(x), x^1696 mod p(x) */ + .octa 0xcf5f000017ef0000ccbe000023d30000 + + /* x^1664 mod p(x), x^1632 mod p(x), x^1600 mod p(x), x^1568 mod p(x) */ + .octa 0x6d0c0000a30e00000920000042630000 + + /* x^1536 mod p(x), x^1504 mod p(x), x^1472 mod p(x), x^1440 mod p(x) */ + .octa 0x21d30000932b0000a7a00000efcc0000 + + /* x^1408 mod p(x), x^1376 mod p(x), x^1344 mod p(x), x^1312 mod p(x) */ + .octa 0x10be00000b310000666f00000d1c0000 + + /* x^1280 mod p(x), x^1248 mod p(x), x^1216 mod p(x), x^1184 mod p(x) */ + .octa 0x1f240000ce9e0000caad0000589e0000 + + /* x^1152 mod p(x), x^1120 mod p(x), x^1088 mod p(x), x^1056 mod p(x) */ + .octa 0x29610000d02b000039b400007cf50000 + + /* x^1024 mod p(x), x^992 mod p(x), x^960 mod p(x), x^928 mod p(x) */ + .octa 0x51d100009d9d00003c0e0000bfd60000 + + /* x^896 mod p(x), x^864 mod p(x), x^832 mod p(x), x^800 mod p(x) */ + .octa 0xda390000ceae000013830000713c0000 + + /* x^768 mod p(x), x^736 mod p(x), x^704 mod p(x), x^672 mod p(x) */ + .octa 0xb67800001e16000085c0000080a60000 + + /* x^640 mod p(x), x^608 mod p(x), x^576 mod p(x), x^544 mod p(x) */ + .octa 0x0db40000f7f90000371d0000e6580000 + + /* x^512 mod p(x), x^480 mod p(x), x^448 mod p(x), x^416 mod p(x) */ + .octa 0x87e70000044c0000aadb0000a4970000 + + /* x^384 mod p(x), x^352 mod p(x), x^320 mod p(x), x^288 mod p(x) */ + .octa 0x1f990000ad180000d8b30000e7b50000 + + /* x^256 mod p(x), x^224 mod p(x), x^192 mod p(x), x^160 mod p(x) */ + .octa 0xbe6c00006ee300004c1a000006df0000 + + /* x^128 mod p(x), x^96 mod p(x), x^64 mod p(x), x^32 mod p(x) */ + .octa 0xfb0b00002d560000136800008bb70000 + + +.barrett_constants: + /* Barrett constant m - (4^32)/n */ + .octa 0x000000000000000000000001f65a57f8 /* x^64 div p(x) */ + /* Barrett constant n */ + .octa 0x0000000000000000000000018bb70000 + +#define CRC_FUNCTION_NAME __crct10dif_vpmsum +#include "crc32-vpmsum_core.S" diff --git a/arch/powerpc/crypto/crct10dif-vpmsum_glue.c b/arch/powerpc/crypto/crct10dif-vpmsum_glue.c new file mode 100644 index 000000000000..bebfc329f746 --- /dev/null +++ b/arch/powerpc/crypto/crct10dif-vpmsum_glue.c @@ -0,0 +1,125 @@ +/* + * Calculate a CRC T10-DIF with vpmsum acceleration + * + * Copyright 2017, Daniel Axtens, IBM Corporation. + * [based on crc32c-vpmsum_glue.c] + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define VMX_ALIGN 16 +#define VMX_ALIGN_MASK (VMX_ALIGN-1) + +#define VECTOR_BREAKPOINT 64 + +u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len); + +static u16 crct10dif_vpmsum(u16 crci, unsigned char const *p, size_t len) +{ + unsigned int prealign; + unsigned int tail; + u32 crc = crci; + + if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || in_interrupt()) + return crc_t10dif_generic(crc, p, len); + + if ((unsigned long)p & VMX_ALIGN_MASK) { + prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); + crc = crc_t10dif_generic(crc, p, prealign); + len -= prealign; + p += prealign; + } + + if (len & ~VMX_ALIGN_MASK) { + crc <<= 16; + pagefault_disable(); + enable_kernel_altivec(); + crc = __crct10dif_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); + pagefault_enable(); + crc >>= 16; + } + + tail = len & VMX_ALIGN_MASK; + if (tail) { + p += len & ~VMX_ALIGN_MASK; + crc = crc_t10dif_generic(crc, p, tail); + } + + return crc & 0xffff; +} + +static int crct10dif_vpmsum_init(struct shash_desc *desc) +{ + u16 *crc = shash_desc_ctx(desc); + + *crc = 0; + return 0; +} + +static int crct10dif_vpmsum_update(struct shash_desc *desc, const u8 *data, + unsigned int length) +{ + u16 *crc = shash_desc_ctx(desc); + + *crc = crct10dif_vpmsum(*crc, data, length); + + return 0; +} + + +static int crct10dif_vpmsum_final(struct shash_desc *desc, u8 *out) +{ + u16 *crcp = shash_desc_ctx(desc); + + *(u16 *)out = *crcp; + return 0; +} + +static struct shash_alg alg = { + .init = crct10dif_vpmsum_init, + .update = crct10dif_vpmsum_update, + .final = crct10dif_vpmsum_final, + .descsize = CRC_T10DIF_DIGEST_SIZE, + .digestsize = CRC_T10DIF_DIGEST_SIZE, + .base = { + .cra_name = "crct10dif", + .cra_driver_name = "crct10dif-vpmsum", + .cra_priority = 200, + .cra_blocksize = CRC_T10DIF_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init crct10dif_vpmsum_mod_init(void) +{ + if (!cpu_has_feature(CPU_FTR_ARCH_207S)) + return -ENODEV; + + return crypto_register_shash(&alg); +} + +static void __exit crct10dif_vpmsum_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, crct10dif_vpmsum_mod_init); +module_exit(crct10dif_vpmsum_mod_fini); + +MODULE_AUTHOR("Daniel Axtens "); +MODULE_DESCRIPTION("CRCT10DIF using vector polynomial multiply-sum instructions"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS_CRYPTO("crct10dif"); +MODULE_ALIAS_CRYPTO("crct10dif-vpmsum"); diff --git a/crypto/Kconfig b/crypto/Kconfig index f37e9cca50e1..9cf63dd84364 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -513,6 +513,15 @@ config CRYPTO_CRCT10DIF_PCLMUL 'crct10dif-plcmul' module, which is faster when computing the crct10dif checksum as compared with the generic table implementation. +config CRYPTO_CRCT10DIF_VPMSUM + tristate "CRC32T10DIF powerpc64 hardware acceleration" + depends on PPC64 && ALTIVEC && CRC_T10DIF + select CRYPTO_HASH + help + CRC10T10DIF algorithm implemented using vector polynomial + multiply-sum (vpmsum) instructions, introduced in POWER8. Enable on + POWER8 and newer processors for improved performance. + config CRYPTO_GHASH tristate "GHASH digest algorithm" select CRYPTO_GF128MUL From 146c8688d99c574d9ff0af17eca51bbd6402a57f Mon Sep 17 00:00:00 2001 From: Daniel Axtens Date: Wed, 15 Mar 2017 23:37:37 +1100 Subject: [PATCH 033/101] crypto: powerpc - Stress test for vpmsum implementations vpmsum implementations often don't kick in for short test vectors. This is a simple test module that does a configurable number of random tests, each up to 64kB and each with random offsets. Both CRC-T10DIF and CRC32C are tested. Cc: Anton Blanchard Signed-off-by: Daniel Axtens Signed-off-by: Herbert Xu --- arch/powerpc/crypto/Makefile | 1 + arch/powerpc/crypto/crc-vpmsum_test.c | 137 ++++++++++++++++++++++++++ crypto/Kconfig | 8 ++ 3 files changed, 146 insertions(+) create mode 100644 arch/powerpc/crypto/crc-vpmsum_test.c diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile index e66aaf19764d..67eca3af9fc7 100644 --- a/arch/powerpc/crypto/Makefile +++ b/arch/powerpc/crypto/Makefile @@ -11,6 +11,7 @@ obj-$(CONFIG_CRYPTO_SHA1_PPC_SPE) += sha1-ppc-spe.o obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o obj-$(CONFIG_CRYPTO_CRC32C_VPMSUM) += crc32c-vpmsum.o obj-$(CONFIG_CRYPTO_CRCT10DIF_VPMSUM) += crct10dif-vpmsum.o +obj-$(CONFIG_CRYPTO_VPMSUM_TESTER) += crc-vpmsum_test.o aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes-spe-glue.o md5-ppc-y := md5-asm.o md5-glue.o diff --git a/arch/powerpc/crypto/crc-vpmsum_test.c b/arch/powerpc/crypto/crc-vpmsum_test.c new file mode 100644 index 000000000000..0153a9c6f4af --- /dev/null +++ b/arch/powerpc/crypto/crc-vpmsum_test.c @@ -0,0 +1,137 @@ +/* + * CRC vpmsum tester + * Copyright 2017 Daniel Axtens, IBM Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned long iterations = 10000; + +#define MAX_CRC_LENGTH 65535 + + +static int __init crc_test_init(void) +{ + u16 crc16 = 0, verify16 = 0; + u32 crc32 = 0, verify32 = 0; + __le32 verify32le = 0; + unsigned char *data; + unsigned long i; + int ret; + + struct crypto_shash *crct10dif_tfm; + struct crypto_shash *crc32c_tfm; + + if (!cpu_has_feature(CPU_FTR_ARCH_207S)) + return -ENODEV; + + data = kmalloc(MAX_CRC_LENGTH, GFP_KERNEL); + if (!data) + return -ENOMEM; + + crct10dif_tfm = crypto_alloc_shash("crct10dif", 0, 0); + + if (IS_ERR(crct10dif_tfm)) { + pr_err("Error allocating crc-t10dif\n"); + goto free_buf; + } + + crc32c_tfm = crypto_alloc_shash("crc32c", 0, 0); + + if (IS_ERR(crc32c_tfm)) { + pr_err("Error allocating crc32c\n"); + goto free_16; + } + + do { + SHASH_DESC_ON_STACK(crct10dif_shash, crct10dif_tfm); + SHASH_DESC_ON_STACK(crc32c_shash, crc32c_tfm); + + crct10dif_shash->tfm = crct10dif_tfm; + ret = crypto_shash_init(crct10dif_shash); + + if (ret) { + pr_err("Error initing crc-t10dif\n"); + goto free_32; + } + + + crc32c_shash->tfm = crc32c_tfm; + ret = crypto_shash_init(crc32c_shash); + + if (ret) { + pr_err("Error initing crc32c\n"); + goto free_32; + } + + pr_info("crc-vpmsum_test begins, %lu iterations\n", iterations); + for (i=0; i"); +MODULE_DESCRIPTION("Vector polynomial multiply-sum CRC tester"); +MODULE_LICENSE("GPL"); diff --git a/crypto/Kconfig b/crypto/Kconfig index 9cf63dd84364..6854c1fe54b7 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -522,6 +522,14 @@ config CRYPTO_CRCT10DIF_VPMSUM multiply-sum (vpmsum) instructions, introduced in POWER8. Enable on POWER8 and newer processors for improved performance. +config CRYPTO_VPMSUM_TESTER + tristate "Powerpc64 vpmsum hardware acceleration tester" + depends on CRYPTO_CRCT10DIF_VPMSUM && CRYPTO_CRC32C_VPMSUM + help + Stress test for CRC32c and CRC-T10DIF algorithms implemented with + POWER8 vpmsum instructions. + Unless you are testing these algorithms, you don't need this. + config CRYPTO_GHASH tristate "GHASH digest algorithm" select CRYPTO_GF128MUL From ccebcf3f224a44ec8e9c5bfca9d8e5d29298a5a8 Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Wed, 15 Mar 2017 13:20:43 -0500 Subject: [PATCH 034/101] crypto: ccp - Add SHA-2 384- and 512-bit support Incorporate 384-bit and 512-bit hashing for a version 5 CCP device Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-crypto-sha.c | 22 +++++++++ drivers/crypto/ccp/ccp-crypto.h | 8 ++-- drivers/crypto/ccp/ccp-ops.c | 72 +++++++++++++++++++++++++++++ include/linux/ccp.h | 2 + 4 files changed, 101 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c index 84a652be4274..6b46eea94932 100644 --- a/drivers/crypto/ccp/ccp-crypto-sha.c +++ b/drivers/crypto/ccp/ccp-crypto-sha.c @@ -146,6 +146,12 @@ static int ccp_do_sha_update(struct ahash_request *req, unsigned int nbytes, case CCP_SHA_TYPE_256: rctx->cmd.u.sha.ctx_len = SHA256_DIGEST_SIZE; break; + case CCP_SHA_TYPE_384: + rctx->cmd.u.sha.ctx_len = SHA384_DIGEST_SIZE; + break; + case CCP_SHA_TYPE_512: + rctx->cmd.u.sha.ctx_len = SHA512_DIGEST_SIZE; + break; default: /* Should never get here */ break; @@ -393,6 +399,22 @@ static struct ccp_sha_def sha_algs[] = { .digest_size = SHA256_DIGEST_SIZE, .block_size = SHA256_BLOCK_SIZE, }, + { + .version = CCP_VERSION(5, 0), + .name = "sha384", + .drv_name = "sha384-ccp", + .type = CCP_SHA_TYPE_384, + .digest_size = SHA384_DIGEST_SIZE, + .block_size = SHA384_BLOCK_SIZE, + }, + { + .version = CCP_VERSION(5, 0), + .name = "sha512", + .drv_name = "sha512-ccp", + .type = CCP_SHA_TYPE_512, + .digest_size = SHA512_DIGEST_SIZE, + .block_size = SHA512_BLOCK_SIZE, + }, }; static int ccp_register_hmac_alg(struct list_head *head, diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h index 8335b32e815e..95cce2764139 100644 --- a/drivers/crypto/ccp/ccp-crypto.h +++ b/drivers/crypto/ccp/ccp-crypto.h @@ -137,9 +137,11 @@ struct ccp_aes_cmac_exp_ctx { u8 buf[AES_BLOCK_SIZE]; }; -/***** SHA related defines *****/ -#define MAX_SHA_CONTEXT_SIZE SHA256_DIGEST_SIZE -#define MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE +/* SHA-related defines + * These values must be large enough to accommodate any variant + */ +#define MAX_SHA_CONTEXT_SIZE SHA512_DIGEST_SIZE +#define MAX_SHA_BLOCK_SIZE SHA512_BLOCK_SIZE struct ccp_sha_ctx { struct scatterlist opad_sg; diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index f1396c3aedac..0d820802cd92 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -41,6 +41,20 @@ static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), }; +static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { + cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1), + cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3), + cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5), + cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7), +}; + +static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { + cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1), + cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3), + cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5), + cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7), +}; + #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \ ccp_gen_jobid(ccp) : 0) @@ -955,6 +969,18 @@ static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) return -EINVAL; block_size = SHA256_BLOCK_SIZE; break; + case CCP_SHA_TYPE_384: + if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) + || sha->ctx_len < SHA384_DIGEST_SIZE) + return -EINVAL; + block_size = SHA384_BLOCK_SIZE; + break; + case CCP_SHA_TYPE_512: + if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) + || sha->ctx_len < SHA512_DIGEST_SIZE) + return -EINVAL; + block_size = SHA512_BLOCK_SIZE; + break; default: return -EINVAL; } @@ -1042,6 +1068,21 @@ static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) sb_count = 1; ooffset = ioffset = 0; break; + case CCP_SHA_TYPE_384: + digest_size = SHA384_DIGEST_SIZE; + init = (void *) ccp_sha384_init; + ctx_size = SHA512_DIGEST_SIZE; + sb_count = 2; + ioffset = 0; + ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE; + break; + case CCP_SHA_TYPE_512: + digest_size = SHA512_DIGEST_SIZE; + init = (void *) ccp_sha512_init; + ctx_size = SHA512_DIGEST_SIZE; + sb_count = 2; + ooffset = ioffset = 0; + break; default: ret = -EINVAL; goto e_data; @@ -1060,6 +1101,11 @@ static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) op.u.sha.type = sha->type; op.u.sha.msg_bits = sha->msg_bits; + /* For SHA1/224/256 the context fits in a single (32-byte) SB entry; + * SHA384/512 require 2 adjacent SB slots, with the right half in the + * first slot, and the left half in the second. Each portion must then + * be in little endian format: use the 256-bit byte swap option. + */ ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES, DMA_BIDIRECTIONAL); if (ret) @@ -1071,6 +1117,13 @@ static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) case CCP_SHA_TYPE_256: memcpy(ctx.address + ioffset, init, ctx_size); break; + case CCP_SHA_TYPE_384: + case CCP_SHA_TYPE_512: + memcpy(ctx.address + ctx_size / 2, init, + ctx_size / 2); + memcpy(ctx.address, init + ctx_size / 2, + ctx_size / 2); + break; default: ret = -EINVAL; goto e_ctx; @@ -1137,6 +1190,15 @@ static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) sha->ctx, 0, digest_size); break; + case CCP_SHA_TYPE_384: + case CCP_SHA_TYPE_512: + ccp_get_dm_area(&ctx, 0, + sha->ctx, LSB_ITEM_SIZE - ooffset, + LSB_ITEM_SIZE); + ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset, + sha->ctx, 0, + LSB_ITEM_SIZE - ooffset); + break; default: ret = -EINVAL; goto e_ctx; @@ -1174,6 +1236,16 @@ static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) ctx.address + ooffset, digest_size); break; + case CCP_SHA_TYPE_384: + case CCP_SHA_TYPE_512: + memcpy(hmac_buf + block_size, + ctx.address + LSB_ITEM_SIZE + ooffset, + LSB_ITEM_SIZE); + memcpy(hmac_buf + block_size + + (LSB_ITEM_SIZE - ooffset), + ctx.address, + LSB_ITEM_SIZE); + break; default: ret = -EINVAL; goto e_ctx; diff --git a/include/linux/ccp.h b/include/linux/ccp.h index c71dd8fa5764..90a1fbe84219 100644 --- a/include/linux/ccp.h +++ b/include/linux/ccp.h @@ -249,6 +249,8 @@ enum ccp_sha_type { CCP_SHA_TYPE_1 = 1, CCP_SHA_TYPE_224, CCP_SHA_TYPE_256, + CCP_SHA_TYPE_384, + CCP_SHA_TYPE_512, CCP_SHA_TYPE__LAST, }; From 990672d48515ce09c76fcf1ceccee48b0dd1942b Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Wed, 15 Mar 2017 13:20:52 -0500 Subject: [PATCH 035/101] crypto: ccp - Enable 3DES function on v5 CCPs Wire up support for Triple DES in ECB mode. Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/Makefile | 1 + drivers/crypto/ccp/ccp-crypto-des3.c | 254 +++++++++++++++++++++++++++ drivers/crypto/ccp/ccp-crypto-main.c | 10 ++ drivers/crypto/ccp/ccp-crypto.h | 22 +++ drivers/crypto/ccp/ccp-dev-v3.c | 1 + drivers/crypto/ccp/ccp-dev-v5.c | 54 ++++++ drivers/crypto/ccp/ccp-dev.h | 14 +- drivers/crypto/ccp/ccp-ops.c | 198 +++++++++++++++++++++ include/linux/ccp.h | 57 +++++- 9 files changed, 608 insertions(+), 3 deletions(-) create mode 100644 drivers/crypto/ccp/ccp-crypto-des3.c diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile index 346ceb8f17bd..d2044b793aa8 100644 --- a/drivers/crypto/ccp/Makefile +++ b/drivers/crypto/ccp/Makefile @@ -12,4 +12,5 @@ ccp-crypto-objs := ccp-crypto-main.o \ ccp-crypto-aes.o \ ccp-crypto-aes-cmac.o \ ccp-crypto-aes-xts.o \ + ccp-crypto-des3.o \ ccp-crypto-sha.o diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c new file mode 100644 index 000000000000..5af7347ae03c --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-des3.c @@ -0,0 +1,254 @@ +/* + * AMD Cryptographic Coprocessor (CCP) DES3 crypto API support + * + * Copyright (C) 2016 Advanced Micro Devices, Inc. + * + * Author: Gary R Hook + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ccp-crypto.h" + +static int ccp_des3_complete(struct crypto_async_request *async_req, int ret) +{ + struct ablkcipher_request *req = ablkcipher_request_cast(async_req); + struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct ccp_des3_req_ctx *rctx = ablkcipher_request_ctx(req); + + if (ret) + return ret; + + if (ctx->u.des3.mode != CCP_DES3_MODE_ECB) + memcpy(req->info, rctx->iv, DES3_EDE_BLOCK_SIZE); + + return 0; +} + +static int ccp_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm)); + struct ccp_crypto_ablkcipher_alg *alg = + ccp_crypto_ablkcipher_alg(crypto_ablkcipher_tfm(tfm)); + u32 *flags = &tfm->base.crt_flags; + + + /* From des_generic.c: + * + * RFC2451: + * If the first two or last two independent 64-bit keys are + * equal (k1 == k2 or k2 == k3), then the DES3 operation is simply the + * same as DES. Implementers MUST reject keys that exhibit this + * property. + */ + const u32 *K = (const u32 *)key; + + if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || + !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && + (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + *flags |= CRYPTO_TFM_RES_WEAK_KEY; + return -EINVAL; + } + + /* It's not clear that there is any support for a keysize of 112. + * If needed, the caller should make K1 == K3 + */ + ctx->u.des3.type = CCP_DES3_TYPE_168; + ctx->u.des3.mode = alg->mode; + ctx->u.des3.key_len = key_len; + + memcpy(ctx->u.des3.key, key, key_len); + sg_init_one(&ctx->u.des3.key_sg, ctx->u.des3.key, key_len); + + return 0; +} + +static int ccp_des3_crypt(struct ablkcipher_request *req, bool encrypt) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct ccp_des3_req_ctx *rctx = ablkcipher_request_ctx(req); + struct scatterlist *iv_sg = NULL; + unsigned int iv_len = 0; + int ret; + + if (!ctx->u.des3.key_len) + return -EINVAL; + + if (((ctx->u.des3.mode == CCP_DES3_MODE_ECB) || + (ctx->u.des3.mode == CCP_DES3_MODE_CBC)) && + (req->nbytes & (DES3_EDE_BLOCK_SIZE - 1))) + return -EINVAL; + + if (ctx->u.des3.mode != CCP_DES3_MODE_ECB) { + if (!req->info) + return -EINVAL; + + memcpy(rctx->iv, req->info, DES3_EDE_BLOCK_SIZE); + iv_sg = &rctx->iv_sg; + iv_len = DES3_EDE_BLOCK_SIZE; + sg_init_one(iv_sg, rctx->iv, iv_len); + } + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_DES3; + rctx->cmd.u.des3.type = ctx->u.des3.type; + rctx->cmd.u.des3.mode = ctx->u.des3.mode; + rctx->cmd.u.des3.action = (encrypt) + ? CCP_DES3_ACTION_ENCRYPT + : CCP_DES3_ACTION_DECRYPT; + rctx->cmd.u.des3.key = &ctx->u.des3.key_sg; + rctx->cmd.u.des3.key_len = ctx->u.des3.key_len; + rctx->cmd.u.des3.iv = iv_sg; + rctx->cmd.u.des3.iv_len = iv_len; + rctx->cmd.u.des3.src = req->src; + rctx->cmd.u.des3.src_len = req->nbytes; + rctx->cmd.u.des3.dst = req->dst; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; +} + +static int ccp_des3_encrypt(struct ablkcipher_request *req) +{ + return ccp_des3_crypt(req, true); +} + +static int ccp_des3_decrypt(struct ablkcipher_request *req) +{ + return ccp_des3_crypt(req, false); +} + +static int ccp_des3_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->complete = ccp_des3_complete; + ctx->u.des3.key_len = 0; + + tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_des3_req_ctx); + + return 0; +} + +static void ccp_des3_cra_exit(struct crypto_tfm *tfm) +{ +} + +static struct crypto_alg ccp_des3_defaults = { + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ccp_ctx), + .cra_priority = CCP_CRA_PRIORITY, + .cra_type = &crypto_ablkcipher_type, + .cra_init = ccp_des3_cra_init, + .cra_exit = ccp_des3_cra_exit, + .cra_module = THIS_MODULE, + .cra_ablkcipher = { + .setkey = ccp_des3_setkey, + .encrypt = ccp_des3_encrypt, + .decrypt = ccp_des3_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + }, +}; + +struct ccp_des3_def { + enum ccp_des3_mode mode; + unsigned int version; + const char *name; + const char *driver_name; + unsigned int blocksize; + unsigned int ivsize; + struct crypto_alg *alg_defaults; +}; + +static struct ccp_des3_def des3_algs[] = { + { + .mode = CCP_DES3_MODE_ECB, + .version = CCP_VERSION(5, 0), + .name = "ecb(des3_ede)", + .driver_name = "ecb-des3-ccp", + .blocksize = DES3_EDE_BLOCK_SIZE, + .ivsize = 0, + .alg_defaults = &ccp_des3_defaults, + }, + { + .mode = CCP_DES3_MODE_CBC, + .version = CCP_VERSION(5, 0), + .name = "cbc(des3_ede)", + .driver_name = "cbc-des3-ccp", + .blocksize = DES3_EDE_BLOCK_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .alg_defaults = &ccp_des3_defaults, + }, +}; + +static int ccp_register_des3_alg(struct list_head *head, + const struct ccp_des3_def *def) +{ + struct ccp_crypto_ablkcipher_alg *ccp_alg; + struct crypto_alg *alg; + int ret; + + ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + + ccp_alg->mode = def->mode; + + /* Copy the defaults and override as necessary */ + alg = &ccp_alg->alg; + *alg = *def->alg_defaults; + snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name); + snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + def->driver_name); + alg->cra_blocksize = def->blocksize; + alg->cra_ablkcipher.ivsize = def->ivsize; + + ret = crypto_register_alg(alg); + if (ret) { + pr_err("%s ablkcipher algorithm registration error (%d)\n", + alg->cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return 0; +} + +int ccp_register_des3_algs(struct list_head *head) +{ + int i, ret; + unsigned int ccpversion = ccp_version(); + + for (i = 0; i < ARRAY_SIZE(des3_algs); i++) { + if (des3_algs[i].version > ccpversion) + continue; + ret = ccp_register_des3_alg(head, &des3_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c index e0380e59c361..3f1e36d7a8bf 100644 --- a/drivers/crypto/ccp/ccp-crypto-main.c +++ b/drivers/crypto/ccp/ccp-crypto-main.c @@ -33,6 +33,10 @@ static unsigned int sha_disable; module_param(sha_disable, uint, 0444); MODULE_PARM_DESC(sha_disable, "Disable use of SHA - any non-zero value"); +static unsigned int des3_disable; +module_param(des3_disable, uint, 0444); +MODULE_PARM_DESC(des3_disable, "Disable use of 3DES - any non-zero value"); + /* List heads for the supported algorithms */ static LIST_HEAD(hash_algs); static LIST_HEAD(cipher_algs); @@ -337,6 +341,12 @@ static int ccp_register_algs(void) return ret; } + if (!des3_disable) { + ret = ccp_register_des3_algs(&cipher_algs); + if (ret) + return ret; + } + if (!sha_disable) { ret = ccp_register_sha_algs(&hash_algs); if (ret) diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h index 95cce2764139..8c8bd3ff9f48 100644 --- a/drivers/crypto/ccp/ccp-crypto.h +++ b/drivers/crypto/ccp/ccp-crypto.h @@ -23,6 +23,8 @@ #include #include +#define CCP_LOG_LEVEL KERN_INFO + #define CCP_CRA_PRIORITY 300 struct ccp_crypto_ablkcipher_alg { @@ -137,6 +139,24 @@ struct ccp_aes_cmac_exp_ctx { u8 buf[AES_BLOCK_SIZE]; }; +/***** 3DES related defines *****/ +struct ccp_des3_ctx { + enum ccp_engine engine; + enum ccp_des3_type type; + enum ccp_des3_mode mode; + + struct scatterlist key_sg; + unsigned int key_len; + u8 key[AES_MAX_KEY_SIZE]; +}; + +struct ccp_des3_req_ctx { + struct scatterlist iv_sg; + u8 iv[AES_BLOCK_SIZE]; + + struct ccp_cmd cmd; +}; + /* SHA-related defines * These values must be large enough to accommodate any variant */ @@ -201,6 +221,7 @@ struct ccp_ctx { union { struct ccp_aes_ctx aes; struct ccp_sha_ctx sha; + struct ccp_des3_ctx des3; } u; }; @@ -213,5 +234,6 @@ int ccp_register_aes_algs(struct list_head *head); int ccp_register_aes_cmac_algs(struct list_head *head); int ccp_register_aes_xts_algs(struct list_head *head); int ccp_register_sha_algs(struct list_head *head); +int ccp_register_des3_algs(struct list_head *head); #endif diff --git a/drivers/crypto/ccp/ccp-dev-v3.c b/drivers/crypto/ccp/ccp-dev-v3.c index 7bc09989e18a..a3689a66cb1d 100644 --- a/drivers/crypto/ccp/ccp-dev-v3.c +++ b/drivers/crypto/ccp/ccp-dev-v3.c @@ -553,6 +553,7 @@ static irqreturn_t ccp_irq_handler(int irq, void *data) static const struct ccp_actions ccp3_actions = { .aes = ccp_perform_aes, .xts_aes = ccp_perform_xts_aes, + .des3 = NULL, .sha = ccp_perform_sha, .rsa = ccp_perform_rsa, .passthru = ccp_perform_passthru, diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c index 41cc853f8569..fc5666eb59f2 100644 --- a/drivers/crypto/ccp/ccp-dev-v5.c +++ b/drivers/crypto/ccp/ccp-dev-v5.c @@ -107,6 +107,12 @@ union ccp_function { u16 rsvd:5; u16 type:2; } aes_xts; + struct { + u16 size:7; + u16 encrypt:1; + u16 mode:5; + u16 type:2; + } des3; struct { u16 rsvd1:10; u16 type:4; @@ -139,6 +145,10 @@ union ccp_function { #define CCP_AES_TYPE(p) ((p)->aes.type) #define CCP_XTS_SIZE(p) ((p)->aes_xts.size) #define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt) +#define CCP_DES3_SIZE(p) ((p)->des3.size) +#define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt) +#define CCP_DES3_MODE(p) ((p)->des3.mode) +#define CCP_DES3_TYPE(p) ((p)->des3.type) #define CCP_SHA_TYPE(p) ((p)->sha.type) #define CCP_RSA_SIZE(p) ((p)->rsa.size) #define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap) @@ -388,6 +398,47 @@ static int ccp5_perform_sha(struct ccp_op *op) return ccp5_do_cmd(&desc, op->cmd_q); } +static int ccp5_perform_des3(struct ccp_op *op) +{ + struct ccp5_desc desc; + union ccp_function function; + u32 key_addr = op->sb_key * LSB_ITEM_SIZE; + + /* Zero out all the fields of the command desc */ + memset(&desc, 0, sizeof(struct ccp5_desc)); + + CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_DES3; + + CCP5_CMD_SOC(&desc) = op->soc; + CCP5_CMD_IOC(&desc) = 1; + CCP5_CMD_INIT(&desc) = op->init; + CCP5_CMD_EOM(&desc) = op->eom; + CCP5_CMD_PROT(&desc) = 0; + + function.raw = 0; + CCP_DES3_ENCRYPT(&function) = op->u.des3.action; + CCP_DES3_MODE(&function) = op->u.des3.mode; + CCP_DES3_TYPE(&function) = op->u.des3.type; + CCP5_CMD_FUNCTION(&desc) = cpu_to_le32(function.raw); + + CCP5_CMD_LEN(&desc) = cpu_to_le32(op->src.u.dma.length); + + CCP5_CMD_SRC_LO(&desc) = cpu_to_le32(ccp_addr_lo(&op->src.u.dma)); + CCP5_CMD_SRC_HI(&desc) = cpu_to_le32(ccp_addr_hi(&op->src.u.dma)); + CCP5_CMD_SRC_MEM(&desc) = cpu_to_le32(CCP_MEMTYPE_SYSTEM); + + CCP5_CMD_DST_LO(&desc) = cpu_to_le32(ccp_addr_lo(&op->dst.u.dma)); + CCP5_CMD_DST_HI(&desc) = cpu_to_le32(ccp_addr_hi(&op->dst.u.dma)); + CCP5_CMD_DST_MEM(&desc) = cpu_to_le32(CCP_MEMTYPE_SYSTEM); + + CCP5_CMD_KEY_LO(&desc) = cpu_to_le32(lower_32_bits(key_addr)); + CCP5_CMD_KEY_HI(&desc) = 0; + CCP5_CMD_KEY_MEM(&desc) = cpu_to_le32(CCP_MEMTYPE_SB); + CCP5_CMD_LSB_ID(&desc) = cpu_to_le32(op->sb_ctx); + + return ccp5_do_cmd(&desc, op->cmd_q); +} + static int ccp5_perform_rsa(struct ccp_op *op) { struct ccp5_desc desc; @@ -435,6 +486,7 @@ static int ccp5_perform_passthru(struct ccp_op *op) struct ccp_dma_info *saddr = &op->src.u.dma; struct ccp_dma_info *daddr = &op->dst.u.dma; + memset(&desc, 0, Q_DESC_SIZE); CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU; @@ -729,6 +781,7 @@ static int ccp5_init(struct ccp_device *ccp) dev_dbg(dev, "queue #%u available\n", i); } + if (ccp->cmd_q_count == 0) { dev_notice(dev, "no command queues available\n"); ret = -EIO; @@ -994,6 +1047,7 @@ static const struct ccp_actions ccp5_actions = { .aes = ccp5_perform_aes, .xts_aes = ccp5_perform_xts_aes, .sha = ccp5_perform_sha, + .des3 = ccp5_perform_des3, .rsa = ccp5_perform_rsa, .passthru = ccp5_perform_passthru, .ecc = ccp5_perform_ecc, diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h index 2b5c01fade05..754e9c2f6ee3 100644 --- a/drivers/crypto/ccp/ccp-dev.h +++ b/drivers/crypto/ccp/ccp-dev.h @@ -190,6 +190,9 @@ #define CCP_XTS_AES_KEY_SB_COUNT 1 #define CCP_XTS_AES_CTX_SB_COUNT 1 +#define CCP_DES3_KEY_SB_COUNT 1 +#define CCP_DES3_CTX_SB_COUNT 1 + #define CCP_SHA_SB_COUNT 1 #define CCP_RSA_MAX_WIDTH 4096 @@ -475,6 +478,12 @@ struct ccp_xts_aes_op { enum ccp_xts_aes_unit_size unit_size; }; +struct ccp_des3_op { + enum ccp_des3_type type; + enum ccp_des3_mode mode; + enum ccp_des3_action action; +}; + struct ccp_sha_op { enum ccp_sha_type type; u64 msg_bits; @@ -512,6 +521,7 @@ struct ccp_op { union { struct ccp_aes_op aes; struct ccp_xts_aes_op xts; + struct ccp_des3_op des3; struct ccp_sha_op sha; struct ccp_rsa_op rsa; struct ccp_passthru_op passthru; @@ -620,13 +630,13 @@ void ccp_dmaengine_unregister(struct ccp_device *ccp); struct ccp_actions { int (*aes)(struct ccp_op *); int (*xts_aes)(struct ccp_op *); + int (*des3)(struct ccp_op *); int (*sha)(struct ccp_op *); int (*rsa)(struct ccp_op *); int (*passthru)(struct ccp_op *); int (*ecc)(struct ccp_op *); u32 (*sballoc)(struct ccp_cmd_queue *, unsigned int); - void (*sbfree)(struct ccp_cmd_queue *, unsigned int, - unsigned int); + void (*sbfree)(struct ccp_cmd_queue *, unsigned int, unsigned int); unsigned int (*get_free_slots)(struct ccp_cmd_queue *); int (*init)(struct ccp_device *); void (*destroy)(struct ccp_device *); diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index 0d820802cd92..0de961a22ab4 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "ccp-dev.h" @@ -939,6 +940,200 @@ e_key: return ret; } +static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_des3_engine *des3 = &cmd->u.des3; + + struct ccp_dm_workarea key, ctx; + struct ccp_data src, dst; + struct ccp_op op; + unsigned int dm_offset; + unsigned int len_singlekey; + bool in_place = false; + int ret; + + /* Error checks */ + if (!cmd_q->ccp->vdata->perform->des3) + return -EINVAL; + + if (des3->key_len != DES3_EDE_KEY_SIZE) + return -EINVAL; + + if (((des3->mode == CCP_DES3_MODE_ECB) || + (des3->mode == CCP_DES3_MODE_CBC)) && + (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1))) + return -EINVAL; + + if (!des3->key || !des3->src || !des3->dst) + return -EINVAL; + + if (des3->mode != CCP_DES3_MODE_ECB) { + if (des3->iv_len != DES3_EDE_BLOCK_SIZE) + return -EINVAL; + + if (!des3->iv) + return -EINVAL; + } + + ret = -EIO; + /* Zero out all the fields of the command desc */ + memset(&op, 0, sizeof(op)); + + /* Set up the Function field */ + op.cmd_q = cmd_q; + op.jobid = CCP_NEW_JOBID(cmd_q->ccp); + op.sb_key = cmd_q->sb_key; + + op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1; + op.u.des3.type = des3->type; + op.u.des3.mode = des3->mode; + op.u.des3.action = des3->action; + + /* + * All supported key sizes fit in a single (32-byte) KSB entry and + * (like AES) must be in little endian format. Use the 256-bit byte + * swap passthru option to convert from big endian to little endian. + */ + ret = ccp_init_dm_workarea(&key, cmd_q, + CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + /* + * The contents of the key triplet are in the reverse order of what + * is required by the engine. Copy the 3 pieces individually to put + * them where they belong. + */ + dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */ + + len_singlekey = des3->key_len / 3; + ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey, + des3->key, 0, len_singlekey); + ccp_set_dm_area(&key, dm_offset + len_singlekey, + des3->key, len_singlekey, len_singlekey); + ccp_set_dm_area(&key, dm_offset, + des3->key, 2 * len_singlekey, len_singlekey); + + /* Copy the key to the SB */ + ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_key; + } + + /* + * The DES3 context fits in a single (32-byte) KSB entry and + * must be in little endian format. Use the 256-bit byte swap + * passthru option to convert from big endian to little endian. + */ + if (des3->mode != CCP_DES3_MODE_ECB) { + u32 load_mode; + + op.sb_ctx = cmd_q->sb_ctx; + + ret = ccp_init_dm_workarea(&ctx, cmd_q, + CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + goto e_key; + + /* Load the context into the LSB */ + dm_offset = CCP_SB_BYTES - des3->iv_len; + ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0, des3->iv_len); + + if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) + load_mode = CCP_PASSTHRU_BYTESWAP_NOOP; + else + load_mode = CCP_PASSTHRU_BYTESWAP_256BIT; + ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + load_mode); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + } + + /* + * Prepare the input and output data workareas. For in-place + * operations we need to set the dma direction to BIDIRECTIONAL + * and copy the src workarea to the dst workarea. + */ + if (sg_virt(des3->src) == sg_virt(des3->dst)) + in_place = true; + + ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len, + DES3_EDE_BLOCK_SIZE, + in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + if (in_place) + dst = src; + else { + ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len, + DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE); + if (ret) + goto e_src; + } + + /* Send data to the CCP DES3 engine */ + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true); + if (!src.sg_wa.bytes_left) { + op.eom = 1; + + /* Since we don't retrieve the context in ECB mode + * we have to wait for the operation to complete + * on the last piece of data + */ + op.soc = 0; + } + + ret = cmd_q->ccp->vdata->perform->des3(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ccp_process_data(&src, &dst, &op); + } + + if (des3->mode != CCP_DES3_MODE_ECB) { + /* Retrieve the context and make BE */ + ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */ + if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) + dm_offset = CCP_SB_BYTES - des3->iv_len; + else + dm_offset = 0; + ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0, + DES3_EDE_BLOCK_SIZE); + } +e_dst: + if (!in_place) + ccp_free_data(&dst, cmd_q); + +e_src: + ccp_free_data(&src, cmd_q); + +e_ctx: + if (des3->mode != CCP_DES3_MODE_ECB) + ccp_dm_free(&ctx); + +e_key: + ccp_dm_free(&key); + + return ret; +} + static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) { struct ccp_sha_engine *sha = &cmd->u.sha; @@ -1903,6 +2098,9 @@ int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) case CCP_ENGINE_XTS_AES_128: ret = ccp_run_xts_aes_cmd(cmd_q, cmd); break; + case CCP_ENGINE_DES3: + ret = ccp_run_des3_cmd(cmd_q, cmd); + break; case CCP_ENGINE_SHA: ret = ccp_run_sha_cmd(cmd_q, cmd); break; diff --git a/include/linux/ccp.h b/include/linux/ccp.h index 90a1fbe84219..fa0261748920 100644 --- a/include/linux/ccp.h +++ b/include/linux/ccp.h @@ -292,6 +292,60 @@ struct ccp_sha_engine { * final sha cmd */ }; +/***** 3DES engine *****/ +enum ccp_des3_mode { + CCP_DES3_MODE_ECB = 0, + CCP_DES3_MODE_CBC, + CCP_DES3_MODE_CFB, + CCP_DES3_MODE__LAST, +}; + +enum ccp_des3_type { + CCP_DES3_TYPE_168 = 1, + CCP_DES3_TYPE__LAST, + }; + +enum ccp_des3_action { + CCP_DES3_ACTION_DECRYPT = 0, + CCP_DES3_ACTION_ENCRYPT, + CCP_DES3_ACTION__LAST, +}; + +/** + * struct ccp_des3_engine - CCP SHA operation + * @type: Type of 3DES operation + * @mode: cipher mode + * @action: 3DES operation (decrypt/encrypt) + * @key: key to be used for this 3DES operation + * @key_len: length of key (in bytes) + * @iv: IV to be used for this AES operation + * @iv_len: length in bytes of iv + * @src: input data to be used for this operation + * @src_len: length of input data used for this operation (in bytes) + * @dst: output data produced by this operation + * + * Variables required to be set when calling ccp_enqueue_cmd(): + * - type, mode, action, key, key_len, src, dst, src_len + * - iv, iv_len for any mode other than ECB + * + * The iv variable is used as both input and output. On completion of the + * 3DES operation the new IV overwrites the old IV. + */ +struct ccp_des3_engine { + enum ccp_des3_type type; + enum ccp_des3_mode mode; + enum ccp_des3_action action; + + struct scatterlist *key; + u32 key_len; /* In bytes */ + + struct scatterlist *iv; + u32 iv_len; /* In bytes */ + + struct scatterlist *src, *dst; + u64 src_len; /* In bytes */ +}; + /***** RSA engine *****/ /** * struct ccp_rsa_engine - CCP RSA operation @@ -541,7 +595,7 @@ struct ccp_ecc_engine { enum ccp_engine { CCP_ENGINE_AES = 0, CCP_ENGINE_XTS_AES_128, - CCP_ENGINE_RSVD1, + CCP_ENGINE_DES3, CCP_ENGINE_SHA, CCP_ENGINE_RSA, CCP_ENGINE_PASSTHRU, @@ -589,6 +643,7 @@ struct ccp_cmd { union { struct ccp_aes_engine aes; struct ccp_xts_aes_engine xts; + struct ccp_des3_engine des3; struct ccp_sha_engine sha; struct ccp_rsa_engine rsa; struct ccp_passthru_engine passthru; From 36cf515b9bbe298e1ce7384620f0d4ec45ad3328 Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Wed, 15 Mar 2017 13:21:01 -0500 Subject: [PATCH 036/101] crypto: ccp - Enable support for AES GCM on v5 CCPs A version 5 device provides the primitive commands required for AES GCM. This patch adds support for en/decryption. Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/Makefile | 1 + drivers/crypto/ccp/ccp-crypto-aes-galois.c | 252 +++++++++++++++++++++ drivers/crypto/ccp/ccp-crypto-main.c | 12 + drivers/crypto/ccp/ccp-crypto.h | 14 ++ drivers/crypto/ccp/ccp-ops.c | 252 +++++++++++++++++++++ include/linux/ccp.h | 9 + 6 files changed, 540 insertions(+) create mode 100644 drivers/crypto/ccp/ccp-crypto-aes-galois.c diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile index d2044b793aa8..60919a3ec53b 100644 --- a/drivers/crypto/ccp/Makefile +++ b/drivers/crypto/ccp/Makefile @@ -12,5 +12,6 @@ ccp-crypto-objs := ccp-crypto-main.o \ ccp-crypto-aes.o \ ccp-crypto-aes-cmac.o \ ccp-crypto-aes-xts.o \ + ccp-crypto-aes-galois.o \ ccp-crypto-des3.o \ ccp-crypto-sha.o diff --git a/drivers/crypto/ccp/ccp-crypto-aes-galois.c b/drivers/crypto/ccp/ccp-crypto-aes-galois.c new file mode 100644 index 000000000000..38ee6f348ea9 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-aes-galois.c @@ -0,0 +1,252 @@ +/* + * AMD Cryptographic Coprocessor (CCP) AES GCM crypto API support + * + * Copyright (C) 2016 Advanced Micro Devices, Inc. + * + * Author: Gary R Hook + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ccp-crypto.h" + +#define AES_GCM_IVSIZE 12 + +static int ccp_aes_gcm_complete(struct crypto_async_request *async_req, int ret) +{ + return ret; +} + +static int ccp_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_aead_ctx(tfm); + + switch (key_len) { + case AES_KEYSIZE_128: + ctx->u.aes.type = CCP_AES_TYPE_128; + break; + case AES_KEYSIZE_192: + ctx->u.aes.type = CCP_AES_TYPE_192; + break; + case AES_KEYSIZE_256: + ctx->u.aes.type = CCP_AES_TYPE_256; + break; + default: + crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + ctx->u.aes.mode = CCP_AES_MODE_GCM; + ctx->u.aes.key_len = key_len; + + memcpy(ctx->u.aes.key, key, key_len); + sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); + + return 0; +} + +static int ccp_aes_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + return 0; +} + +static int ccp_aes_gcm_crypt(struct aead_request *req, bool encrypt) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct ccp_ctx *ctx = crypto_aead_ctx(tfm); + struct ccp_aes_req_ctx *rctx = aead_request_ctx(req); + struct scatterlist *iv_sg = NULL; + unsigned int iv_len = 0; + int i; + int ret = 0; + + if (!ctx->u.aes.key_len) + return -EINVAL; + + if (ctx->u.aes.mode != CCP_AES_MODE_GCM) + return -EINVAL; + + if (!req->iv) + return -EINVAL; + + /* + * 5 parts: + * plaintext/ciphertext input + * AAD + * key + * IV + * Destination+tag buffer + */ + + /* Prepare the IV: 12 bytes + an integer (counter) */ + memcpy(rctx->iv, req->iv, AES_GCM_IVSIZE); + for (i = 0; i < 3; i++) + rctx->iv[i + AES_GCM_IVSIZE] = 0; + rctx->iv[AES_BLOCK_SIZE - 1] = 1; + + /* Set up a scatterlist for the IV */ + iv_sg = &rctx->iv_sg; + iv_len = AES_BLOCK_SIZE; + sg_init_one(iv_sg, rctx->iv, iv_len); + + /* The AAD + plaintext are concatenated in the src buffer */ + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_AES; + rctx->cmd.u.aes.type = ctx->u.aes.type; + rctx->cmd.u.aes.mode = ctx->u.aes.mode; + rctx->cmd.u.aes.action = encrypt; + rctx->cmd.u.aes.key = &ctx->u.aes.key_sg; + rctx->cmd.u.aes.key_len = ctx->u.aes.key_len; + rctx->cmd.u.aes.iv = iv_sg; + rctx->cmd.u.aes.iv_len = iv_len; + rctx->cmd.u.aes.src = req->src; + rctx->cmd.u.aes.src_len = req->cryptlen; + rctx->cmd.u.aes.aad_len = req->assoclen; + + /* The cipher text + the tag are in the dst buffer */ + rctx->cmd.u.aes.dst = req->dst; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; +} + +static int ccp_aes_gcm_encrypt(struct aead_request *req) +{ + return ccp_aes_gcm_crypt(req, CCP_AES_ACTION_ENCRYPT); +} + +static int ccp_aes_gcm_decrypt(struct aead_request *req) +{ + return ccp_aes_gcm_crypt(req, CCP_AES_ACTION_DECRYPT); +} + +static int ccp_aes_gcm_cra_init(struct crypto_aead *tfm) +{ + struct ccp_ctx *ctx = crypto_aead_ctx(tfm); + + ctx->complete = ccp_aes_gcm_complete; + ctx->u.aes.key_len = 0; + + crypto_aead_set_reqsize(tfm, sizeof(struct ccp_aes_req_ctx)); + + return 0; +} + +static void ccp_aes_gcm_cra_exit(struct crypto_tfm *tfm) +{ +} + +static struct aead_alg ccp_aes_gcm_defaults = { + .setkey = ccp_aes_gcm_setkey, + .setauthsize = ccp_aes_gcm_setauthsize, + .encrypt = ccp_aes_gcm_encrypt, + .decrypt = ccp_aes_gcm_decrypt, + .init = ccp_aes_gcm_cra_init, + .ivsize = AES_GCM_IVSIZE, + .maxauthsize = AES_BLOCK_SIZE, + .base = { + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ccp_ctx), + .cra_priority = CCP_CRA_PRIORITY, + .cra_type = &crypto_ablkcipher_type, + .cra_exit = ccp_aes_gcm_cra_exit, + .cra_module = THIS_MODULE, + }, +}; + +struct ccp_aes_aead_def { + enum ccp_aes_mode mode; + unsigned int version; + const char *name; + const char *driver_name; + unsigned int blocksize; + unsigned int ivsize; + struct aead_alg *alg_defaults; +}; + +static struct ccp_aes_aead_def aes_aead_algs[] = { + { + .mode = CCP_AES_MODE_GHASH, + .version = CCP_VERSION(5, 0), + .name = "gcm(aes)", + .driver_name = "gcm-aes-ccp", + .blocksize = 1, + .ivsize = AES_BLOCK_SIZE, + .alg_defaults = &ccp_aes_gcm_defaults, + }, +}; + +static int ccp_register_aes_aead(struct list_head *head, + const struct ccp_aes_aead_def *def) +{ + struct ccp_crypto_aead *ccp_aead; + struct aead_alg *alg; + int ret; + + ccp_aead = kzalloc(sizeof(*ccp_aead), GFP_KERNEL); + if (!ccp_aead) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_aead->entry); + + ccp_aead->mode = def->mode; + + /* Copy the defaults and override as necessary */ + alg = &ccp_aead->alg; + *alg = *def->alg_defaults; + snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name); + snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + def->driver_name); + alg->base.cra_blocksize = def->blocksize; + alg->base.cra_ablkcipher.ivsize = def->ivsize; + + ret = crypto_register_aead(alg); + if (ret) { + pr_err("%s ablkcipher algorithm registration error (%d)\n", + alg->base.cra_name, ret); + kfree(ccp_aead); + return ret; + } + + list_add(&ccp_aead->entry, head); + + return 0; +} + +int ccp_register_aes_aeads(struct list_head *head) +{ + int i, ret; + unsigned int ccpversion = ccp_version(); + + for (i = 0; i < ARRAY_SIZE(aes_aead_algs); i++) { + if (aes_aead_algs[i].version > ccpversion) + continue; + ret = ccp_register_aes_aead(head, &aes_aead_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c index 3f1e36d7a8bf..8dccbddabef1 100644 --- a/drivers/crypto/ccp/ccp-crypto-main.c +++ b/drivers/crypto/ccp/ccp-crypto-main.c @@ -40,6 +40,7 @@ MODULE_PARM_DESC(des3_disable, "Disable use of 3DES - any non-zero value"); /* List heads for the supported algorithms */ static LIST_HEAD(hash_algs); static LIST_HEAD(cipher_algs); +static LIST_HEAD(aead_algs); /* For any tfm, requests for that tfm must be returned on the order * received. With multiple queues available, the CCP can process more @@ -339,6 +340,10 @@ static int ccp_register_algs(void) ret = ccp_register_aes_xts_algs(&cipher_algs); if (ret) return ret; + + ret = ccp_register_aes_aeads(&aead_algs); + if (ret) + return ret; } if (!des3_disable) { @@ -360,6 +365,7 @@ static void ccp_unregister_algs(void) { struct ccp_crypto_ahash_alg *ahash_alg, *ahash_tmp; struct ccp_crypto_ablkcipher_alg *ablk_alg, *ablk_tmp; + struct ccp_crypto_aead *aead_alg, *aead_tmp; list_for_each_entry_safe(ahash_alg, ahash_tmp, &hash_algs, entry) { crypto_unregister_ahash(&ahash_alg->alg); @@ -372,6 +378,12 @@ static void ccp_unregister_algs(void) list_del(&ablk_alg->entry); kfree(ablk_alg); } + + list_for_each_entry_safe(aead_alg, aead_tmp, &aead_algs, entry) { + crypto_unregister_aead(&aead_alg->alg); + list_del(&aead_alg->entry); + kfree(aead_alg); + } } static int ccp_crypto_init(void) diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h index 8c8bd3ff9f48..dd5bf15f06e5 100644 --- a/drivers/crypto/ccp/ccp-crypto.h +++ b/drivers/crypto/ccp/ccp-crypto.h @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include #include #include @@ -35,6 +37,14 @@ struct ccp_crypto_ablkcipher_alg { struct crypto_alg alg; }; +struct ccp_crypto_aead { + struct list_head entry; + + u32 mode; + + struct aead_alg alg; +}; + struct ccp_crypto_ahash_alg { struct list_head entry; @@ -97,6 +107,9 @@ struct ccp_aes_req_ctx { struct scatterlist iv_sg; u8 iv[AES_BLOCK_SIZE]; + struct scatterlist tag_sg; + u8 tag[AES_BLOCK_SIZE]; + /* Fields used for RFC3686 requests */ u8 *rfc3686_info; u8 rfc3686_iv[AES_BLOCK_SIZE]; @@ -233,6 +246,7 @@ struct scatterlist *ccp_crypto_sg_table_add(struct sg_table *table, int ccp_register_aes_algs(struct list_head *head); int ccp_register_aes_cmac_algs(struct list_head *head); int ccp_register_aes_xts_algs(struct list_head *head); +int ccp_register_aes_aeads(struct list_head *head); int ccp_register_sha_algs(struct list_head *head); int ccp_register_des3_algs(struct list_head *head); diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index 0de961a22ab4..c0dfdacbdff5 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -601,6 +601,255 @@ e_key: return ret; } +static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, + struct ccp_cmd *cmd) +{ + struct ccp_aes_engine *aes = &cmd->u.aes; + struct ccp_dm_workarea key, ctx, final_wa, tag; + struct ccp_data src, dst; + struct ccp_data aad; + struct ccp_op op; + + unsigned long long *final; + unsigned int dm_offset; + unsigned int ilen; + bool in_place = true; /* Default value */ + int ret; + + struct scatterlist *p_inp, sg_inp[2]; + struct scatterlist *p_tag, sg_tag[2]; + struct scatterlist *p_outp, sg_outp[2]; + struct scatterlist *p_aad; + + if (!aes->iv) + return -EINVAL; + + if (!((aes->key_len == AES_KEYSIZE_128) || + (aes->key_len == AES_KEYSIZE_192) || + (aes->key_len == AES_KEYSIZE_256))) + return -EINVAL; + + if (!aes->key) /* Gotta have a key SGL */ + return -EINVAL; + + /* First, decompose the source buffer into AAD & PT, + * and the destination buffer into AAD, CT & tag, or + * the input into CT & tag. + * It is expected that the input and output SGs will + * be valid, even if the AAD and input lengths are 0. + */ + p_aad = aes->src; + p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len); + p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len); + if (aes->action == CCP_AES_ACTION_ENCRYPT) { + ilen = aes->src_len; + p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen); + } else { + /* Input length for decryption includes tag */ + ilen = aes->src_len - AES_BLOCK_SIZE; + p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen); + } + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = CCP_NEW_JOBID(cmd_q->ccp); + op.sb_key = cmd_q->sb_key; /* Pre-allocated */ + op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ + op.init = 1; + op.u.aes.type = aes->type; + + /* Copy the key to the LSB */ + ret = ccp_init_dm_workarea(&key, cmd_q, + CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + dm_offset = CCP_SB_BYTES - aes->key_len; + ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); + ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_key; + } + + /* Copy the context (IV) to the LSB. + * There is an assumption here that the IV is 96 bits in length, plus + * a nonce of 32 bits. If no IV is present, use a zeroed buffer. + */ + ret = ccp_init_dm_workarea(&ctx, cmd_q, + CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + goto e_key; + + dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len; + ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + + ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + + op.init = 1; + if (aes->aad_len > 0) { + /* Step 1: Run a GHASH over the Additional Authenticated Data */ + ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len, + AES_BLOCK_SIZE, + DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + op.u.aes.mode = CCP_AES_MODE_GHASH; + op.u.aes.action = CCP_AES_GHASHAAD; + + while (aad.sg_wa.bytes_left) { + ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true); + + ret = cmd_q->ccp->vdata->perform->aes(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_aad; + } + + ccp_process_data(&aad, NULL, &op); + op.init = 0; + } + } + + op.u.aes.mode = CCP_AES_MODE_GCTR; + op.u.aes.action = aes->action; + + if (ilen > 0) { + /* Step 2: Run a GCTR over the plaintext */ + in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false; + + ret = ccp_init_data(&src, cmd_q, p_inp, ilen, + AES_BLOCK_SIZE, + in_place ? DMA_BIDIRECTIONAL + : DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + if (in_place) { + dst = src; + } else { + ret = ccp_init_data(&dst, cmd_q, p_outp, ilen, + AES_BLOCK_SIZE, DMA_FROM_DEVICE); + if (ret) + goto e_src; + } + + op.soc = 0; + op.eom = 0; + op.init = 1; + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); + if (!src.sg_wa.bytes_left) { + unsigned int nbytes = aes->src_len + % AES_BLOCK_SIZE; + + if (nbytes) { + op.eom = 1; + op.u.aes.size = (nbytes * 8) - 1; + } + } + + ret = cmd_q->ccp->vdata->perform->aes(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ccp_process_data(&src, &dst, &op); + op.init = 0; + } + } + + /* Step 3: Update the IV portion of the context with the original IV */ + ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + + ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + /* Step 4: Concatenate the lengths of the AAD and source, and + * hash that 16 byte buffer. + */ + ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE, + DMA_BIDIRECTIONAL); + if (ret) + goto e_dst; + final = (unsigned long long *) final_wa.address; + final[0] = cpu_to_be64(aes->aad_len * 8); + final[1] = cpu_to_be64(ilen * 8); + + op.u.aes.mode = CCP_AES_MODE_GHASH; + op.u.aes.action = CCP_AES_GHASHFINAL; + op.src.type = CCP_MEMTYPE_SYSTEM; + op.src.u.dma.address = final_wa.dma.address; + op.src.u.dma.length = AES_BLOCK_SIZE; + op.dst.type = CCP_MEMTYPE_SYSTEM; + op.dst.u.dma.address = final_wa.dma.address; + op.dst.u.dma.length = AES_BLOCK_SIZE; + op.eom = 1; + op.u.aes.size = 0; + ret = cmd_q->ccp->vdata->perform->aes(&op); + if (ret) + goto e_dst; + + if (aes->action == CCP_AES_ACTION_ENCRYPT) { + /* Put the ciphered tag after the ciphertext. */ + ccp_get_dm_area(&final_wa, 0, p_tag, 0, AES_BLOCK_SIZE); + } else { + /* Does this ciphered tag match the input? */ + ret = ccp_init_dm_workarea(&tag, cmd_q, AES_BLOCK_SIZE, + DMA_BIDIRECTIONAL); + if (ret) + goto e_tag; + ccp_set_dm_area(&tag, 0, p_tag, 0, AES_BLOCK_SIZE); + + ret = memcmp(tag.address, final_wa.address, AES_BLOCK_SIZE); + ccp_dm_free(&tag); + } + +e_tag: + ccp_dm_free(&final_wa); + +e_dst: + if (aes->src_len && !in_place) + ccp_free_data(&dst, cmd_q); + +e_src: + if (aes->src_len) + ccp_free_data(&src, cmd_q); + +e_aad: + if (aes->aad_len) + ccp_free_data(&aad, cmd_q); + +e_ctx: + ccp_dm_free(&ctx); + +e_key: + ccp_dm_free(&key); + + return ret; +} + static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) { struct ccp_aes_engine *aes = &cmd->u.aes; @@ -614,6 +863,9 @@ static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) if (aes->mode == CCP_AES_MODE_CMAC) return ccp_run_aes_cmac_cmd(cmd_q, cmd); + if (aes->mode == CCP_AES_MODE_GCM) + return ccp_run_aes_gcm_cmd(cmd_q, cmd); + if (!((aes->key_len == AES_KEYSIZE_128) || (aes->key_len == AES_KEYSIZE_192) || (aes->key_len == AES_KEYSIZE_256))) diff --git a/include/linux/ccp.h b/include/linux/ccp.h index fa0261748920..dbe8aa0f2940 100644 --- a/include/linux/ccp.h +++ b/include/linux/ccp.h @@ -123,6 +123,10 @@ enum ccp_aes_mode { CCP_AES_MODE_CFB, CCP_AES_MODE_CTR, CCP_AES_MODE_CMAC, + CCP_AES_MODE_GHASH, + CCP_AES_MODE_GCTR, + CCP_AES_MODE_GCM, + CCP_AES_MODE_GMAC, CCP_AES_MODE__LAST, }; @@ -137,6 +141,9 @@ enum ccp_aes_action { CCP_AES_ACTION_ENCRYPT, CCP_AES_ACTION__LAST, }; +/* Overloaded field */ +#define CCP_AES_GHASHAAD CCP_AES_ACTION_DECRYPT +#define CCP_AES_GHASHFINAL CCP_AES_ACTION_ENCRYPT /** * struct ccp_aes_engine - CCP AES operation @@ -181,6 +188,8 @@ struct ccp_aes_engine { struct scatterlist *cmac_key; /* K1/K2 cmac key required for * final cmac cmd */ u32 cmac_key_len; /* In bytes */ + + u32 aad_len; /* In bytes */ }; /***** XTS-AES engine *****/ From fdb2726f4e61c5e3abc052f547d5a5f6c0dc5504 Mon Sep 17 00:00:00 2001 From: Michael Davidson Date: Wed, 15 Mar 2017 15:36:00 -0700 Subject: [PATCH 037/101] crypto, x86: aesni - fix token pasting for clang aes_ctrby8_avx-x86_64.S uses the C preprocessor for token pasting of character sequences that are not valid preprocessor tokens. While this is allowed when preprocessing assembler files it exposes an incompatibilty between the clang and gcc preprocessors where clang does not strip leading white space from macro parameters, leading to the CONCAT(%xmm, i) macro expansion on line 96 resulting in a token with a space character embedded in it. While this could be resolved by deleting the offending space character, the assembler is perfectly capable of doing the token pasting correctly for itself so we can just get rid of the preprocessor macros. Signed-off-by: Michael Davidson Signed-off-by: Herbert Xu --- arch/x86/crypto/aes_ctrby8_avx-x86_64.S | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S index a916c4a61165..5f6a5af9c489 100644 --- a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S +++ b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S @@ -65,7 +65,6 @@ #include #include -#define CONCAT(a,b) a##b #define VMOVDQ vmovdqu #define xdata0 %xmm0 @@ -92,8 +91,6 @@ #define num_bytes %r8 #define tmp %r10 -#define DDQ(i) CONCAT(ddq_add_,i) -#define XMM(i) CONCAT(%xmm, i) #define DDQ_DATA 0 #define XDATA 1 #define KEY_128 1 @@ -131,12 +128,12 @@ ddq_add_8: /* generate a unique variable for ddq_add_x */ .macro setddq n - var_ddq_add = DDQ(\n) + var_ddq_add = ddq_add_\n .endm /* generate a unique variable for xmm register */ .macro setxdata n - var_xdata = XMM(\n) + var_xdata = %xmm\n .endm /* club the numeric 'id' to the symbol 'name' */ From 3c7eb3cc8360736123a3139a1ec727d746de3252 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Thu, 16 Mar 2017 15:18:57 +0100 Subject: [PATCH 038/101] md5: remove from lib and only live in crypto The md5_transform function is no longer used any where in the tree, except for the crypto api's actual implementation of md5, so we can drop the function from lib and put it as a static function of the crypto file, where it belongs. There should be no new users of md5_transform, anyway, since there are more modern ways of doing what it once achieved. Signed-off-by: Jason A. Donenfeld Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/md5.c | 95 +++++++++++++++++++++++++++++++++++++- include/linux/cryptohash.h | 5 -- lib/Makefile | 2 +- lib/md5.c | 95 -------------------------------------- 4 files changed, 95 insertions(+), 102 deletions(-) delete mode 100644 lib/md5.c diff --git a/crypto/md5.c b/crypto/md5.c index 2355a7c25c45..f7ae1a48225b 100644 --- a/crypto/md5.c +++ b/crypto/md5.c @@ -21,9 +21,11 @@ #include #include #include -#include #include +#define MD5_DIGEST_WORDS 4 +#define MD5_MESSAGE_BYTES 64 + const u8 md5_zero_message_hash[MD5_DIGEST_SIZE] = { 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e, @@ -47,6 +49,97 @@ static inline void cpu_to_le32_array(u32 *buf, unsigned int words) } } +#define F1(x, y, z) (z ^ (x & (y ^ z))) +#define F2(x, y, z) F1(z, x, y) +#define F3(x, y, z) (x ^ y ^ z) +#define F4(x, y, z) (y ^ (x | ~z)) + +#define MD5STEP(f, w, x, y, z, in, s) \ + (w += f(x, y, z) + in, w = (w<>(32-s)) + x) + +static void md5_transform(__u32 *hash, __u32 const *in) +{ + u32 a, b, c, d; + + a = hash[0]; + b = hash[1]; + c = hash[2]; + d = hash[3]; + + MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7); + MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12); + MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17); + MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22); + MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7); + MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12); + MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17); + MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22); + MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7); + MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12); + MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17); + MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22); + MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7); + MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12); + MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17); + MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22); + + MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5); + MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9); + MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14); + MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20); + MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5); + MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9); + MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14); + MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20); + MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5); + MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9); + MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14); + MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20); + MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5); + MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9); + MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14); + MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20); + + MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4); + MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11); + MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16); + MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23); + MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4); + MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11); + MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16); + MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23); + MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4); + MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11); + MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16); + MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23); + MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4); + MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11); + MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16); + MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23); + + MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6); + MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10); + MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15); + MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21); + MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6); + MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10); + MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15); + MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21); + MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6); + MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10); + MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15); + MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21); + MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6); + MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10); + MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15); + MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21); + + hash[0] += a; + hash[1] += b; + hash[2] += c; + hash[3] += d; +} + static inline void md5_transform_helper(struct md5_state *ctx) { le32_to_cpu_array(ctx->block, sizeof(ctx->block) / sizeof(u32)); diff --git a/include/linux/cryptohash.h b/include/linux/cryptohash.h index 3252799832cf..df4d3e943d28 100644 --- a/include/linux/cryptohash.h +++ b/include/linux/cryptohash.h @@ -10,9 +10,4 @@ void sha_init(__u32 *buf); void sha_transform(__u32 *digest, const char *data, __u32 *W); -#define MD5_DIGEST_WORDS 4 -#define MD5_MESSAGE_BYTES 64 - -void md5_transform(__u32 *hash, __u32 const *in); - #endif diff --git a/lib/Makefile b/lib/Makefile index 320ac46a8725..acbc16bed9af 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -19,7 +19,7 @@ KCOV_INSTRUMENT_dynamic_debug.o := n lib-y := ctype.o string.o vsprintf.o cmdline.o \ rbtree.o radix-tree.o dump_stack.o timerqueue.o\ idr.o int_sqrt.o extable.o \ - sha1.o chacha20.o md5.o irq_regs.o argv_split.o \ + sha1.o chacha20.o irq_regs.o argv_split.o \ flex_proportions.o ratelimit.o show_mem.o \ is_single_threaded.o plist.o decompress.o kobject_uevent.o \ earlycpio.o seq_buf.o siphash.o \ diff --git a/lib/md5.c b/lib/md5.c deleted file mode 100644 index bb0cd01d356d..000000000000 --- a/lib/md5.c +++ /dev/null @@ -1,95 +0,0 @@ -#include -#include -#include - -#define F1(x, y, z) (z ^ (x & (y ^ z))) -#define F2(x, y, z) F1(z, x, y) -#define F3(x, y, z) (x ^ y ^ z) -#define F4(x, y, z) (y ^ (x | ~z)) - -#define MD5STEP(f, w, x, y, z, in, s) \ - (w += f(x, y, z) + in, w = (w<>(32-s)) + x) - -void md5_transform(__u32 *hash, __u32 const *in) -{ - u32 a, b, c, d; - - a = hash[0]; - b = hash[1]; - c = hash[2]; - d = hash[3]; - - MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7); - MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12); - MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17); - MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22); - MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7); - MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12); - MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17); - MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22); - MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7); - MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12); - MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17); - MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22); - MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7); - MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12); - MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17); - MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22); - - MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5); - MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9); - MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14); - MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20); - MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5); - MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9); - MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14); - MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20); - MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5); - MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9); - MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14); - MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20); - MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5); - MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9); - MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14); - MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20); - - MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4); - MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11); - MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16); - MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23); - MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4); - MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11); - MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16); - MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23); - MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4); - MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11); - MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16); - MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23); - MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4); - MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11); - MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16); - MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23); - - MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6); - MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10); - MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15); - MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21); - MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6); - MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10); - MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15); - MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21); - MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6); - MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10); - MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15); - MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21); - MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6); - MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10); - MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15); - MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21); - - hash[0] += a; - hash[1] += b; - hash[2] += c; - hash[3] += d; -} -EXPORT_SYMBOL(md5_transform); From 1662e9319b999402d450a2dbaefc49752986a46b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 17 Mar 2017 12:05:56 +0200 Subject: [PATCH 039/101] soc/qman: export volatile dequeue related structs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since qman_volatile_dequeue() is already exported, move the related structures into the public header too. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/soc/fsl/qbman/qman_priv.h | 36 ------------------------------- include/soc/fsl/qman.h | 36 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/drivers/soc/fsl/qbman/qman_priv.h b/drivers/soc/fsl/qbman/qman_priv.h index 53685b59718e..64781eff6974 100644 --- a/drivers/soc/fsl/qbman/qman_priv.h +++ b/drivers/soc/fsl/qbman/qman_priv.h @@ -271,42 +271,6 @@ const struct qm_portal_config *qman_destroy_affine_portal(void); */ int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd); -/* - * For qman_volatile_dequeue(); Choose one PRECEDENCE. EXACT is optional. Use - * NUMFRAMES(n) (6-bit) or NUMFRAMES_TILLEMPTY to fill in the frame-count. Use - * FQID(n) to fill in the frame queue ID. - */ -#define QM_VDQCR_PRECEDENCE_VDQCR 0x0 -#define QM_VDQCR_PRECEDENCE_SDQCR 0x80000000 -#define QM_VDQCR_EXACT 0x40000000 -#define QM_VDQCR_NUMFRAMES_MASK 0x3f000000 -#define QM_VDQCR_NUMFRAMES_SET(n) (((n) & 0x3f) << 24) -#define QM_VDQCR_NUMFRAMES_GET(n) (((n) >> 24) & 0x3f) -#define QM_VDQCR_NUMFRAMES_TILLEMPTY QM_VDQCR_NUMFRAMES_SET(0) - -#define QMAN_VOLATILE_FLAG_WAIT 0x00000001 /* wait if VDQCR is in use */ -#define QMAN_VOLATILE_FLAG_WAIT_INT 0x00000002 /* if wait, interruptible? */ -#define QMAN_VOLATILE_FLAG_FINISH 0x00000004 /* wait till VDQCR completes */ - -/* - * qman_volatile_dequeue - Issue a volatile dequeue command - * @fq: the frame queue object to dequeue from - * @flags: a bit-mask of QMAN_VOLATILE_FLAG_*** options - * @vdqcr: bit mask of QM_VDQCR_*** options, as per qm_dqrr_vdqcr_set() - * - * Attempts to lock access to the portal's VDQCR volatile dequeue functionality. - * The function will block and sleep if QMAN_VOLATILE_FLAG_WAIT is specified and - * the VDQCR is already in use, otherwise returns non-zero for failure. If - * QMAN_VOLATILE_FLAG_FINISH is specified, the function will only return once - * the VDQCR command has finished executing (ie. once the callback for the last - * DQRR entry resulting from the VDQCR command has been called). If not using - * the FINISH flag, completion can be determined either by detecting the - * presence of the QM_DQRR_STAT_UNSCHEDULED and QM_DQRR_STAT_DQCR_EXPIRED bits - * in the "stat" parameter passed to the FQ's dequeue callback, or by waiting - * for the QMAN_FQ_STATE_VDQCR bit to disappear. - */ -int qman_volatile_dequeue(struct qman_fq *fq, u32 flags, u32 vdqcr); - int qman_alloc_fq_table(u32 num_fqids); /* QMan s/w corenet portal, low-level i/face */ diff --git a/include/soc/fsl/qman.h b/include/soc/fsl/qman.h index 3d4df74a96de..4de1ffcc8982 100644 --- a/include/soc/fsl/qman.h +++ b/include/soc/fsl/qman.h @@ -791,6 +791,23 @@ struct qman_cgr { #define QMAN_INITFQ_FLAG_SCHED 0x00000001 /* schedule rather than park */ #define QMAN_INITFQ_FLAG_LOCAL 0x00000004 /* set dest portal */ +/* + * For qman_volatile_dequeue(); Choose one PRECEDENCE. EXACT is optional. Use + * NUMFRAMES(n) (6-bit) or NUMFRAMES_TILLEMPTY to fill in the frame-count. Use + * FQID(n) to fill in the frame queue ID. + */ +#define QM_VDQCR_PRECEDENCE_VDQCR 0x0 +#define QM_VDQCR_PRECEDENCE_SDQCR 0x80000000 +#define QM_VDQCR_EXACT 0x40000000 +#define QM_VDQCR_NUMFRAMES_MASK 0x3f000000 +#define QM_VDQCR_NUMFRAMES_SET(n) (((n) & 0x3f) << 24) +#define QM_VDQCR_NUMFRAMES_GET(n) (((n) >> 24) & 0x3f) +#define QM_VDQCR_NUMFRAMES_TILLEMPTY QM_VDQCR_NUMFRAMES_SET(0) + +#define QMAN_VOLATILE_FLAG_WAIT 0x00000001 /* wait if VDQCR is in use */ +#define QMAN_VOLATILE_FLAG_WAIT_INT 0x00000002 /* if wait, interruptible? */ +#define QMAN_VOLATILE_FLAG_FINISH 0x00000004 /* wait till VDQCR completes */ + /* Portal Management */ /** * qman_p_irqsource_add - add processing sources to be interrupt-driven @@ -963,6 +980,25 @@ int qman_retire_fq(struct qman_fq *fq, u32 *flags); */ int qman_oos_fq(struct qman_fq *fq); +/* + * qman_volatile_dequeue - Issue a volatile dequeue command + * @fq: the frame queue object to dequeue from + * @flags: a bit-mask of QMAN_VOLATILE_FLAG_*** options + * @vdqcr: bit mask of QM_VDQCR_*** options, as per qm_dqrr_vdqcr_set() + * + * Attempts to lock access to the portal's VDQCR volatile dequeue functionality. + * The function will block and sleep if QMAN_VOLATILE_FLAG_WAIT is specified and + * the VDQCR is already in use, otherwise returns non-zero for failure. If + * QMAN_VOLATILE_FLAG_FINISH is specified, the function will only return once + * the VDQCR command has finished executing (ie. once the callback for the last + * DQRR entry resulting from the VDQCR command has been called). If not using + * the FINISH flag, completion can be determined either by detecting the + * presence of the QM_DQRR_STAT_UNSCHEDULED and QM_DQRR_STAT_DQCR_EXPIRED bits + * in the "stat" parameter passed to the FQ's dequeue callback, or by waiting + * for the QMAN_FQ_STATE_VDQCR bit to disappear. + */ +int qman_volatile_dequeue(struct qman_fq *fq, u32 flags, u32 vdqcr); + /** * qman_enqueue - Enqueue a frame to a frame queue * @fq: the frame queue object to enqueue to From 329d09089c473cd49e131dae1a292e319dc0d91a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 17 Mar 2017 12:05:57 +0200 Subject: [PATCH 040/101] soc/qman: add dedicated channel ID for CAAM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add and export the ID of the channel serviced by the CAAM (Cryptographic Acceleration and Assurance Module) DCP. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/soc/fsl/qbman/qman_ccsr.c | 6 +++++- include/soc/fsl/qman.h | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/soc/fsl/qbman/qman_ccsr.c b/drivers/soc/fsl/qbman/qman_ccsr.c index f4e6e70de259..90bc40c48675 100644 --- a/drivers/soc/fsl/qbman/qman_ccsr.c +++ b/drivers/soc/fsl/qbman/qman_ccsr.c @@ -34,6 +34,8 @@ u16 qman_ip_rev; EXPORT_SYMBOL(qman_ip_rev); u16 qm_channel_pool1 = QMAN_CHANNEL_POOL1; EXPORT_SYMBOL(qm_channel_pool1); +u16 qm_channel_caam = QMAN_CHANNEL_CAAM; +EXPORT_SYMBOL(qm_channel_caam); /* Register offsets */ #define REG_QCSP_LIO_CFG(n) (0x0000 + ((n) * 0x10)) @@ -720,8 +722,10 @@ static int fsl_qman_probe(struct platform_device *pdev) return -ENODEV; } - if ((qman_ip_rev & 0xff00) >= QMAN_REV30) + if ((qman_ip_rev & 0xff00) >= QMAN_REV30) { qm_channel_pool1 = QMAN_CHANNEL_POOL1_REV3; + qm_channel_caam = QMAN_CHANNEL_CAAM_REV3; + } ret = zero_priv_mem(dev, node, fqd_a, fqd_sz); WARN_ON(ret); diff --git a/include/soc/fsl/qman.h b/include/soc/fsl/qman.h index 4de1ffcc8982..10b549783ec5 100644 --- a/include/soc/fsl/qman.h +++ b/include/soc/fsl/qman.h @@ -36,8 +36,11 @@ /* Hardware constants */ #define QM_CHANNEL_SWPORTAL0 0 #define QMAN_CHANNEL_POOL1 0x21 +#define QMAN_CHANNEL_CAAM 0x80 #define QMAN_CHANNEL_POOL1_REV3 0x401 +#define QMAN_CHANNEL_CAAM_REV3 0x840 extern u16 qm_channel_pool1; +extern u16 qm_channel_caam; /* Portal processing (interrupt) sources */ #define QM_PIRQ_CSCI 0x00100000 /* Congestion State Change */ From 8496272d8116295053139a5d309d59694902f043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 17 Mar 2017 12:05:58 +0200 Subject: [PATCH 041/101] soc/qman: export non-programmable FQD fields query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export qman_query_fq_np() function and related structures. This will be needed in the caam/qi driver, where "queue empty" condition will be decided based on the frm_cnt. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/soc/fsl/qbman/qman.c | 4 +- drivers/soc/fsl/qbman/qman_priv.h | 61 --------------------------- include/soc/fsl/qman.h | 68 +++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 63 deletions(-) diff --git a/drivers/soc/fsl/qbman/qman.c b/drivers/soc/fsl/qbman/qman.c index 6f509f68085e..3d891db57ee6 100644 --- a/drivers/soc/fsl/qbman/qman.c +++ b/drivers/soc/fsl/qbman/qman.c @@ -2019,8 +2019,7 @@ out: return ret; } -static int qman_query_fq_np(struct qman_fq *fq, - struct qm_mcr_queryfq_np *np) +int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np) { union qm_mc_command *mcc; union qm_mc_result *mcr; @@ -2046,6 +2045,7 @@ out: put_affine_portal(); return ret; } +EXPORT_SYMBOL(qman_query_fq_np); static int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd) diff --git a/drivers/soc/fsl/qbman/qman_priv.h b/drivers/soc/fsl/qbman/qman_priv.h index 64781eff6974..22725bdc6f15 100644 --- a/drivers/soc/fsl/qbman/qman_priv.h +++ b/drivers/soc/fsl/qbman/qman_priv.h @@ -89,67 +89,6 @@ static inline u64 qm_mcr_querycgr_a_get64(const struct qm_mcr_querycgr *q) return ((u64)q->a_bcnt_hi << 32) | be32_to_cpu(q->a_bcnt_lo); } -/* "Query FQ Non-Programmable Fields" */ - -struct qm_mcr_queryfq_np { - u8 verb; - u8 result; - u8 __reserved1; - u8 state; /* QM_MCR_NP_STATE_*** */ - u32 fqd_link; /* 24-bit, _res2[24-31] */ - u16 odp_seq; /* 14-bit, _res3[14-15] */ - u16 orp_nesn; /* 14-bit, _res4[14-15] */ - u16 orp_ea_hseq; /* 15-bit, _res5[15] */ - u16 orp_ea_tseq; /* 15-bit, _res6[15] */ - u32 orp_ea_hptr; /* 24-bit, _res7[24-31] */ - u32 orp_ea_tptr; /* 24-bit, _res8[24-31] */ - u32 pfdr_hptr; /* 24-bit, _res9[24-31] */ - u32 pfdr_tptr; /* 24-bit, _res10[24-31] */ - u8 __reserved2[5]; - u8 is; /* 1-bit, _res12[1-7] */ - u16 ics_surp; - u32 byte_cnt; - u32 frm_cnt; /* 24-bit, _res13[24-31] */ - u32 __reserved3; - u16 ra1_sfdr; /* QM_MCR_NP_RA1_*** */ - u16 ra2_sfdr; /* QM_MCR_NP_RA2_*** */ - u16 __reserved4; - u16 od1_sfdr; /* QM_MCR_NP_OD1_*** */ - u16 od2_sfdr; /* QM_MCR_NP_OD2_*** */ - u16 od3_sfdr; /* QM_MCR_NP_OD3_*** */ -} __packed; - -#define QM_MCR_NP_STATE_FE 0x10 -#define QM_MCR_NP_STATE_R 0x08 -#define QM_MCR_NP_STATE_MASK 0x07 /* Reads FQD::STATE; */ -#define QM_MCR_NP_STATE_OOS 0x00 -#define QM_MCR_NP_STATE_RETIRED 0x01 -#define QM_MCR_NP_STATE_TEN_SCHED 0x02 -#define QM_MCR_NP_STATE_TRU_SCHED 0x03 -#define QM_MCR_NP_STATE_PARKED 0x04 -#define QM_MCR_NP_STATE_ACTIVE 0x05 -#define QM_MCR_NP_PTR_MASK 0x07ff /* for RA[12] & OD[123] */ -#define QM_MCR_NP_RA1_NRA(v) (((v) >> 14) & 0x3) /* FQD::NRA */ -#define QM_MCR_NP_RA2_IT(v) (((v) >> 14) & 0x1) /* FQD::IT */ -#define QM_MCR_NP_OD1_NOD(v) (((v) >> 14) & 0x3) /* FQD::NOD */ -#define QM_MCR_NP_OD3_NPC(v) (((v) >> 14) & 0x3) /* FQD::NPC */ - -enum qm_mcr_queryfq_np_masks { - qm_mcr_fqd_link_mask = BIT(24)-1, - qm_mcr_odp_seq_mask = BIT(14)-1, - qm_mcr_orp_nesn_mask = BIT(14)-1, - qm_mcr_orp_ea_hseq_mask = BIT(15)-1, - qm_mcr_orp_ea_tseq_mask = BIT(15)-1, - qm_mcr_orp_ea_hptr_mask = BIT(24)-1, - qm_mcr_orp_ea_tptr_mask = BIT(24)-1, - qm_mcr_pfdr_hptr_mask = BIT(24)-1, - qm_mcr_pfdr_tptr_mask = BIT(24)-1, - qm_mcr_is_mask = BIT(1)-1, - qm_mcr_frm_cnt_mask = BIT(24)-1, -}; -#define qm_mcr_np_get(np, field) \ - ((np)->field & (qm_mcr_##field##_mask)) - /* Congestion Groups */ /* diff --git a/include/soc/fsl/qman.h b/include/soc/fsl/qman.h index 10b549783ec5..0252c32f7421 100644 --- a/include/soc/fsl/qman.h +++ b/include/soc/fsl/qman.h @@ -811,6 +811,67 @@ struct qman_cgr { #define QMAN_VOLATILE_FLAG_WAIT_INT 0x00000002 /* if wait, interruptible? */ #define QMAN_VOLATILE_FLAG_FINISH 0x00000004 /* wait till VDQCR completes */ +/* "Query FQ Non-Programmable Fields" */ +struct qm_mcr_queryfq_np { + u8 verb; + u8 result; + u8 __reserved1; + u8 state; /* QM_MCR_NP_STATE_*** */ + u32 fqd_link; /* 24-bit, _res2[24-31] */ + u16 odp_seq; /* 14-bit, _res3[14-15] */ + u16 orp_nesn; /* 14-bit, _res4[14-15] */ + u16 orp_ea_hseq; /* 15-bit, _res5[15] */ + u16 orp_ea_tseq; /* 15-bit, _res6[15] */ + u32 orp_ea_hptr; /* 24-bit, _res7[24-31] */ + u32 orp_ea_tptr; /* 24-bit, _res8[24-31] */ + u32 pfdr_hptr; /* 24-bit, _res9[24-31] */ + u32 pfdr_tptr; /* 24-bit, _res10[24-31] */ + u8 __reserved2[5]; + u8 is; /* 1-bit, _res12[1-7] */ + u16 ics_surp; + u32 byte_cnt; + u32 frm_cnt; /* 24-bit, _res13[24-31] */ + u32 __reserved3; + u16 ra1_sfdr; /* QM_MCR_NP_RA1_*** */ + u16 ra2_sfdr; /* QM_MCR_NP_RA2_*** */ + u16 __reserved4; + u16 od1_sfdr; /* QM_MCR_NP_OD1_*** */ + u16 od2_sfdr; /* QM_MCR_NP_OD2_*** */ + u16 od3_sfdr; /* QM_MCR_NP_OD3_*** */ +} __packed; + +#define QM_MCR_NP_STATE_FE 0x10 +#define QM_MCR_NP_STATE_R 0x08 +#define QM_MCR_NP_STATE_MASK 0x07 /* Reads FQD::STATE; */ +#define QM_MCR_NP_STATE_OOS 0x00 +#define QM_MCR_NP_STATE_RETIRED 0x01 +#define QM_MCR_NP_STATE_TEN_SCHED 0x02 +#define QM_MCR_NP_STATE_TRU_SCHED 0x03 +#define QM_MCR_NP_STATE_PARKED 0x04 +#define QM_MCR_NP_STATE_ACTIVE 0x05 +#define QM_MCR_NP_PTR_MASK 0x07ff /* for RA[12] & OD[123] */ +#define QM_MCR_NP_RA1_NRA(v) (((v) >> 14) & 0x3) /* FQD::NRA */ +#define QM_MCR_NP_RA2_IT(v) (((v) >> 14) & 0x1) /* FQD::IT */ +#define QM_MCR_NP_OD1_NOD(v) (((v) >> 14) & 0x3) /* FQD::NOD */ +#define QM_MCR_NP_OD3_NPC(v) (((v) >> 14) & 0x3) /* FQD::NPC */ + +enum qm_mcr_queryfq_np_masks { + qm_mcr_fqd_link_mask = BIT(24) - 1, + qm_mcr_odp_seq_mask = BIT(14) - 1, + qm_mcr_orp_nesn_mask = BIT(14) - 1, + qm_mcr_orp_ea_hseq_mask = BIT(15) - 1, + qm_mcr_orp_ea_tseq_mask = BIT(15) - 1, + qm_mcr_orp_ea_hptr_mask = BIT(24) - 1, + qm_mcr_orp_ea_tptr_mask = BIT(24) - 1, + qm_mcr_pfdr_hptr_mask = BIT(24) - 1, + qm_mcr_pfdr_tptr_mask = BIT(24) - 1, + qm_mcr_is_mask = BIT(1) - 1, + qm_mcr_frm_cnt_mask = BIT(24) - 1, +}; + +#define qm_mcr_np_get(np, field) \ + ((np)->field & (qm_mcr_##field##_mask)) + /* Portal Management */ /** * qman_p_irqsource_add - add processing sources to be interrupt-driven @@ -1033,6 +1094,13 @@ int qman_alloc_fqid_range(u32 *result, u32 count); */ int qman_release_fqid(u32 fqid); +/** + * qman_query_fq_np - Queries non-programmable FQD fields + * @fq: the frame queue object to be queried + * @np: storage for the queried FQD fields + */ +int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np); + /* Pool-channel management */ /** * qman_alloc_pool_range - Allocate a contiguous range of pool-channel IDs From e5c748a5043b67be188a53286ce203d6c25cf090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 17 Mar 2017 12:05:59 +0200 Subject: [PATCH 042/101] soc/qman: add macros needed by caam/qi driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A few other things need to be added in soc/qman, such that caam/qi won't open-code them. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- include/soc/fsl/qman.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/soc/fsl/qman.h b/include/soc/fsl/qman.h index 0252c32f7421..d4dfefdee6c1 100644 --- a/include/soc/fsl/qman.h +++ b/include/soc/fsl/qman.h @@ -168,6 +168,7 @@ static inline void qm_fd_set_param(struct qm_fd *fd, enum qm_fd_format fmt, #define qm_fd_set_contig_big(fd, len) \ qm_fd_set_param(fd, qm_fd_contig_big, 0, len) #define qm_fd_set_sg_big(fd, len) qm_fd_set_param(fd, qm_fd_sg_big, 0, len) +#define qm_fd_set_compound(fd, len) qm_fd_set_param(fd, qm_fd_compound, 0, len) static inline void qm_fd_clear_fd(struct qm_fd *fd) { @@ -642,6 +643,7 @@ struct qm_mcc_initcgr { #define QM_CGR_WE_MODE 0x0001 #define QMAN_CGR_FLAG_USE_INIT 0x00000001 +#define QMAN_CGR_MODE_FRAME 0x00000001 /* Portal and Frame Queues */ /* Represents a managed portal */ From cd373691d262afea523f4ae6ebf3dee3a204c8a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 17 Mar 2017 12:06:00 +0200 Subject: [PATCH 043/101] crypto: caam - avoid double inclusion in desc_constr.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/desc_constr.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index b9c8d98ef826..d8e83ca104e0 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -4,6 +4,9 @@ * Copyright 2008-2012 Freescale Semiconductor, Inc. */ +#ifndef DESC_CONSTR_H +#define DESC_CONSTR_H + #include "desc.h" #include "regs.h" @@ -491,3 +494,5 @@ static inline int desc_inline_query(unsigned int sd_base_len, return (rem_bytes >= 0) ? 0 : -1; } + +#endif /* DESC_CONSTR_H */ From 67c2315def06c1ef18492b214686531e69682800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 17 Mar 2017 12:06:01 +0200 Subject: [PATCH 044/101] crypto: caam - add Queue Interface (QI) backend support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CAAM engine supports two interfaces for crypto job submission: -job ring interface - already existing caam/jr driver -Queue Interface (QI) - caam/qi driver added in current patch QI is present in CAAM engines found on DPAA platforms. QI gets its I/O (frame descriptors) from QMan (Queue Manager) queues. This patch adds a platform device for accessing CAAM's queue interface. The requests are submitted to CAAM using one frame queue per cryptographic context. Each crypto context has one shared descriptor. This shared descriptor is attached to frame queue associated with corresponding driver context using context_a. The driver hides the mechanics of FQ creation, initialisation from its applications. Each cryptographic context needs to be associated with driver context which houses the FQ to be used to transport the job to CAAM. The driver provides API for: (a) Context creation (b) Job submission (c) Context deletion (d) Congestion indication - whether path to/from CAAM is congested The driver supports affining its context to a particular CPU. This means that any responses from CAAM for the context in question would arrive at the given CPU. This helps in implementing one CPU per packet round trip in IPsec application. The driver processes CAAM responses under NAPI contexts. NAPI contexts are instantiated only on cores with affined portals since only cores having their own portal can receive responses from DQRR. The responses from CAAM for all cryptographic contexts ride on a fixed set of FQs. We use one response FQ per portal owning core. The response FQ is configured in each core's and thus portal's dedicated channel. This gives the flexibility to direct CAAM's responses for a crypto context on a given core. Signed-off-by: Vakul Garg Signed-off-by: Alex Porosanu Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/Makefile | 4 + drivers/crypto/caam/ctrl.c | 58 +-- drivers/crypto/caam/intern.h | 24 ++ drivers/crypto/caam/qi.c | 805 +++++++++++++++++++++++++++++++++++ drivers/crypto/caam/qi.h | 201 +++++++++ 5 files changed, 1064 insertions(+), 28 deletions(-) create mode 100644 drivers/crypto/caam/qi.c create mode 100644 drivers/crypto/caam/qi.h diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile index 6554742f357e..2e60e45c2bf1 100644 --- a/drivers/crypto/caam/Makefile +++ b/drivers/crypto/caam/Makefile @@ -16,3 +16,7 @@ obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o caam-objs := ctrl.o caam_jr-objs := jr.o key_gen.o error.o caam_pkc-y := caampkc.o pkc_desc.o +ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),) + ccflags-y += -DCONFIG_CAAM_QI + caam-objs += qi.o +endif diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index fef39f9f41ee..b3a94d5eff26 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -18,6 +18,10 @@ bool caam_little_end; EXPORT_SYMBOL(caam_little_end); +#ifdef CONFIG_CAAM_QI +#include "qi.h" +#endif + /* * i.MX targets tend to have clock control subsystems that can * enable/disable clocking to our device. @@ -311,6 +315,11 @@ static int caam_remove(struct platform_device *pdev) for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) of_device_unregister(ctrlpriv->jrpdev[ring]); +#ifdef CONFIG_CAAM_QI + if (ctrlpriv->qidev) + caam_qi_shutdown(ctrlpriv->qidev); +#endif + /* De-initialize RNG state handles initialized by this driver. */ if (ctrlpriv->rng4_sh_init) deinstantiate_rng(ctrldev, ctrlpriv->rng4_sh_init); @@ -401,23 +410,6 @@ int caam_get_era(void) } EXPORT_SYMBOL(caam_get_era); -#ifdef CONFIG_DEBUG_FS -static int caam_debugfs_u64_get(void *data, u64 *val) -{ - *val = caam64_to_cpu(*(u64 *)data); - return 0; -} - -static int caam_debugfs_u32_get(void *data, u64 *val) -{ - *val = caam32_to_cpu(*(u32 *)data); - return 0; -} - -DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); -DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); -#endif - /* Probe routine for CAAM top (controller) level */ static int caam_probe(struct platform_device *pdev) { @@ -615,6 +607,17 @@ static int caam_probe(struct platform_device *pdev) goto iounmap_ctrl; } +#ifdef CONFIG_DEBUG_FS + /* + * FIXME: needs better naming distinction, as some amalgamation of + * "caam" and nprop->full_name. The OF name isn't distinctive, + * but does separate instances + */ + perfmon = (struct caam_perfmon __force *)&ctrl->perfmon; + + ctrlpriv->dfs_root = debugfs_create_dir(dev_name(dev), NULL); + ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root); +#endif ring = 0; ridx = 0; ctrlpriv->total_jobrs = 0; @@ -650,6 +653,13 @@ static int caam_probe(struct platform_device *pdev) ); /* This is all that's required to physically enable QI */ wr_reg32(&ctrlpriv->qi->qi_control_lo, QICTL_DQEN); + + /* If QMAN driver is present, init CAAM-QI backend */ +#ifdef CONFIG_CAAM_QI + ret = caam_qi_init(pdev); + if (ret) + dev_err(dev, "caam qi i/f init failed: %d\n", ret); +#endif } /* If no QI and no rings specified, quit and go home */ @@ -737,17 +747,6 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->total_jobrs, ctrlpriv->qi_present); #ifdef CONFIG_DEBUG_FS - /* - * FIXME: needs better naming distinction, as some amalgamation of - * "caam" and nprop->full_name. The OF name isn't distinctive, - * but does separate instances - */ - perfmon = (struct caam_perfmon __force *)&ctrl->perfmon; - - ctrlpriv->dfs_root = debugfs_create_dir(dev_name(dev), NULL); - ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root); - - /* Controller-level - performance monitor counters */ ctrlpriv->ctl_rq_dequeued = debugfs_create_file("rq_dequeued", @@ -830,6 +829,9 @@ static int caam_probe(struct platform_device *pdev) return 0; caam_remove: +#ifdef CONFIG_DEBUG_FS + debugfs_remove_recursive(ctrlpriv->dfs_root); +#endif caam_remove(pdev); return ret; diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index e2bcacc1a921..c334df638ff6 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -67,6 +67,9 @@ struct caam_drv_private { struct device *dev; struct platform_device **jrpdev; /* Alloc'ed array per sub-device */ +#ifdef CONFIG_CAAM_QI + struct device *qidev; +#endif struct platform_device *pdev; /* Physical-presence section */ @@ -110,9 +113,30 @@ struct caam_drv_private { struct debugfs_blob_wrapper ctl_kek_wrap, ctl_tkek_wrap, ctl_tdsk_wrap; struct dentry *ctl_kek, *ctl_tkek, *ctl_tdsk; +#ifdef CONFIG_CAAM_QI + struct dentry *qi_congested; +#endif #endif }; void caam_jr_algapi_init(struct device *dev); void caam_jr_algapi_remove(struct device *dev); + +#ifdef CONFIG_DEBUG_FS +static int caam_debugfs_u64_get(void *data, u64 *val) +{ + *val = caam64_to_cpu(*(u64 *)data); + return 0; +} + +static int caam_debugfs_u32_get(void *data, u64 *val) +{ + *val = caam32_to_cpu(*(u32 *)data); + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); +#endif + #endif /* INTERN_H */ diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c new file mode 100644 index 000000000000..45de8fd87774 --- /dev/null +++ b/drivers/crypto/caam/qi.c @@ -0,0 +1,805 @@ +/* + * CAAM/SEC 4.x QI transport/backend driver + * Queue Interface backend functionality + * + * Copyright 2013-2016 Freescale Semiconductor, Inc. + * Copyright 2016-2017 NXP + */ + +#include +#include +#include + +#include "regs.h" +#include "qi.h" +#include "desc.h" +#include "intern.h" +#include "desc_constr.h" + +#define PREHDR_RSLS_SHIFT 31 + +/* + * Use a reasonable backlog of frames (per CPU) as congestion threshold, + * so that resources used by the in-flight buffers do not become a memory hog. + */ +#define MAX_RSP_FQ_BACKLOG_PER_CPU 256 + +/* Length of a single buffer in the QI driver memory cache */ +#define CAAM_QI_MEMCACHE_SIZE 512 + +#define CAAM_QI_ENQUEUE_RETRIES 10000 + +#define CAAM_NAPI_WEIGHT 63 + +/* + * caam_napi - struct holding CAAM NAPI-related params + * @irqtask: IRQ task for QI backend + * @p: QMan portal + */ +struct caam_napi { + struct napi_struct irqtask; + struct qman_portal *p; +}; + +/* + * caam_qi_pcpu_priv - percpu private data structure to main list of pending + * responses expected on each cpu. + * @caam_napi: CAAM NAPI params + * @net_dev: netdev used by NAPI + * @rsp_fq: response FQ from CAAM + */ +struct caam_qi_pcpu_priv { + struct caam_napi caam_napi; + struct net_device net_dev; + struct qman_fq *rsp_fq; +} ____cacheline_aligned; + +static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv); + +/* + * caam_qi_priv - CAAM QI backend private params + * @cgr: QMan congestion group + * @qi_pdev: platform device for QI backend + */ +struct caam_qi_priv { + struct qman_cgr cgr; + struct platform_device *qi_pdev; +}; + +static struct caam_qi_priv qipriv ____cacheline_aligned; + +/* + * This is written by only one core - the one that initialized the CGR - and + * read by multiple cores (all the others). + */ +bool caam_congested __read_mostly; +EXPORT_SYMBOL(caam_congested); + +#ifdef CONFIG_DEBUG_FS +/* + * This is a counter for the number of times the congestion group (where all + * the request and response queueus are) reached congestion. Incremented + * each time the congestion callback is called with congested == true. + */ +static u64 times_congested; +#endif + +/* + * CPU from where the module initialised. This is required because QMan driver + * requires CGRs to be removed from same CPU from where they were originally + * allocated. + */ +static int mod_init_cpu; + +/* + * This is a a cache of buffers, from which the users of CAAM QI driver + * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than + * doing malloc on the hotpath. + * NOTE: A more elegant solution would be to have some headroom in the frames + * being processed. This could be added by the dpaa-ethernet driver. + * This would pose a problem for userspace application processing which + * cannot know of this limitation. So for now, this will work. + * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here + */ +static struct kmem_cache *qi_cache; + +int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req) +{ + struct qm_fd fd; + dma_addr_t addr; + int ret; + int num_retries = 0; + + qm_fd_clear_fd(&fd); + qm_fd_set_compound(&fd, qm_sg_entry_get_len(&req->fd_sgt[1])); + + addr = dma_map_single(qidev, req->fd_sgt, sizeof(req->fd_sgt), + DMA_BIDIRECTIONAL); + if (dma_mapping_error(qidev, addr)) { + dev_err(qidev, "DMA mapping error for QI enqueue request\n"); + return -EIO; + } + qm_fd_addr_set64(&fd, addr); + + do { + ret = qman_enqueue(req->drv_ctx->req_fq, &fd); + if (likely(!ret)) + return 0; + + if (ret != -EBUSY) + break; + num_retries++; + } while (num_retries < CAAM_QI_ENQUEUE_RETRIES); + + dev_err(qidev, "qman_enqueue failed: %d\n", ret); + + return ret; +} +EXPORT_SYMBOL(caam_qi_enqueue); + +static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq, + const union qm_mr_entry *msg) +{ + const struct qm_fd *fd; + struct caam_drv_req *drv_req; + struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev); + + fd = &msg->ern.fd; + + if (qm_fd_get_format(fd) != qm_fd_compound) { + dev_err(qidev, "Non-compound FD from CAAM\n"); + return; + } + + drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd)); + if (!drv_req) { + dev_err(qidev, + "Can't find original request for CAAM response\n"); + return; + } + + dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd), + sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL); + + drv_req->cbk(drv_req, -EIO); +} + +static struct qman_fq *create_caam_req_fq(struct device *qidev, + struct qman_fq *rsp_fq, + dma_addr_t hwdesc, + int fq_sched_flag) +{ + int ret; + struct qman_fq *req_fq; + struct qm_mcc_initfq opts; + + req_fq = kzalloc(sizeof(*req_fq), GFP_ATOMIC); + if (!req_fq) + return ERR_PTR(-ENOMEM); + + req_fq->cb.ern = caam_fq_ern_cb; + req_fq->cb.fqs = NULL; + + ret = qman_create_fq(0, QMAN_FQ_FLAG_DYNAMIC_FQID | + QMAN_FQ_FLAG_TO_DCPORTAL, req_fq); + if (ret) { + dev_err(qidev, "Failed to create session req FQ\n"); + goto create_req_fq_fail; + } + + memset(&opts, 0, sizeof(opts)); + opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ | + QM_INITFQ_WE_CONTEXTB | + QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID); + opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE); + qm_fqd_set_destwq(&opts.fqd, qm_channel_caam, 2); + opts.fqd.context_b = cpu_to_be32(qman_fq_fqid(rsp_fq)); + qm_fqd_context_a_set64(&opts.fqd, hwdesc); + opts.fqd.cgid = qipriv.cgr.cgrid; + + ret = qman_init_fq(req_fq, fq_sched_flag, &opts); + if (ret) { + dev_err(qidev, "Failed to init session req FQ\n"); + goto init_req_fq_fail; + } + + dev_info(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid, + smp_processor_id()); + return req_fq; + +init_req_fq_fail: + qman_destroy_fq(req_fq); +create_req_fq_fail: + kfree(req_fq); + return ERR_PTR(ret); +} + +static int empty_retired_fq(struct device *qidev, struct qman_fq *fq) +{ + int ret; + + ret = qman_volatile_dequeue(fq, QMAN_VOLATILE_FLAG_WAIT_INT | + QMAN_VOLATILE_FLAG_FINISH, + QM_VDQCR_PRECEDENCE_VDQCR | + QM_VDQCR_NUMFRAMES_TILLEMPTY); + if (ret) { + dev_err(qidev, "Volatile dequeue fail for FQ: %u\n", fq->fqid); + return ret; + } + + do { + struct qman_portal *p; + + p = qman_get_affine_portal(smp_processor_id()); + qman_p_poll_dqrr(p, 16); + } while (fq->flags & QMAN_FQ_STATE_NE); + + return 0; +} + +static int kill_fq(struct device *qidev, struct qman_fq *fq) +{ + u32 flags; + int ret; + + ret = qman_retire_fq(fq, &flags); + if (ret < 0) { + dev_err(qidev, "qman_retire_fq failed: %d\n", ret); + return ret; + } + + if (!ret) + goto empty_fq; + + /* Async FQ retirement condition */ + if (ret == 1) { + /* Retry till FQ gets in retired state */ + do { + msleep(20); + } while (fq->state != qman_fq_state_retired); + + WARN_ON(fq->flags & QMAN_FQ_STATE_BLOCKOOS); + WARN_ON(fq->flags & QMAN_FQ_STATE_ORL); + } + +empty_fq: + if (fq->flags & QMAN_FQ_STATE_NE) { + ret = empty_retired_fq(qidev, fq); + if (ret) { + dev_err(qidev, "empty_retired_fq fail for FQ: %u\n", + fq->fqid); + return ret; + } + } + + ret = qman_oos_fq(fq); + if (ret) + dev_err(qidev, "OOS of FQID: %u failed\n", fq->fqid); + + qman_destroy_fq(fq); + + return ret; +} + +static int empty_caam_fq(struct qman_fq *fq) +{ + int ret; + struct qm_mcr_queryfq_np np; + + /* Wait till the older CAAM FQ get empty */ + do { + ret = qman_query_fq_np(fq, &np); + if (ret) + return ret; + + if (!qm_mcr_np_get(&np, frm_cnt)) + break; + + msleep(20); + } while (1); + + /* + * Give extra time for pending jobs from this FQ in holding tanks + * to get processed + */ + msleep(20); + return 0; +} + +int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc) +{ + int ret; + u32 num_words; + struct qman_fq *new_fq, *old_fq; + struct device *qidev = drv_ctx->qidev; + + num_words = desc_len(sh_desc); + if (num_words > MAX_SDLEN) { + dev_err(qidev, "Invalid descriptor len: %d words\n", num_words); + return -EINVAL; + } + + /* Note down older req FQ */ + old_fq = drv_ctx->req_fq; + + /* Create a new req FQ in parked state */ + new_fq = create_caam_req_fq(drv_ctx->qidev, drv_ctx->rsp_fq, + drv_ctx->context_a, 0); + if (unlikely(IS_ERR_OR_NULL(new_fq))) { + dev_err(qidev, "FQ allocation for shdesc update failed\n"); + return PTR_ERR(new_fq); + } + + /* Hook up new FQ to context so that new requests keep queuing */ + drv_ctx->req_fq = new_fq; + + /* Empty and remove the older FQ */ + ret = empty_caam_fq(old_fq); + if (ret) { + dev_err(qidev, "Old CAAM FQ empty failed: %d\n", ret); + + /* We can revert to older FQ */ + drv_ctx->req_fq = old_fq; + + if (kill_fq(qidev, new_fq)) + dev_warn(qidev, "New CAAM FQ: %u kill failed\n", + new_fq->fqid); + + return ret; + } + + /* + * Re-initialise pre-header. Set RSLS and SDLEN. + * Update the shared descriptor for driver context. + */ + drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | + num_words); + memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); + dma_sync_single_for_device(qidev, drv_ctx->context_a, + sizeof(drv_ctx->sh_desc) + + sizeof(drv_ctx->prehdr), + DMA_BIDIRECTIONAL); + + /* Put the new FQ in scheduled state */ + ret = qman_schedule_fq(new_fq); + if (ret) { + dev_err(qidev, "Fail to sched new CAAM FQ, ecode = %d\n", ret); + + /* + * We can kill new FQ and revert to old FQ. + * Since the desc is already modified, it is success case + */ + + drv_ctx->req_fq = old_fq; + + if (kill_fq(qidev, new_fq)) + dev_warn(qidev, "New CAAM FQ: %u kill failed\n", + new_fq->fqid); + } else if (kill_fq(qidev, old_fq)) { + dev_warn(qidev, "Old CAAM FQ: %u kill failed\n", old_fq->fqid); + } + + return 0; +} +EXPORT_SYMBOL(caam_drv_ctx_update); + +struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, + int *cpu, + u32 *sh_desc) +{ + size_t size; + u32 num_words; + dma_addr_t hwdesc; + struct caam_drv_ctx *drv_ctx; + const cpumask_t *cpus = qman_affine_cpus(); + static DEFINE_PER_CPU(int, last_cpu); + + num_words = desc_len(sh_desc); + if (num_words > MAX_SDLEN) { + dev_err(qidev, "Invalid descriptor len: %d words\n", + num_words); + return ERR_PTR(-EINVAL); + } + + drv_ctx = kzalloc(sizeof(*drv_ctx), GFP_ATOMIC); + if (!drv_ctx) + return ERR_PTR(-ENOMEM); + + /* + * Initialise pre-header - set RSLS and SDLEN - and shared descriptor + * and dma-map them. + */ + drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | + num_words); + memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); + size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc); + hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(qidev, hwdesc)) { + dev_err(qidev, "DMA map error for preheader + shdesc\n"); + kfree(drv_ctx); + return ERR_PTR(-ENOMEM); + } + drv_ctx->context_a = hwdesc; + + /* If given CPU does not own the portal, choose another one that does */ + if (!cpumask_test_cpu(*cpu, cpus)) { + int *pcpu = &get_cpu_var(last_cpu); + + *pcpu = cpumask_next(*pcpu, cpus); + if (*pcpu >= nr_cpu_ids) + *pcpu = cpumask_first(cpus); + *cpu = *pcpu; + + put_cpu_var(last_cpu); + } + drv_ctx->cpu = *cpu; + + /* Find response FQ hooked with this CPU */ + drv_ctx->rsp_fq = per_cpu(pcpu_qipriv.rsp_fq, drv_ctx->cpu); + + /* Attach request FQ */ + drv_ctx->req_fq = create_caam_req_fq(qidev, drv_ctx->rsp_fq, hwdesc, + QMAN_INITFQ_FLAG_SCHED); + if (unlikely(IS_ERR_OR_NULL(drv_ctx->req_fq))) { + dev_err(qidev, "create_caam_req_fq failed\n"); + dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL); + kfree(drv_ctx); + return ERR_PTR(-ENOMEM); + } + + drv_ctx->qidev = qidev; + return drv_ctx; +} +EXPORT_SYMBOL(caam_drv_ctx_init); + +void *qi_cache_alloc(gfp_t flags) +{ + return kmem_cache_alloc(qi_cache, flags); +} +EXPORT_SYMBOL(qi_cache_alloc); + +void qi_cache_free(void *obj) +{ + kmem_cache_free(qi_cache, obj); +} +EXPORT_SYMBOL(qi_cache_free); + +static int caam_qi_poll(struct napi_struct *napi, int budget) +{ + struct caam_napi *np = container_of(napi, struct caam_napi, irqtask); + + int cleaned = qman_p_poll_dqrr(np->p, budget); + + if (cleaned < budget) { + napi_complete(napi); + qman_p_irqsource_add(np->p, QM_PIRQ_DQRI); + } + + return cleaned; +} + +void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx) +{ + if (IS_ERR_OR_NULL(drv_ctx)) + return; + + /* Remove request FQ */ + if (kill_fq(drv_ctx->qidev, drv_ctx->req_fq)) + dev_err(drv_ctx->qidev, "Crypto session req FQ kill failed\n"); + + dma_unmap_single(drv_ctx->qidev, drv_ctx->context_a, + sizeof(drv_ctx->sh_desc) + sizeof(drv_ctx->prehdr), + DMA_BIDIRECTIONAL); + kfree(drv_ctx); +} +EXPORT_SYMBOL(caam_drv_ctx_rel); + +int caam_qi_shutdown(struct device *qidev) +{ + int i, ret; + struct caam_qi_priv *priv = dev_get_drvdata(qidev); + const cpumask_t *cpus = qman_affine_cpus(); + struct cpumask old_cpumask = current->cpus_allowed; + + for_each_cpu(i, cpus) { + struct napi_struct *irqtask; + + irqtask = &per_cpu_ptr(&pcpu_qipriv.caam_napi, i)->irqtask; + napi_disable(irqtask); + netif_napi_del(irqtask); + + if (kill_fq(qidev, per_cpu(pcpu_qipriv.rsp_fq, i))) + dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n", i); + kfree(per_cpu(pcpu_qipriv.rsp_fq, i)); + } + + /* + * QMan driver requires CGRs to be deleted from same CPU from where they + * were instantiated. Hence we get the module removal execute from the + * same CPU from where it was originally inserted. + */ + set_cpus_allowed_ptr(current, get_cpu_mask(mod_init_cpu)); + + ret = qman_delete_cgr(&priv->cgr); + if (ret) + dev_err(qidev, "Deletion of CGR failed: %d\n", ret); + else + qman_release_cgrid(priv->cgr.cgrid); + + kmem_cache_destroy(qi_cache); + + /* Now that we're done with the CGRs, restore the cpus allowed mask */ + set_cpus_allowed_ptr(current, &old_cpumask); + + platform_device_unregister(priv->qi_pdev); + return ret; +} + +static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested) +{ + caam_congested = congested; + + if (congested) { +#ifdef CONFIG_DEBUG_FS + times_congested++; +#endif + pr_debug_ratelimited("CAAM entered congestion\n"); + + } else { + pr_debug_ratelimited("CAAM exited congestion\n"); + } +} + +static int caam_qi_napi_schedule(struct qman_portal *p, struct caam_napi *np) +{ + /* + * In case of threaded ISR, for RT kernels in_irq() does not return + * appropriate value, so use in_serving_softirq to distinguish between + * softirq and irq contexts. + */ + if (unlikely(in_irq() || !in_serving_softirq())) { + /* Disable QMan IRQ source and invoke NAPI */ + qman_p_irqsource_remove(p, QM_PIRQ_DQRI); + np->p = p; + napi_schedule(&np->irqtask); + return 1; + } + return 0; +} + +static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p, + struct qman_fq *rsp_fq, + const struct qm_dqrr_entry *dqrr) +{ + struct caam_napi *caam_napi = raw_cpu_ptr(&pcpu_qipriv.caam_napi); + struct caam_drv_req *drv_req; + const struct qm_fd *fd; + struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev); + u32 status; + + if (caam_qi_napi_schedule(p, caam_napi)) + return qman_cb_dqrr_stop; + + fd = &dqrr->fd; + status = be32_to_cpu(fd->status); + if (unlikely(status)) + dev_err(qidev, "Error: %#x in CAAM response FD\n", status); + + if (unlikely(qm_fd_get_format(fd) != qm_fd_compound)) { + dev_err(qidev, "Non-compound FD from CAAM\n"); + return qman_cb_dqrr_consume; + } + + drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd)); + if (unlikely(!drv_req)) { + dev_err(qidev, + "Can't find original request for caam response\n"); + return qman_cb_dqrr_consume; + } + + dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd), + sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL); + + drv_req->cbk(drv_req, status); + return qman_cb_dqrr_consume; +} + +static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu) +{ + struct qm_mcc_initfq opts; + struct qman_fq *fq; + int ret; + + fq = kzalloc(sizeof(*fq), GFP_KERNEL | GFP_DMA); + if (!fq) + return -ENOMEM; + + fq->cb.dqrr = caam_rsp_fq_dqrr_cb; + + ret = qman_create_fq(0, QMAN_FQ_FLAG_NO_ENQUEUE | + QMAN_FQ_FLAG_DYNAMIC_FQID, fq); + if (ret) { + dev_err(qidev, "Rsp FQ create failed\n"); + kfree(fq); + return -ENODEV; + } + + memset(&opts, 0, sizeof(opts)); + opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ | + QM_INITFQ_WE_CONTEXTB | + QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID); + opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CTXASTASHING | + QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE); + qm_fqd_set_destwq(&opts.fqd, qman_affine_channel(cpu), 3); + opts.fqd.cgid = qipriv.cgr.cgrid; + opts.fqd.context_a.stashing.exclusive = QM_STASHING_EXCL_CTX | + QM_STASHING_EXCL_DATA; + qm_fqd_set_stashing(&opts.fqd, 0, 1, 1); + + ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &opts); + if (ret) { + dev_err(qidev, "Rsp FQ init failed\n"); + kfree(fq); + return -ENODEV; + } + + per_cpu(pcpu_qipriv.rsp_fq, cpu) = fq; + + dev_info(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu); + return 0; +} + +static int init_cgr(struct device *qidev) +{ + int ret; + struct qm_mcc_initcgr opts; + const u64 cpus = *(u64 *)qman_affine_cpus(); + const int num_cpus = hweight64(cpus); + const u64 val = num_cpus * MAX_RSP_FQ_BACKLOG_PER_CPU; + + ret = qman_alloc_cgrid(&qipriv.cgr.cgrid); + if (ret) { + dev_err(qidev, "CGR alloc failed for rsp FQs: %d\n", ret); + return ret; + } + + qipriv.cgr.cb = cgr_cb; + memset(&opts, 0, sizeof(opts)); + opts.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES | + QM_CGR_WE_MODE); + opts.cgr.cscn_en = QM_CGR_EN; + opts.cgr.mode = QMAN_CGR_MODE_FRAME; + qm_cgr_cs_thres_set64(&opts.cgr.cs_thres, val, 1); + + ret = qman_create_cgr(&qipriv.cgr, QMAN_CGR_FLAG_USE_INIT, &opts); + if (ret) { + dev_err(qidev, "Error %d creating CAAM CGRID: %u\n", ret, + qipriv.cgr.cgrid); + return ret; + } + + dev_info(qidev, "Congestion threshold set to %llu\n", val); + return 0; +} + +static int alloc_rsp_fqs(struct device *qidev) +{ + int ret, i; + const cpumask_t *cpus = qman_affine_cpus(); + + /*Now create response FQs*/ + for_each_cpu(i, cpus) { + ret = alloc_rsp_fq_cpu(qidev, i); + if (ret) { + dev_err(qidev, "CAAM rsp FQ alloc failed, cpu: %u", i); + return ret; + } + } + + return 0; +} + +static void free_rsp_fqs(void) +{ + int i; + const cpumask_t *cpus = qman_affine_cpus(); + + for_each_cpu(i, cpus) + kfree(per_cpu(pcpu_qipriv.rsp_fq, i)); +} + +int caam_qi_init(struct platform_device *caam_pdev) +{ + int err, i; + struct platform_device *qi_pdev; + struct device *ctrldev = &caam_pdev->dev, *qidev; + struct caam_drv_private *ctrlpriv; + const cpumask_t *cpus = qman_affine_cpus(); + struct cpumask old_cpumask = current->cpus_allowed; + static struct platform_device_info qi_pdev_info = { + .name = "caam_qi", + .id = PLATFORM_DEVID_NONE + }; + + /* + * QMAN requires CGRs to be removed from same CPU+portal from where it + * was originally allocated. Hence we need to note down the + * initialisation CPU and use the same CPU for module exit. + * We select the first CPU to from the list of portal owning CPUs. + * Then we pin module init to this CPU. + */ + mod_init_cpu = cpumask_first(cpus); + set_cpus_allowed_ptr(current, get_cpu_mask(mod_init_cpu)); + + qi_pdev_info.parent = ctrldev; + qi_pdev_info.dma_mask = dma_get_mask(ctrldev); + qi_pdev = platform_device_register_full(&qi_pdev_info); + if (IS_ERR(qi_pdev)) + return PTR_ERR(qi_pdev); + + ctrlpriv = dev_get_drvdata(ctrldev); + qidev = &qi_pdev->dev; + + qipriv.qi_pdev = qi_pdev; + dev_set_drvdata(qidev, &qipriv); + + /* Initialize the congestion detection */ + err = init_cgr(qidev); + if (err) { + dev_err(qidev, "CGR initialization failed: %d\n", err); + platform_device_unregister(qi_pdev); + return err; + } + + /* Initialise response FQs */ + err = alloc_rsp_fqs(qidev); + if (err) { + dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err); + free_rsp_fqs(); + platform_device_unregister(qi_pdev); + return err; + } + + /* + * Enable the NAPI contexts on each of the core which has an affine + * portal. + */ + for_each_cpu(i, cpus) { + struct caam_qi_pcpu_priv *priv = per_cpu_ptr(&pcpu_qipriv, i); + struct caam_napi *caam_napi = &priv->caam_napi; + struct napi_struct *irqtask = &caam_napi->irqtask; + struct net_device *net_dev = &priv->net_dev; + + net_dev->dev = *qidev; + INIT_LIST_HEAD(&net_dev->napi_list); + + netif_napi_add(net_dev, irqtask, caam_qi_poll, + CAAM_NAPI_WEIGHT); + + napi_enable(irqtask); + } + + /* Hook up QI device to parent controlling caam device */ + ctrlpriv->qidev = qidev; + + qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0, + SLAB_CACHE_DMA, NULL); + if (!qi_cache) { + dev_err(qidev, "Can't allocate CAAM cache\n"); + free_rsp_fqs(); + platform_device_unregister(qi_pdev); + return err; + } + + /* Done with the CGRs; restore the cpus allowed mask */ + set_cpus_allowed_ptr(current, &old_cpumask); +#ifdef CONFIG_DEBUG_FS + ctrlpriv->qi_congested = debugfs_create_file("qi_congested", 0444, + ctrlpriv->ctl, + ×_congested, + &caam_fops_u64_ro); +#endif + dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n"); + return 0; +} diff --git a/drivers/crypto/caam/qi.h b/drivers/crypto/caam/qi.h new file mode 100644 index 000000000000..33b0433f5f22 --- /dev/null +++ b/drivers/crypto/caam/qi.h @@ -0,0 +1,201 @@ +/* + * Public definitions for the CAAM/QI (Queue Interface) backend. + * + * Copyright 2013-2016 Freescale Semiconductor, Inc. + * Copyright 2016-2017 NXP + */ + +#ifndef __QI_H__ +#define __QI_H__ + +#include +#include "compat.h" +#include "desc.h" +#include "desc_constr.h" + +/* + * CAAM hardware constructs a job descriptor which points to a shared descriptor + * (as pointed by context_a of to-CAAM FQ). + * When the job descriptor is executed by DECO, the whole job descriptor + * together with shared descriptor gets loaded in DECO buffer, which is + * 64 words (each 32-bit) long. + * + * The job descriptor constructed by CAAM hardware has the following layout: + * + * HEADER (1 word) + * Shdesc ptr (1 or 2 words) + * SEQ_OUT_PTR (1 word) + * Out ptr (1 or 2 words) + * Out length (1 word) + * SEQ_IN_PTR (1 word) + * In ptr (1 or 2 words) + * In length (1 word) + * + * The shdesc ptr is used to fetch shared descriptor contents into DECO buffer. + * + * Apart from shdesc contents, the total number of words that get loaded in DECO + * buffer are '8' or '11'. The remaining words in DECO buffer can be used for + * storing shared descriptor. + */ +#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ) + +extern bool caam_congested __read_mostly; + +/* + * This is the request structure the driver application should fill while + * submitting a job to driver. + */ +struct caam_drv_req; + +/* + * caam_qi_cbk - application's callback function invoked by the driver when the + * request has been successfully processed. + * @drv_req: original request that was submitted + * @status: completion status of request (0 - success, non-zero - error code) + */ +typedef void (*caam_qi_cbk)(struct caam_drv_req *drv_req, u32 status); + +enum optype { + ENCRYPT, + DECRYPT, + GIVENCRYPT, + NUM_OP +}; + +/** + * caam_drv_ctx - CAAM/QI backend driver context + * + * The jobs are processed by the driver against a driver context. + * With every cryptographic context, a driver context is attached. + * The driver context contains data for private use by driver. + * For the applications, this is an opaque structure. + * + * @prehdr: preheader placed before shrd desc + * @sh_desc: shared descriptor + * @context_a: shared descriptor dma address + * @req_fq: to-CAAM request frame queue + * @rsp_fq: from-CAAM response frame queue + * @cpu: cpu on which to receive CAAM response + * @op_type: operation type + * @qidev: device pointer for CAAM/QI backend + */ +struct caam_drv_ctx { + u32 prehdr[2]; + u32 sh_desc[MAX_SDLEN]; + dma_addr_t context_a; + struct qman_fq *req_fq; + struct qman_fq *rsp_fq; + int cpu; + enum optype op_type; + struct device *qidev; +} ____cacheline_aligned; + +/** + * caam_drv_req - The request structure the driver application should fill while + * submitting a job to driver. + * @fd_sgt: QMan S/G pointing to output (fd_sgt[0]) and input (fd_sgt[1]) + * buffers. + * @cbk: callback function to invoke when job is completed + * @app_ctx: arbitrary context attached with request by the application + * + * The fields mentioned below should not be used by application. + * These are for private use by driver. + * + * @hdr__: linked list header to maintain list of outstanding requests to CAAM + * @hwaddr: DMA address for the S/G table. + */ +struct caam_drv_req { + struct qm_sg_entry fd_sgt[2]; + struct caam_drv_ctx *drv_ctx; + caam_qi_cbk cbk; + void *app_ctx; +} ____cacheline_aligned; + +/** + * caam_drv_ctx_init - Initialise a CAAM/QI driver context + * + * A CAAM/QI driver context must be attached with each cryptographic context. + * This function allocates memory for CAAM/QI context and returns a handle to + * the application. This handle must be submitted along with each enqueue + * request to the driver by the application. + * + * @cpu: CPU where the application prefers to the driver to receive CAAM + * responses. The request completion callback would be issued from this + * CPU. + * @sh_desc: shared descriptor pointer to be attached with CAAM/QI driver + * context. + * + * Returns a driver context on success or negative error code on failure. + */ +struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, int *cpu, + u32 *sh_desc); + +/** + * caam_qi_enqueue - Submit a request to QI backend driver. + * + * The request structure must be properly filled as described above. + * + * @qidev: device pointer for QI backend + * @req: CAAM QI request structure + * + * Returns 0 on success or negative error code on failure. + */ +int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req); + +/** + * caam_drv_ctx_busy - Check if there are too many jobs pending with CAAM + * or too many CAAM responses are pending to be processed. + * @drv_ctx: driver context for which job is to be submitted + * + * Returns caam congestion status 'true/false' + */ +bool caam_drv_ctx_busy(struct caam_drv_ctx *drv_ctx); + +/** + * caam_drv_ctx_update - Update QI driver context + * + * Invoked when shared descriptor is required to be change in driver context. + * + * @drv_ctx: driver context to be updated + * @sh_desc: new shared descriptor pointer to be updated in QI driver context + * + * Returns 0 on success or negative error code on failure. + */ +int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc); + +/** + * caam_drv_ctx_rel - Release a QI driver context + * @drv_ctx: context to be released + */ +void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx); + +int caam_qi_init(struct platform_device *pdev); +int caam_qi_shutdown(struct device *dev); + +/** + * qi_cache_alloc - Allocate buffers from CAAM-QI cache + * + * Invoked when a user of the CAAM-QI (i.e. caamalg-qi) needs data which has + * to be allocated on the hotpath. Instead of using malloc, one can use the + * services of the CAAM QI memory cache (backed by kmem_cache). The buffers + * will have a size of 256B, which is sufficient for hosting 16 SG entries. + * + * @flags: flags that would be used for the equivalent malloc(..) call + * + * Returns a pointer to a retrieved buffer on success or NULL on failure. + */ +void *qi_cache_alloc(gfp_t flags); + +/** + * qi_cache_free - Frees buffers allocated from CAAM-QI cache + * + * Invoked when a user of the CAAM-QI (i.e. caamalg-qi) no longer needs + * the buffer previously allocated by a qi_cache_alloc call. + * No checking is being done, the call is a passthrough call to + * kmem_cache_free(...) + * + * @obj: object previously allocated using qi_cache_alloc() + */ +void qi_cache_free(void *obj); + +#endif /* __QI_H__ */ From b189817cf7894e03fd3700acd923221d3007259e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 17 Mar 2017 12:06:02 +0200 Subject: [PATCH 045/101] crypto: caam/qi - add ablkcipher and authenc algorithms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support to submit ablkcipher and authenc algorithms via the QI backend: -ablkcipher: cbc({aes,des,des3_ede}) ctr(aes), rfc3686(ctr(aes)) xts(aes) -authenc: authenc(hmac(md5),cbc({aes,des,des3_ede})) authenc(hmac(sha*),cbc({aes,des,des3_ede})) caam/qi being a new driver, let's wait some time to settle down without interfering with existing caam/jr driver. Accordingly, for now all caam/qi algorithms (caamalg_qi module) are marked to be of lower priority than caam/jr ones (caamalg module). Signed-off-by: Vakul Garg Signed-off-by: Alex Porosanu Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/Kconfig | 20 +- drivers/crypto/caam/Makefile | 1 + drivers/crypto/caam/caamalg.c | 9 +- drivers/crypto/caam/caamalg_desc.c | 77 +- drivers/crypto/caam/caamalg_desc.h | 15 +- drivers/crypto/caam/caamalg_qi.c | 2387 ++++++++++++++++++++++++++++ drivers/crypto/caam/sg_sw_qm.h | 108 ++ 7 files changed, 2601 insertions(+), 16 deletions(-) create mode 100644 drivers/crypto/caam/caamalg_qi.c create mode 100644 drivers/crypto/caam/sg_sw_qm.h diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig index bc0d3569f8d9..e36aeacd7635 100644 --- a/drivers/crypto/caam/Kconfig +++ b/drivers/crypto/caam/Kconfig @@ -87,6 +87,23 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API To compile this as a module, choose M here: the module will be called caamalg. +config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI + tristate "Queue Interface as Crypto API backend" + depends on CRYPTO_DEV_FSL_CAAM_JR && FSL_DPAA && NET + default y + select CRYPTO_AUTHENC + select CRYPTO_BLKCIPHER + help + Selecting this will use CAAM Queue Interface (QI) for sending + & receiving crypto jobs to/from CAAM. This gives better performance + than job ring interface when the number of cores are more than the + number of job rings assigned to the kernel. The number of portals + assigned to the kernel should also be more than the number of + job rings. + + To compile this as a module, choose M here: the module + will be called caamalg_qi. + config CRYPTO_DEV_FSL_CAAM_AHASH_API tristate "Register hash algorithm implementations with Crypto API" depends on CRYPTO_DEV_FSL_CAAM_JR @@ -136,4 +153,5 @@ config CRYPTO_DEV_FSL_CAAM_DEBUG information in the CAAM driver. config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC - def_tristate CRYPTO_DEV_FSL_CAAM_CRYPTO_API + def_tristate (CRYPTO_DEV_FSL_CAAM_CRYPTO_API || \ + CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile index 2e60e45c2bf1..9e2e98856b9b 100644 --- a/drivers/crypto/caam/Makefile +++ b/drivers/crypto/caam/Makefile @@ -8,6 +8,7 @@ endif obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC) += caamalg_desc.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 9bc80eb06934..398807d1b77e 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -266,8 +266,9 @@ static int aead_set_sh_desc(struct crypto_aead *aead) /* aead_encrypt shared descriptor */ desc = ctx->sh_desc_enc; - cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ctx->authsize, - is_rfc3686, nonce, ctx1_iv_off); + cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize, + ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, + false); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), DMA_TO_DEVICE); @@ -299,7 +300,7 @@ skip_enc: desc = ctx->sh_desc_dec; cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, alg->caam.geniv, is_rfc3686, - nonce, ctx1_iv_off); + nonce, ctx1_iv_off, false); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, desc_bytes(desc), DMA_TO_DEVICE); @@ -333,7 +334,7 @@ skip_enc: desc = ctx->sh_desc_enc; cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, is_rfc3686, nonce, - ctx1_iv_off); + ctx1_iv_off, false); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), DMA_TO_DEVICE); diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c index f3f48c10b9d6..6f9c7ec0e339 100644 --- a/drivers/crypto/caam/caamalg_desc.c +++ b/drivers/crypto/caam/caamalg_desc.c @@ -265,17 +265,19 @@ static void init_sh_desc_key_aead(u32 * const desc, * split key is to be used, the size of the split key itself is * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. + * @ivsize: initialization vector size * @icvsize: integrity check value (ICV) size (truncated or full) * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template * @nonce: pointer to rfc3686 nonce * @ctx1_iv_off: IV offset in CONTEXT1 register + * @is_qi: true when called from caam/qi * * Note: Requires an MDHA split key. */ void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, - struct alginfo *adata, unsigned int icvsize, - const bool is_rfc3686, u32 *nonce, - const u32 ctx1_iv_off) + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool is_rfc3686, + u32 *nonce, const u32 ctx1_iv_off, const bool is_qi) { /* Note: Context registers are saved. */ init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce); @@ -284,6 +286,25 @@ void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ctx1_iv_off << LDST_OFFSET_SHIFT)); + } + /* Read and write assoclen bytes */ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); @@ -338,6 +359,7 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_encap); * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template * @nonce: pointer to rfc3686 nonce * @ctx1_iv_off: IV offset in CONTEXT1 register + * @is_qi: true when called from caam/qi * * Note: Requires an MDHA split key. */ @@ -345,7 +367,7 @@ void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, struct alginfo *adata, unsigned int ivsize, unsigned int icvsize, const bool geniv, const bool is_rfc3686, u32 *nonce, - const u32 ctx1_iv_off) + const u32 ctx1_iv_off, const bool is_qi) { /* Note: Context registers are saved. */ init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce); @@ -354,6 +376,26 @@ void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + if (!geniv) + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ctx1_iv_off << LDST_OFFSET_SHIFT)); + } + /* Read and write assoclen bytes */ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); if (geniv) @@ -423,21 +465,44 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_decap); * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template * @nonce: pointer to rfc3686 nonce * @ctx1_iv_off: IV offset in CONTEXT1 register + * @is_qi: true when called from caam/qi * * Note: Requires an MDHA split key. */ void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, struct alginfo *adata, unsigned int ivsize, unsigned int icvsize, const bool is_rfc3686, - u32 *nonce, const u32 ctx1_iv_off) + u32 *nonce, const u32 ctx1_iv_off, + const bool is_qi) { u32 geniv, moveiv; /* Note: Context registers are saved. */ init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce); - if (is_rfc3686) + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + } + + if (is_rfc3686) { + if (is_qi) + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ctx1_iv_off << LDST_OFFSET_SHIFT)); + goto copy_iv; + } /* Generate IV */ geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h index 95551737333a..8731e4a7ff05 100644 --- a/drivers/crypto/caam/caamalg_desc.h +++ b/drivers/crypto/caam/caamalg_desc.h @@ -12,6 +12,9 @@ #define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ) #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) +#define DESC_QI_AEAD_ENC_LEN (DESC_AEAD_ENC_LEN + 3 * CAAM_CMD_SZ) +#define DESC_QI_AEAD_DEC_LEN (DESC_AEAD_DEC_LEN + 3 * CAAM_CMD_SZ) +#define DESC_QI_AEAD_GIVENC_LEN (DESC_AEAD_GIVENC_LEN + 3 * CAAM_CMD_SZ) /* Note: Nonce is counted in cdata.keylen */ #define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ) @@ -45,20 +48,22 @@ void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, unsigned int icvsize); void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, - struct alginfo *adata, unsigned int icvsize, - const bool is_rfc3686, u32 *nonce, - const u32 ctx1_iv_off); + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool is_rfc3686, + u32 *nonce, const u32 ctx1_iv_off, + const bool is_qi); void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, struct alginfo *adata, unsigned int ivsize, unsigned int icvsize, const bool geniv, const bool is_rfc3686, u32 *nonce, - const u32 ctx1_iv_off); + const u32 ctx1_iv_off, const bool is_qi); void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, struct alginfo *adata, unsigned int ivsize, unsigned int icvsize, const bool is_rfc3686, - u32 *nonce, const u32 ctx1_iv_off); + u32 *nonce, const u32 ctx1_iv_off, + const bool is_qi); void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, unsigned int icvsize); diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c new file mode 100644 index 000000000000..ea0e5b8b9171 --- /dev/null +++ b/drivers/crypto/caam/caamalg_qi.c @@ -0,0 +1,2387 @@ +/* + * Freescale FSL CAAM support for crypto API over QI backend. + * Based on caamalg.c + * + * Copyright 2013-2016 Freescale Semiconductor, Inc. + * Copyright 2016-2017 NXP + */ + +#include "compat.h" + +#include "regs.h" +#include "intern.h" +#include "desc_constr.h" +#include "error.h" +#include "sg_sw_sec4.h" +#include "sg_sw_qm.h" +#include "key_gen.h" +#include "qi.h" +#include "jr.h" +#include "caamalg_desc.h" + +/* + * crypto alg + */ +#define CAAM_CRA_PRIORITY 2000 +/* max key is sum of AES_MAX_KEY_SIZE, max split key size */ +#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ + SHA512_DIGEST_SIZE * 2) + +#define DESC_MAX_USED_BYTES (DESC_QI_AEAD_GIVENC_LEN + \ + CAAM_MAX_KEY_SIZE) +#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) + +struct caam_alg_entry { + int class1_alg_type; + int class2_alg_type; + bool rfc3686; + bool geniv; +}; + +struct caam_aead_alg { + struct aead_alg aead; + struct caam_alg_entry caam; + bool registered; +}; + +/* + * per-session context + */ +struct caam_ctx { + struct device *jrdev; + u32 sh_desc_enc[DESC_MAX_USED_LEN]; + u32 sh_desc_dec[DESC_MAX_USED_LEN]; + u32 sh_desc_givenc[DESC_MAX_USED_LEN]; + u8 key[CAAM_MAX_KEY_SIZE]; + dma_addr_t key_dma; + struct alginfo adata; + struct alginfo cdata; + unsigned int authsize; + struct device *qidev; + spinlock_t lock; /* Protects multiple init of driver context */ + struct caam_drv_ctx *drv_ctx[NUM_OP]; +}; + +static int aead_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + typeof(*alg), aead); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + u32 ctx1_iv_off = 0; + u32 *nonce = NULL; + unsigned int data_len[2]; + u32 inl_mask; + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = alg->caam.rfc3686; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ctx1_iv_off = 16; + + /* + * RFC3686 specific: + * CONTEXT1[255:128] = {NONCE, IV, COUNTER} + */ + if (is_rfc3686) { + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + + ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); + } + + data_len[0] = ctx->adata.keylen_pad; + data_len[1] = ctx->cdata.keylen; + + if (alg->caam.geniv) + goto skip_enc; + + /* aead_encrypt shared descriptor */ + if (desc_inline_query(DESC_QI_AEAD_ENC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + if (inl_mask & 1) + ctx->adata.key_virt = ctx->key; + else + ctx->adata.key_dma = ctx->key_dma; + + if (inl_mask & 2) + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + else + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, is_rfc3686, nonce, + ctx1_iv_off, true); + +skip_enc: + /* aead_decrypt shared descriptor */ + if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + if (inl_mask & 1) + ctx->adata.key_virt = ctx->key; + else + ctx->adata.key_dma = ctx->key_dma; + + if (inl_mask & 2) + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + else + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, alg->caam.geniv, + is_rfc3686, nonce, ctx1_iv_off, true); + + if (!alg->caam.geniv) + goto skip_givenc; + + /* aead_givencrypt shared descriptor */ + if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + if (inl_mask & 1) + ctx->adata.key_virt = ctx->key; + else + ctx->adata.key_dma = ctx->key_dma; + + if (inl_mask & 2) + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + else + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, is_rfc3686, nonce, + ctx1_iv_off, true); + +skip_givenc: + return 0; +} + +static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx(authenc); + + ctx->authsize = authsize; + aead_set_sh_desc(authenc); + + return 0; +} + +static int aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *jrdev = ctx->jrdev; + struct crypto_authenc_keys keys; + int ret = 0; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + goto badkey; + +#ifdef DEBUG + dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n", + keys.authkeylen + keys.enckeylen, keys.enckeylen, + keys.authkeylen); + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + + ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey, + keys.authkeylen, CAAM_MAX_KEY_SIZE - + keys.enckeylen); + if (ret) + goto badkey; + + /* postpend encryption key to auth split key */ + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + + keys.enckeylen, DMA_TO_DEVICE); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, + ctx->adata.keylen_pad + keys.enckeylen, 1); +#endif + + ctx->cdata.keylen = keys.enckeylen; + + ret = aead_set_sh_desc(aead); + if (ret) + goto badkey; + + /* Now update the driver contexts with the new shared descriptor */ + if (ctx->drv_ctx[ENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], + ctx->sh_desc_enc); + if (ret) { + dev_err(jrdev, "driver enc context update failed\n"); + goto badkey; + } + } + + if (ctx->drv_ctx[DECRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], + ctx->sh_desc_dec); + if (ret) { + dev_err(jrdev, "driver dec context update failed\n"); + goto badkey; + } + } + + return ret; +badkey: + crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; +} + +static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); + const char *alg_name = crypto_tfm_alg_name(tfm); + struct device *jrdev = ctx->jrdev; + unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + u32 ctx1_iv_off = 0; + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = (ctr_mode && strstr(alg_name, "rfc3686")); + int ret = 0; + + memcpy(ctx->key, key, keylen); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ctx1_iv_off = 16; + + /* + * RFC3686 specific: + * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} + * | *key = {KEY, NONCE} + */ + if (is_rfc3686) { + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + keylen -= CTR_RFC3686_NONCE_SIZE; + } + + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); + ctx->cdata.keylen = keylen; + ctx->cdata.key_virt = ctx->key; + ctx->cdata.key_inline = true; + + /* ablkcipher encrypt, decrypt, givencrypt shared descriptors */ + cnstr_shdsc_ablkcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, + is_rfc3686, ctx1_iv_off); + cnstr_shdsc_ablkcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, + is_rfc3686, ctx1_iv_off); + cnstr_shdsc_ablkcipher_givencap(ctx->sh_desc_givenc, &ctx->cdata, + ivsize, is_rfc3686, ctx1_iv_off); + + /* Now update the driver contexts with the new shared descriptor */ + if (ctx->drv_ctx[ENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], + ctx->sh_desc_enc); + if (ret) { + dev_err(jrdev, "driver enc context update failed\n"); + goto badkey; + } + } + + if (ctx->drv_ctx[DECRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], + ctx->sh_desc_dec); + if (ret) { + dev_err(jrdev, "driver dec context update failed\n"); + goto badkey; + } + } + + if (ctx->drv_ctx[GIVENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[GIVENCRYPT], + ctx->sh_desc_givenc); + if (ret) { + dev_err(jrdev, "driver givenc context update failed\n"); + goto badkey; + } + } + + return ret; +badkey: + crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; +} + +static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *jrdev = ctx->jrdev; + int ret = 0; + + if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { + crypto_ablkcipher_set_flags(ablkcipher, + CRYPTO_TFM_RES_BAD_KEY_LEN); + dev_err(jrdev, "key size mismatch\n"); + return -EINVAL; + } + + memcpy(ctx->key, key, keylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); + ctx->cdata.keylen = keylen; + ctx->cdata.key_virt = ctx->key; + ctx->cdata.key_inline = true; + + /* xts ablkcipher encrypt, decrypt shared descriptors */ + cnstr_shdsc_xts_ablkcipher_encap(ctx->sh_desc_enc, &ctx->cdata); + cnstr_shdsc_xts_ablkcipher_decap(ctx->sh_desc_dec, &ctx->cdata); + + /* Now update the driver contexts with the new shared descriptor */ + if (ctx->drv_ctx[ENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], + ctx->sh_desc_enc); + if (ret) { + dev_err(jrdev, "driver enc context update failed\n"); + goto badkey; + } + } + + if (ctx->drv_ctx[DECRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], + ctx->sh_desc_dec); + if (ret) { + dev_err(jrdev, "driver dec context update failed\n"); + goto badkey; + } + } + + return ret; +badkey: + crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return 0; +} + +/* + * aead_edesc - s/w-extended aead descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @qm_sg_bytes: length of dma mapped h/w link table + * @qm_sg_dma: bus physical mapped address of h/w link table + * @assoclen_dma: bus physical mapped address of req->assoclen + * @drv_req: driver-specific request structure + * @sgt: the h/w link table + */ +struct aead_edesc { + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int qm_sg_bytes; + dma_addr_t qm_sg_dma; + dma_addr_t assoclen_dma; + struct caam_drv_req drv_req; + struct qm_sg_entry sgt[0]; +}; + +/* + * ablkcipher_edesc - s/w-extended ablkcipher descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @qm_sg_bytes: length of dma mapped h/w link table + * @qm_sg_dma: bus physical mapped address of h/w link table + * @drv_req: driver-specific request structure + * @sgt: the h/w link table + */ +struct ablkcipher_edesc { + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int qm_sg_bytes; + dma_addr_t qm_sg_dma; + struct caam_drv_req drv_req; + struct qm_sg_entry sgt[0]; +}; + +static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, + enum optype type) +{ + /* + * This function is called on the fast path with values of 'type' + * known at compile time. Invalid arguments are not expected and + * thus no checks are made. + */ + struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type]; + u32 *desc; + + if (unlikely(!drv_ctx)) { + spin_lock(&ctx->lock); + + /* Read again to check if some other core init drv_ctx */ + drv_ctx = ctx->drv_ctx[type]; + if (!drv_ctx) { + int cpu; + + if (type == ENCRYPT) + desc = ctx->sh_desc_enc; + else if (type == DECRYPT) + desc = ctx->sh_desc_dec; + else /* (type == GIVENCRYPT) */ + desc = ctx->sh_desc_givenc; + + cpu = smp_processor_id(); + drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc); + if (likely(!IS_ERR_OR_NULL(drv_ctx))) + drv_ctx->op_type = type; + + ctx->drv_ctx[type] = drv_ctx; + } + + spin_unlock(&ctx->lock); + } + + return drv_ctx; +} + +static void caam_unmap(struct device *dev, struct scatterlist *src, + struct scatterlist *dst, int src_nents, + int dst_nents, dma_addr_t iv_dma, int ivsize, + enum optype op_type, dma_addr_t qm_sg_dma, + int qm_sg_bytes) +{ + if (dst != src) { + if (src_nents) + dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); + } else { + dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); + } + + if (iv_dma) + dma_unmap_single(dev, iv_dma, ivsize, + op_type == GIVENCRYPT ? DMA_FROM_DEVICE : + DMA_TO_DEVICE); + if (qm_sg_bytes) + dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); +} + +static void aead_unmap(struct device *dev, + struct aead_edesc *edesc, + struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + int ivsize = crypto_aead_ivsize(aead); + + caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, + edesc->iv_dma, ivsize, edesc->drv_req.drv_ctx->op_type, + edesc->qm_sg_dma, edesc->qm_sg_bytes); + dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); +} + +static void ablkcipher_unmap(struct device *dev, + struct ablkcipher_edesc *edesc, + struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + + caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, + edesc->iv_dma, ivsize, edesc->drv_req.drv_ctx->op_type, + edesc->qm_sg_dma, edesc->qm_sg_bytes); +} + +static void aead_done(struct caam_drv_req *drv_req, u32 status) +{ + struct device *qidev; + struct aead_edesc *edesc; + struct aead_request *aead_req = drv_req->app_ctx; + struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); + struct caam_ctx *caam_ctx = crypto_aead_ctx(aead); + int ecode = 0; + + qidev = caam_ctx->qidev; + + if (unlikely(status)) { + caam_jr_strstatus(qidev, status); + ecode = -EIO; + } + + edesc = container_of(drv_req, typeof(*edesc), drv_req); + aead_unmap(qidev, edesc, aead_req); + + aead_request_complete(aead_req, ecode); + qi_cache_free(edesc); +} + +/* + * allocate and map the aead extended descriptor + */ +static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, + bool encrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + typeof(*alg), aead); + struct device *qidev = ctx->qidev; + gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + struct aead_edesc *edesc; + dma_addr_t qm_sg_dma, iv_dma = 0; + int ivsize = 0; + unsigned int authsize = ctx->authsize; + int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes; + int in_len, out_len; + struct qm_sg_entry *sg_table, *fd_sgt; + struct caam_drv_ctx *drv_ctx; + enum optype op_type = encrypt ? ENCRYPT : DECRYPT; + + drv_ctx = get_drv_ctx(ctx, op_type); + if (unlikely(IS_ERR_OR_NULL(drv_ctx))) + return (struct aead_edesc *)drv_ctx; + + /* allocate space for base edesc and hw desc commands, link tables */ + edesc = qi_cache_alloc(GFP_DMA | flags); + if (unlikely(!edesc)) { + dev_err(qidev, "could not allocate extended descriptor\n"); + return ERR_PTR(-ENOMEM); + } + + if (likely(req->src == req->dst)) { + src_nents = sg_nents_for_len(req->src, req->assoclen + + req->cryptlen + + (encrypt ? authsize : 0)); + if (unlikely(src_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", + req->assoclen + req->cryptlen + + (encrypt ? authsize : 0)); + qi_cache_free(edesc); + return ERR_PTR(src_nents); + } + + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } else { + src_nents = sg_nents_for_len(req->src, req->assoclen + + req->cryptlen); + if (unlikely(src_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", + req->assoclen + req->cryptlen); + qi_cache_free(edesc); + return ERR_PTR(src_nents); + } + + dst_nents = sg_nents_for_len(req->dst, req->assoclen + + req->cryptlen + + (encrypt ? authsize : + (-authsize))); + if (unlikely(dst_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", + req->assoclen + req->cryptlen + + (encrypt ? authsize : (-authsize))); + qi_cache_free(edesc); + return ERR_PTR(dst_nents); + } + + if (src_nents) { + mapped_src_nents = dma_map_sg(qidev, req->src, + src_nents, DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = 0; + } + + mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(qidev, "unable to map destination\n"); + dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } + + if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) { + ivsize = crypto_aead_ivsize(aead); + iv_dma = dma_map_single(qidev, req->iv, ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(qidev, iv_dma)) { + dev_err(qidev, "unable to map IV\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, + dst_nents, 0, 0, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } + + /* + * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. + * Input is not contiguous. + */ + qm_sg_ents = 1 + !!ivsize + mapped_src_nents + + (mapped_dst_nents > 1 ? mapped_dst_nents : 0); + sg_table = &edesc->sgt[0]; + qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + edesc->drv_req.app_ctx = req; + edesc->drv_req.cbk = aead_done; + edesc->drv_req.drv_ctx = drv_ctx; + + edesc->assoclen_dma = dma_map_single(qidev, &req->assoclen, 4, + DMA_TO_DEVICE); + if (dma_mapping_error(qidev, edesc->assoclen_dma)) { + dev_err(qidev, "unable to map assoclen\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); + qm_sg_index++; + if (ivsize) { + dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); + qm_sg_index++; + } + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); + qm_sg_index += mapped_src_nents; + + if (mapped_dst_nents > 1) + sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + + qm_sg_index, 0); + + qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(qidev, qm_sg_dma)) { + dev_err(qidev, "unable to map S/G table\n"); + dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + edesc->qm_sg_dma = qm_sg_dma; + edesc->qm_sg_bytes = qm_sg_bytes; + + out_len = req->assoclen + req->cryptlen + + (encrypt ? ctx->authsize : (-ctx->authsize)); + in_len = 4 + ivsize + req->assoclen + req->cryptlen; + + fd_sgt = &edesc->drv_req.fd_sgt[0]; + dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0); + + if (req->dst == req->src) { + if (mapped_src_nents == 1) + dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), + out_len, 0); + else + dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + + (1 + !!ivsize) * sizeof(*sg_table), + out_len, 0); + } else if (mapped_dst_nents == 1) { + dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len, + 0); + } else { + dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) * + qm_sg_index, out_len, 0); + } + + return edesc; +} + +static inline int aead_crypt(struct aead_request *req, bool encrypt) +{ + struct aead_edesc *edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + int ret; + + if (unlikely(caam_congested)) + return -EAGAIN; + + /* allocate extended descriptor */ + edesc = aead_edesc_alloc(req, encrypt); + if (IS_ERR_OR_NULL(edesc)) + return PTR_ERR(edesc); + + /* Create and submit job descriptor */ + ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); + if (!ret) { + ret = -EINPROGRESS; + } else { + aead_unmap(ctx->qidev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int aead_encrypt(struct aead_request *req) +{ + return aead_crypt(req, true); +} + +static int aead_decrypt(struct aead_request *req) +{ + return aead_crypt(req, false); +} + +static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status) +{ + struct ablkcipher_edesc *edesc; + struct ablkcipher_request *req = drv_req->app_ctx; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *caam_ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *qidev = caam_ctx->qidev; +#ifdef DEBUG + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + + dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); +#endif + + edesc = container_of(drv_req, typeof(*edesc), drv_req); + + if (status) + caam_jr_strstatus(qidev, status); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->info, + edesc->src_nents > 1 ? 100 : ivsize, 1); + dbg_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->nbytes, 1); +#endif + + ablkcipher_unmap(qidev, edesc, req); + qi_cache_free(edesc); + + ablkcipher_request_complete(req, status); +} + +static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request + *req, bool encrypt) +{ + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *qidev = ctx->qidev; + gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + struct ablkcipher_edesc *edesc; + dma_addr_t iv_dma; + bool in_contig; + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + int dst_sg_idx, qm_sg_ents; + struct qm_sg_entry *sg_table, *fd_sgt; + struct caam_drv_ctx *drv_ctx; + enum optype op_type = encrypt ? ENCRYPT : DECRYPT; + + drv_ctx = get_drv_ctx(ctx, op_type); + if (unlikely(IS_ERR_OR_NULL(drv_ctx))) + return (struct ablkcipher_edesc *)drv_ctx; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (unlikely(src_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", + req->nbytes); + return ERR_PTR(src_nents); + } + + if (unlikely(req->src != req->dst)) { + dst_nents = sg_nents_for_len(req->dst, req->nbytes); + if (unlikely(dst_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", + req->nbytes); + return ERR_PTR(dst_nents); + } + + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, + DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + + mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(qidev, "unable to map destination\n"); + dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + } + + iv_dma = dma_map_single(qidev, req->info, ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(qidev, iv_dma)) { + dev_err(qidev, "unable to map IV\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, + 0, 0, 0, 0); + return ERR_PTR(-ENOMEM); + } + + if (mapped_src_nents == 1 && + iv_dma + ivsize == sg_dma_address(req->src)) { + in_contig = true; + qm_sg_ents = 0; + } else { + in_contig = false; + qm_sg_ents = 1 + mapped_src_nents; + } + dst_sg_idx = qm_sg_ents; + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_alloc(GFP_DMA | flags); + if (unlikely(!edesc)) { + dev_err(qidev, "could not allocate extended descriptor\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + return ERR_PTR(-ENOMEM); + } + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + sg_table = &edesc->sgt[0]; + edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); + edesc->drv_req.app_ctx = req; + edesc->drv_req.cbk = ablkcipher_done; + edesc->drv_req.drv_ctx = drv_ctx; + + if (!in_contig) { + dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); + } + + if (mapped_dst_nents > 1) + sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + + dst_sg_idx, 0); + + edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { + dev_err(qidev, "unable to map S/G table\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + fd_sgt = &edesc->drv_req.fd_sgt[0]; + + if (!in_contig) + dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma, + ivsize + req->nbytes, 0); + else + dma_to_qm_sg_one_last(&fd_sgt[1], iv_dma, ivsize + req->nbytes, + 0); + + if (req->src == req->dst) { + if (!in_contig) + dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + + sizeof(*sg_table), req->nbytes, 0); + else + dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), + req->nbytes, 0); + } else if (mapped_dst_nents > 1) { + dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * + sizeof(*sg_table), req->nbytes, 0); + } else { + dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), + req->nbytes, 0); + } + + return edesc; +} + +static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( + struct skcipher_givcrypt_request *creq) +{ + struct ablkcipher_request *req = &creq->creq; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *qidev = ctx->qidev; + gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents; + struct ablkcipher_edesc *edesc; + dma_addr_t iv_dma; + bool out_contig; + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + struct qm_sg_entry *sg_table, *fd_sgt; + int dst_sg_idx, qm_sg_ents; + struct caam_drv_ctx *drv_ctx; + + drv_ctx = get_drv_ctx(ctx, GIVENCRYPT); + if (unlikely(IS_ERR_OR_NULL(drv_ctx))) + return (struct ablkcipher_edesc *)drv_ctx; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (unlikely(src_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", + req->nbytes); + return ERR_PTR(src_nents); + } + + if (unlikely(req->src != req->dst)) { + dst_nents = sg_nents_for_len(req->dst, req->nbytes); + if (unlikely(dst_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", + req->nbytes); + return ERR_PTR(dst_nents); + } + + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, + DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + + mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(qidev, "unable to map destination\n"); + dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + + dst_nents = src_nents; + mapped_dst_nents = src_nents; + } + + iv_dma = dma_map_single(qidev, creq->giv, ivsize, DMA_FROM_DEVICE); + if (dma_mapping_error(qidev, iv_dma)) { + dev_err(qidev, "unable to map IV\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, + 0, 0, 0, 0); + return ERR_PTR(-ENOMEM); + } + + qm_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0; + dst_sg_idx = qm_sg_ents; + if (mapped_dst_nents == 1 && + iv_dma + ivsize == sg_dma_address(req->dst)) { + out_contig = true; + } else { + out_contig = false; + qm_sg_ents += 1 + mapped_dst_nents; + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_alloc(GFP_DMA | flags); + if (!edesc) { + dev_err(qidev, "could not allocate extended descriptor\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, GIVENCRYPT, 0, 0); + return ERR_PTR(-ENOMEM); + } + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + sg_table = &edesc->sgt[0]; + edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); + edesc->drv_req.app_ctx = req; + edesc->drv_req.cbk = ablkcipher_done; + edesc->drv_req.drv_ctx = drv_ctx; + + if (mapped_src_nents > 1) + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table, 0); + + if (!out_contig) { + dma_to_qm_sg_one(sg_table + dst_sg_idx, iv_dma, ivsize, 0); + sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + + dst_sg_idx + 1, 0); + } + + edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { + dev_err(qidev, "unable to map S/G table\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, GIVENCRYPT, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + fd_sgt = &edesc->drv_req.fd_sgt[0]; + + if (mapped_src_nents > 1) + dma_to_qm_sg_one_ext(&fd_sgt[1], edesc->qm_sg_dma, req->nbytes, + 0); + else + dma_to_qm_sg_one(&fd_sgt[1], sg_dma_address(req->src), + req->nbytes, 0); + + if (!out_contig) + dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * + sizeof(*sg_table), ivsize + req->nbytes, + 0); + else + dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), + ivsize + req->nbytes, 0); + + return edesc; +} + +static inline int ablkcipher_crypt(struct ablkcipher_request *req, bool encrypt) +{ + struct ablkcipher_edesc *edesc; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + int ret; + + if (unlikely(caam_congested)) + return -EAGAIN; + + /* allocate extended descriptor */ + edesc = ablkcipher_edesc_alloc(req, encrypt); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); + if (!ret) { + ret = -EINPROGRESS; + } else { + ablkcipher_unmap(ctx->qidev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int ablkcipher_encrypt(struct ablkcipher_request *req) +{ + return ablkcipher_crypt(req, true); +} + +static int ablkcipher_decrypt(struct ablkcipher_request *req) +{ + return ablkcipher_crypt(req, false); +} + +static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) +{ + struct ablkcipher_request *req = &creq->creq; + struct ablkcipher_edesc *edesc; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + int ret; + + if (unlikely(caam_congested)) + return -EAGAIN; + + /* allocate extended descriptor */ + edesc = ablkcipher_giv_edesc_alloc(creq); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); + if (!ret) { + ret = -EINPROGRESS; + } else { + ablkcipher_unmap(ctx->qidev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +#define template_ablkcipher template_u.ablkcipher +struct caam_alg_template { + char name[CRYPTO_MAX_ALG_NAME]; + char driver_name[CRYPTO_MAX_ALG_NAME]; + unsigned int blocksize; + u32 type; + union { + struct ablkcipher_alg ablkcipher; + } template_u; + u32 class1_alg_type; + u32 class2_alg_type; +}; + +static struct caam_alg_template driver_algs[] = { + /* ablkcipher descriptor */ + { + .name = "cbc(aes)", + .driver_name = "cbc-aes-caam-qi", + .blocksize = AES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "", + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + }, + { + .name = "cbc(des3_ede)", + .driver_name = "cbc-3des-caam-qi", + .blocksize = DES3_EDE_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "", + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + }, + { + .name = "cbc(des)", + .driver_name = "cbc-des-caam-qi", + .blocksize = DES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "", + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + }, + { + .name = "ctr(aes)", + .driver_name = "ctr-aes-caam-qi", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .geniv = "chainiv", + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + }, + { + .name = "rfc3686(ctr(aes))", + .driver_name = "rfc3686-ctr-aes-caam-qi", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "", + .min_keysize = AES_MIN_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + }, + { + .name = "xts(aes)", + .driver_name = "xts-aes-caam-qi", + .blocksize = AES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .template_ablkcipher = { + .setkey = xts_ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .geniv = "eseqiv", + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, + }, +}; + +static struct caam_aead_alg driver_aeads[] = { + /* single-pass ipsec_esp descriptor */ + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-aes-" + "caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-aes-" + "caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-aes-" + "caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-des-" + "caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-desi-" + "caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-des-" + "caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-des-" + "caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, +}; + +struct caam_crypto_alg { + struct list_head entry; + struct crypto_alg crypto_alg; + struct caam_alg_entry caam; +}; + +static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) +{ + struct caam_drv_private *priv; + + /* + * distribute tfms across job rings to ensure in-order + * crypto request processing per tfm + */ + ctx->jrdev = caam_jr_alloc(); + if (IS_ERR(ctx->jrdev)) { + pr_err("Job Ring Device allocation for transform failed\n"); + return PTR_ERR(ctx->jrdev); + } + + ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key), + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) { + dev_err(ctx->jrdev, "unable to map key\n"); + caam_jr_free(ctx->jrdev); + return -ENOMEM; + } + + /* copy descriptor header template value */ + ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; + ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; + + priv = dev_get_drvdata(ctx->jrdev->parent); + ctx->qidev = priv->qidev; + + spin_lock_init(&ctx->lock); + ctx->drv_ctx[ENCRYPT] = NULL; + ctx->drv_ctx[DECRYPT] = NULL; + ctx->drv_ctx[GIVENCRYPT] = NULL; + + return 0; +} + +static int caam_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct caam_crypto_alg *caam_alg = container_of(alg, typeof(*caam_alg), + crypto_alg); + struct caam_ctx *ctx = crypto_tfm_ctx(tfm); + + return caam_init_common(ctx, &caam_alg->caam); +} + +static int caam_aead_init(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), + aead); + struct caam_ctx *ctx = crypto_aead_ctx(tfm); + + return caam_init_common(ctx, &caam_alg->caam); +} + +static void caam_exit_common(struct caam_ctx *ctx) +{ + caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]); + caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); + caam_drv_ctx_rel(ctx->drv_ctx[GIVENCRYPT]); + + dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), + DMA_TO_DEVICE); + + caam_jr_free(ctx->jrdev); +} + +static void caam_cra_exit(struct crypto_tfm *tfm) +{ + caam_exit_common(crypto_tfm_ctx(tfm)); +} + +static void caam_aead_exit(struct crypto_aead *tfm) +{ + caam_exit_common(crypto_aead_ctx(tfm)); +} + +static struct list_head alg_list; +static void __exit caam_qi_algapi_exit(void) +{ + struct caam_crypto_alg *t_alg, *n; + int i; + + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { + struct caam_aead_alg *t_alg = driver_aeads + i; + + if (t_alg->registered) + crypto_unregister_aead(&t_alg->aead); + } + + if (!alg_list.next) + return; + + list_for_each_entry_safe(t_alg, n, &alg_list, entry) { + crypto_unregister_alg(&t_alg->crypto_alg); + list_del(&t_alg->entry); + kfree(t_alg); + } +} + +static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template + *template) +{ + struct caam_crypto_alg *t_alg; + struct crypto_alg *alg; + + t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); + if (!t_alg) + return ERR_PTR(-ENOMEM); + + alg = &t_alg->crypto_alg; + + snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); + snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + template->driver_name); + alg->cra_module = THIS_MODULE; + alg->cra_init = caam_cra_init; + alg->cra_exit = caam_cra_exit; + alg->cra_priority = CAAM_CRA_PRIORITY; + alg->cra_blocksize = template->blocksize; + alg->cra_alignmask = 0; + alg->cra_ctxsize = sizeof(struct caam_ctx); + alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | + template->type; + switch (template->type) { + case CRYPTO_ALG_TYPE_GIVCIPHER: + alg->cra_type = &crypto_givcipher_type; + alg->cra_ablkcipher = template->template_ablkcipher; + break; + case CRYPTO_ALG_TYPE_ABLKCIPHER: + alg->cra_type = &crypto_ablkcipher_type; + alg->cra_ablkcipher = template->template_ablkcipher; + break; + } + + t_alg->caam.class1_alg_type = template->class1_alg_type; + t_alg->caam.class2_alg_type = template->class2_alg_type; + + return t_alg; +} + +static void caam_aead_alg_init(struct caam_aead_alg *t_alg) +{ + struct aead_alg *alg = &t_alg->aead; + + alg->base.cra_module = THIS_MODULE; + alg->base.cra_priority = CAAM_CRA_PRIORITY; + alg->base.cra_ctxsize = sizeof(struct caam_ctx); + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; + + alg->init = caam_aead_init; + alg->exit = caam_aead_exit; +} + +static int __init caam_qi_algapi_init(void) +{ + struct device_node *dev_node; + struct platform_device *pdev; + struct device *ctrldev; + struct caam_drv_private *priv; + int i = 0, err = 0; + u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst; + unsigned int md_limit = SHA512_DIGEST_SIZE; + bool registered = false; + + dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); + if (!dev_node) { + dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); + if (!dev_node) + return -ENODEV; + } + + pdev = of_find_device_by_node(dev_node); + of_node_put(dev_node); + if (!pdev) + return -ENODEV; + + ctrldev = &pdev->dev; + priv = dev_get_drvdata(ctrldev); + + /* + * If priv is NULL, it's probably because the caam driver wasn't + * properly initialized (e.g. RNG4 init failed). Thus, bail out here. + */ + if (!priv || !priv->qi_present) + return -ENODEV; + + INIT_LIST_HEAD(&alg_list); + + /* + * Register crypto algorithms the device supports. + * First, detect presence and attributes of DES, AES, and MD blocks. + */ + cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); + cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); + des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT; + aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT; + md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; + + /* If MD is present, limit digest size based on LP256 */ + if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)) + md_limit = SHA256_DIGEST_SIZE; + + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { + struct caam_crypto_alg *t_alg; + struct caam_alg_template *alg = driver_algs + i; + u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!des_inst && + ((alg_sel == OP_ALG_ALGSEL_3DES) || + (alg_sel == OP_ALG_ALGSEL_DES))) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) + continue; + + t_alg = caam_alg_alloc(alg); + if (IS_ERR(t_alg)) { + err = PTR_ERR(t_alg); + dev_warn(priv->qidev, "%s alg allocation failed\n", + alg->driver_name); + continue; + } + + err = crypto_register_alg(&t_alg->crypto_alg); + if (err) { + dev_warn(priv->qidev, "%s alg registration failed\n", + t_alg->crypto_alg.cra_driver_name); + kfree(t_alg); + continue; + } + + list_add_tail(&t_alg->entry, &alg_list); + registered = true; + } + + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { + struct caam_aead_alg *t_alg = driver_aeads + i; + u32 c1_alg_sel = t_alg->caam.class1_alg_type & + OP_ALG_ALGSEL_MASK; + u32 c2_alg_sel = t_alg->caam.class2_alg_type & + OP_ALG_ALGSEL_MASK; + u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!des_inst && + ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || + (c1_alg_sel == OP_ALG_ALGSEL_DES))) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) + continue; + + /* + * Check support for AES algorithms not available + * on LP devices. + */ + if (((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) && + (alg_aai == OP_ALG_AAI_GCM)) + continue; + + /* + * Skip algorithms requiring message digests + * if MD or MD size is not supported by device. + */ + if (c2_alg_sel && + (!md_inst || (t_alg->aead.maxauthsize > md_limit))) + continue; + + caam_aead_alg_init(t_alg); + + err = crypto_register_aead(&t_alg->aead); + if (err) { + pr_warn("%s alg registration failed\n", + t_alg->aead.base.cra_driver_name); + continue; + } + + t_alg->registered = true; + registered = true; + } + + if (registered) + dev_info(priv->qidev, "algorithms registered in /proc/crypto\n"); + + return err; +} + +module_init(caam_qi_algapi_init); +module_exit(caam_qi_algapi_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend"); +MODULE_AUTHOR("Freescale Semiconductor"); diff --git a/drivers/crypto/caam/sg_sw_qm.h b/drivers/crypto/caam/sg_sw_qm.h new file mode 100644 index 000000000000..d000b4df745f --- /dev/null +++ b/drivers/crypto/caam/sg_sw_qm.h @@ -0,0 +1,108 @@ +/* + * Copyright 2013-2016 Freescale Semiconductor, Inc. + * Copyright 2016-2017 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SG_SW_QM_H +#define __SG_SW_QM_H + +#include +#include "regs.h" + +static inline void __dma_to_qm_sg(struct qm_sg_entry *qm_sg_ptr, dma_addr_t dma, + u16 offset) +{ + qm_sg_entry_set64(qm_sg_ptr, dma); + qm_sg_ptr->__reserved2 = 0; + qm_sg_ptr->bpid = 0; + qm_sg_ptr->offset = cpu_to_be16(offset & QM_SG_OFF_MASK); +} + +static inline void dma_to_qm_sg_one(struct qm_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, u16 offset) +{ + __dma_to_qm_sg(qm_sg_ptr, dma, offset); + qm_sg_entry_set_len(qm_sg_ptr, len); +} + +static inline void dma_to_qm_sg_one_last(struct qm_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, u16 offset) +{ + __dma_to_qm_sg(qm_sg_ptr, dma, offset); + qm_sg_entry_set_f(qm_sg_ptr, len); +} + +static inline void dma_to_qm_sg_one_ext(struct qm_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, u16 offset) +{ + __dma_to_qm_sg(qm_sg_ptr, dma, offset); + qm_sg_ptr->cfg = cpu_to_be32(QM_SG_EXT | (len & QM_SG_LEN_MASK)); +} + +static inline void dma_to_qm_sg_one_last_ext(struct qm_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, + u16 offset) +{ + __dma_to_qm_sg(qm_sg_ptr, dma, offset); + qm_sg_ptr->cfg = cpu_to_be32(QM_SG_EXT | QM_SG_FIN | + (len & QM_SG_LEN_MASK)); +} + +/* + * convert scatterlist to h/w link table format + * but does not have final bit; instead, returns last entry + */ +static inline struct qm_sg_entry * +sg_to_qm_sg(struct scatterlist *sg, int sg_count, + struct qm_sg_entry *qm_sg_ptr, u16 offset) +{ + while (sg_count && sg) { + dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), + sg_dma_len(sg), offset); + qm_sg_ptr++; + sg = sg_next(sg); + sg_count--; + } + return qm_sg_ptr - 1; +} + +/* + * convert scatterlist to h/w link table format + * scatterlist must have been previously dma mapped + */ +static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count, + struct qm_sg_entry *qm_sg_ptr, u16 offset) +{ + qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset); + qm_sg_entry_set_f(qm_sg_ptr, qm_sg_entry_get_len(qm_sg_ptr)); +} + +#endif /* __SG_SW_QM_H */ From 42d5c176b76e190a4a3e0dfeffdae661755955b6 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 17 Mar 2017 16:49:19 +0200 Subject: [PATCH 046/101] crypto: s5p-sss - Close possible race for completed requests Driver is capable of handling only one request at a time and it stores it in its state container struct s5p_aes_dev. This stored request must be protected between concurrent invocations (e.g. completing current request and scheduling new one). Combination of lock and "busy" field is used for that purpose. When "busy" field is true, the driver will not accept new request thus it will not overwrite currently handled data. However commit 28b62b145868 ("crypto: s5p-sss - Fix spinlock recursion on LRW(AES)") moved some of the write to "busy" field out of a lock protected critical section. This might lead to potential race between completing current request and scheduling a new one. Effectively the request completion might try to operate on new crypto request. Cc: # v4.10.x Fixes: 28b62b145868 ("crypto: s5p-sss - Fix spinlock recursion on LRW(AES)") Signed-off-by: Krzysztof Kozlowski Reviewed-by: Bartlomiej Zolnierkiewicz Signed-off-by: Herbert Xu --- drivers/crypto/s5p-sss.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 1b9da3dc799b..6c620487e9c2 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -287,7 +287,6 @@ static void s5p_sg_done(struct s5p_aes_dev *dev) static void s5p_aes_complete(struct s5p_aes_dev *dev, int err) { dev->req->base.complete(&dev->req->base, err); - dev->busy = false; } static void s5p_unset_outdata(struct s5p_aes_dev *dev) @@ -462,7 +461,7 @@ static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) spin_unlock_irqrestore(&dev->lock, flags); s5p_aes_complete(dev, 0); - dev->busy = true; + /* Device is still busy */ tasklet_schedule(&dev->tasklet); } else { /* @@ -483,6 +482,7 @@ static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) error: s5p_sg_done(dev); + dev->busy = false; spin_unlock_irqrestore(&dev->lock, flags); s5p_aes_complete(dev, err); @@ -634,6 +634,7 @@ outdata_error: indata_error: s5p_sg_done(dev); + dev->busy = false; spin_unlock_irqrestore(&dev->lock, flags); s5p_aes_complete(dev, err); } From fb514b2d57eec3c9e1ddb7474f5faaf7fb71bf2e Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 17 Mar 2017 16:49:20 +0200 Subject: [PATCH 047/101] crypto: s5p-sss - Remove unused variant field from state container The driver uses type of device (variant) only during probe so there is no need to store it for later. Signed-off-by: Krzysztof Kozlowski Reviewed-by: Bartlomiej Zolnierkiewicz Signed-off-by: Herbert Xu --- drivers/crypto/s5p-sss.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 6c620487e9c2..35ea84b7d775 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -190,8 +190,6 @@ struct s5p_aes_dev { struct crypto_queue queue; bool busy; spinlock_t lock; - - struct samsung_aes_variant *variant; }; static struct s5p_aes_dev *s5p_dev; @@ -852,7 +850,6 @@ static int s5p_aes_probe(struct platform_device *pdev) } pdata->busy = false; - pdata->variant = variant; pdata->dev = dev; platform_set_drvdata(pdev, pdata); s5p_dev = pdata; From 106d73340f46538d29547ac06271f55de0f4d14e Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 17 Mar 2017 16:49:21 +0200 Subject: [PATCH 048/101] crypto: s5p-sss - Document the struct s5p_aes_dev Add kernel-doc to s5p_aes_dev structure. Signed-off-by: Krzysztof Kozlowski Reviewed-by: Bartlomiej Zolnierkiewicz Signed-off-by: Herbert Xu --- drivers/crypto/s5p-sss.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 35ea84b7d775..7ac657f46d15 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -170,6 +170,32 @@ struct s5p_aes_ctx { int keylen; }; +/** + * struct s5p_aes_dev - Crypto device state container + * @dev: Associated device + * @clk: Clock for accessing hardware + * @ioaddr: Mapped IO memory region + * @aes_ioaddr: Per-varian offset for AES block IO memory + * @irq_fc: Feed control interrupt line + * @req: Crypto request currently handled by the device + * @ctx: Configuration for currently handled crypto request + * @sg_src: Scatter list with source data for currently handled block + * in device. This is DMA-mapped into device. + * @sg_dst: Scatter list with destination data for currently handled block + * in device. This is DMA-mapped into device. + * @sg_src_cpy: In case of unaligned access, copied scatter list + * with source data. + * @sg_dst_cpy: In case of unaligned access, copied scatter list + * with destination data. + * @tasklet: New request scheduling jib + * @queue: Crypto queue + * @busy: Indicates whether the device is currently handling some request + * thus it uses some of the fields from this state, like: + * req, ctx, sg_src/dst (and copies). This essentially + * protects against concurrent access to these fields. + * @lock: Lock for protecting both access to device hardware registers + * and fields related to current request (including the busy field). + */ struct s5p_aes_dev { struct device *dev; struct clk *clk; @@ -182,7 +208,6 @@ struct s5p_aes_dev { struct scatterlist *sg_src; struct scatterlist *sg_dst; - /* In case of unaligned access: */ struct scatterlist *sg_src_cpy; struct scatterlist *sg_dst_cpy; From 0d13d8f26c4684a077b293b8a4615170253f842c Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 17 Mar 2017 23:46:21 +0300 Subject: [PATCH 049/101] crypto: zip - Memory corruption in zip_clear_stats() There is a typo here. It should be "stats" instead of "state". The impact is that we clear 224 bytes instead of 80 and we zero out memory that we shouldn't. Fixes: 09ae5d37e093 ("crypto: zip - Add Compression/Decompression statistics") Signed-off-by: Dan Carpenter Signed-off-by: Herbert Xu --- drivers/crypto/cavium/zip/zip_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c index 0951e20b395b..6ff13d80d82e 100644 --- a/drivers/crypto/cavium/zip/zip_main.c +++ b/drivers/crypto/cavium/zip/zip_main.c @@ -530,7 +530,7 @@ static int zip_clear_stats(struct seq_file *s, void *unused) for (index = 0; index < MAX_ZIP_DEVICES; index++) { if (zip_dev[index]) { memset(&zip_dev[index]->stats, 0, - sizeof(struct zip_state)); + sizeof(struct zip_stats)); seq_printf(s, "Cleared stats for zip %d\n", index); } } From d64069ee1869b9f77eef0bf9dfceffee2bfd7692 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 20 Mar 2017 13:39:16 +0100 Subject: [PATCH 050/101] crypto: zip - add a cast for printing atomic64_t values kernelci.org reports a build-time regression on linux-next, with a harmless warning in x86 allmodconfig: drivers/crypto/cavium/zip/zip_main.c:489:18: warning: format '%ld' expects argument of type 'long int', but argument 7 has type 'long long int' [-Wformat=] drivers/crypto/cavium/zip/zip_main.c:489:18: warning: format '%ld' expects argument of type 'long int', but argument 6 has type 'long long int' [-Wformat=] drivers/crypto/cavium/zip/zip_main.c:489:18: warning: format '%ld' expects argument of type 'long int', but argument 5 has type 'long long int' [-Wformat=] The return type for atomic64_read() unfortunately differs between architectures, with some defining it as atomic_long_read() and others returning a 64-bit type explicitly. Fixing this in general would be nice, but also require changing other users of these functions, so the simpler workaround is to add a cast here that avoids the warnings on the default build. Fixes: 09ae5d37e093 ("crypto: zip - Add Compression/Decompression statistics") Signed-off-by: Arnd Bergmann Signed-off-by: Herbert Xu --- drivers/crypto/cavium/zip/zip_main.c | 40 ++++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c index 6ff13d80d82e..1cd8aa488185 100644 --- a/drivers/crypto/cavium/zip/zip_main.c +++ b/drivers/crypto/cavium/zip/zip_main.c @@ -488,32 +488,32 @@ static int zip_show_stats(struct seq_file *s, void *unused) atomic64_read(&st->comp_out_bytes)); seq_printf(s, " ZIP Device %d Stats\n" "-----------------------------------\n" - "Comp Req Submitted : \t%ld\n" - "Comp Req Completed : \t%ld\n" - "Compress In Bytes : \t%ld\n" - "Compressed Out Bytes : \t%ld\n" + "Comp Req Submitted : \t%lld\n" + "Comp Req Completed : \t%lld\n" + "Compress In Bytes : \t%lld\n" + "Compressed Out Bytes : \t%lld\n" "Average Chunk size : \t%llu\n" "Average Compression ratio : \t%llu\n" - "Decomp Req Submitted : \t%ld\n" - "Decomp Req Completed : \t%ld\n" - "Decompress In Bytes : \t%ld\n" - "Decompressed Out Bytes : \t%ld\n" - "Decompress Bad requests : \t%ld\n" - "Pending Req : \t%ld\n" + "Decomp Req Submitted : \t%lld\n" + "Decomp Req Completed : \t%lld\n" + "Decompress In Bytes : \t%lld\n" + "Decompressed Out Bytes : \t%lld\n" + "Decompress Bad requests : \t%lld\n" + "Pending Req : \t%lld\n" "---------------------------------\n", index, - atomic64_read(&st->comp_req_submit), - atomic64_read(&st->comp_req_complete), - atomic64_read(&st->comp_in_bytes), - atomic64_read(&st->comp_out_bytes), + (u64)atomic64_read(&st->comp_req_submit), + (u64)atomic64_read(&st->comp_req_complete), + (u64)atomic64_read(&st->comp_in_bytes), + (u64)atomic64_read(&st->comp_out_bytes), avg_chunk, avg_cr, - atomic64_read(&st->decomp_req_submit), - atomic64_read(&st->decomp_req_complete), - atomic64_read(&st->decomp_in_bytes), - atomic64_read(&st->decomp_out_bytes), - atomic64_read(&st->decomp_bad_reqs), - atomic64_read(&st->pending_req)); + (u64)atomic64_read(&st->decomp_req_submit), + (u64)atomic64_read(&st->decomp_req_complete), + (u64)atomic64_read(&st->decomp_in_bytes), + (u64)atomic64_read(&st->decomp_out_bytes), + (u64)atomic64_read(&st->decomp_bad_reqs), + (u64)atomic64_read(&st->pending_req)); /* Reset pending requests count */ atomic64_set(&st->pending_req, 0); From 0d8da104840ab4244fe122e5b25570aa13fb00c0 Mon Sep 17 00:00:00 2001 From: Marcelo Cerri Date: Mon, 20 Mar 2017 17:28:05 -0300 Subject: [PATCH 051/101] crypto: testmgr - mark ctr(des3_ede) as fips_allowed 3DES is missing the fips_allowed flag for CTR mode. Signed-off-by: Marcelo Henrique Cerri Acked-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/testmgr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 89f1dd1f4b13..cd075c7d8ee1 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -2645,6 +2645,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, { .alg = "ctr(des3_ede)", .test = alg_test_skcipher, + .fips_allowed = 1, .suite = { .cipher = { .enc = __VECS(des3_ede_ctr_enc_tv_template), From 44068d5999d372b0034382530899df77d83c70e5 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Wed, 22 Mar 2017 15:26:36 +0100 Subject: [PATCH 052/101] crypto: DRBG - initialize SGL only once An SGL to be initialized only once even when its buffers are written to several times. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/drbg.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crypto/drbg.c b/crypto/drbg.c index 8a4d98b4adba..fa749f470135 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1749,17 +1749,16 @@ static int drbg_kcapi_sym_ctr(struct drbg_state *drbg, u8 *inbuf, u32 inlen, u8 *outbuf, u32 outlen) { - struct scatterlist sg_in; + struct scatterlist sg_in, sg_out; int ret; sg_init_one(&sg_in, inbuf, inlen); + sg_init_one(&sg_out, drbg->outscratchpad, DRBG_OUTSCRATCHLEN); while (outlen) { u32 cryptlen = min3(inlen, outlen, (u32)DRBG_OUTSCRATCHLEN); - struct scatterlist sg_out; /* Output buffer may not be valid for SGL, use scratchpad */ - sg_init_one(&sg_out, drbg->outscratchpad, cryptlen); skcipher_request_set_crypt(drbg->ctr_req, &sg_in, &sg_out, cryptlen, drbg->V); ret = crypto_skcipher_encrypt(drbg->ctr_req); From 796b40c6171456274b02447e1dbbea97456403fe Mon Sep 17 00:00:00 2001 From: Geliang Tang Date: Thu, 23 Mar 2017 21:16:30 +0800 Subject: [PATCH 053/101] crypto: ixp4xx - Use sg_virt() Use sg_virt() instead of open-coding it. Signed-off-by: Geliang Tang Acked-by: David S. Miller Signed-off-by: Herbert Xu --- drivers/crypto/ixp4xx_crypto.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index 7868765a70c5..771dd26c7076 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -806,7 +806,7 @@ static struct buffer_desc *chainup_buffers(struct device *dev, void *ptr; nbytes -= len; - ptr = page_address(sg_page(sg)) + sg->offset; + ptr = sg_virt(sg); next_buf = dma_pool_alloc(buffer_pool, flags, &next_buf_phys); if (!next_buf) { buf = NULL; From c0a680c4cc1b54c3f38b663f540d55c584452394 Mon Sep 17 00:00:00 2001 From: Fabien DESSENNE Date: Tue, 21 Mar 2017 16:13:27 +0100 Subject: [PATCH 054/101] dt-bindings: Document STM32 CRC bindings Document device tree bindings for the STM32 CRC (crypto CRC32) Signed-off-by: Fabien Dessenne Acked-by: Rob Herring Signed-off-by: Herbert Xu --- .../devicetree/bindings/crypto/st,stm32-crc.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 Documentation/devicetree/bindings/crypto/st,stm32-crc.txt diff --git a/Documentation/devicetree/bindings/crypto/st,stm32-crc.txt b/Documentation/devicetree/bindings/crypto/st,stm32-crc.txt new file mode 100644 index 000000000000..3ba92a5e9b36 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/st,stm32-crc.txt @@ -0,0 +1,16 @@ +* STMicroelectronics STM32 CRC + +Required properties: +- compatible: Should be "st,stm32f7-crc". +- reg: The address and length of the peripheral registers space +- clocks: The input clock of the CRC instance + +Optional properties: none + +Example: + +crc: crc@40023000 { + compatible = "st,stm32f7-crc"; + reg = <0x40023000 0x400>; + clocks = <&rcc 0 12>; +}; From b51dbe90912a0ce0c78717d2a8374af80b18ed11 Mon Sep 17 00:00:00 2001 From: Fabien DESSENNE Date: Tue, 21 Mar 2017 16:13:28 +0100 Subject: [PATCH 055/101] crypto: stm32 - Support for STM32 CRC32 crypto module This module registers a CRC32 ("Ethernet") and a CRC32C (Castagnoli) algorithm that make use of the STMicroelectronics STM32 crypto hardware. Theses algorithms are compatible with the little-endian generic ones. Both algorithms use ~0 as default seed (key). With CRC32C the output is xored with ~0. Using TCRYPT CRC32C speed test, this shows up to 900% speedup compared to the crc32c-generic algorithm. Signed-off-by: Fabien Dessenne Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 2 + drivers/crypto/Makefile | 1 + drivers/crypto/stm32/Kconfig | 7 + drivers/crypto/stm32/Makefile | 2 + drivers/crypto/stm32/stm32_crc32.c | 324 +++++++++++++++++++++++++++++ 5 files changed, 336 insertions(+) create mode 100644 drivers/crypto/stm32/Kconfig create mode 100644 drivers/crypto/stm32/Makefile create mode 100644 drivers/crypto/stm32/stm32_crc32.c diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 1a60626937e4..3cd234052775 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -626,4 +626,6 @@ config CRYPTO_DEV_BCM_SPU Secure Processing Unit (SPU). The SPU driver registers ablkcipher, ahash, and aead algorithms with the kernel cryptographic API. +source "drivers/crypto/stm32/Kconfig" + endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 41ca339e89d3..ea26c8dd5e14 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -31,6 +31,7 @@ obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/ obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/ obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o +obj-$(CONFIG_CRYPTO_DEV_STM32) += stm32/ obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sunxi-ss/ obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ diff --git a/drivers/crypto/stm32/Kconfig b/drivers/crypto/stm32/Kconfig new file mode 100644 index 000000000000..09b4ec87c212 --- /dev/null +++ b/drivers/crypto/stm32/Kconfig @@ -0,0 +1,7 @@ +config CRYPTO_DEV_STM32 + tristate "Support for STM32 crypto accelerators" + depends on ARCH_STM32 + select CRYPTO_HASH + help + This enables support for the CRC32 hw accelerator which can be found + on STMicroelectronis STM32 SOC. diff --git a/drivers/crypto/stm32/Makefile b/drivers/crypto/stm32/Makefile new file mode 100644 index 000000000000..73b4c6e47f5f --- /dev/null +++ b/drivers/crypto/stm32/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_STM32) += stm32_cryp.o +stm32_cryp-objs := stm32_crc32.o diff --git a/drivers/crypto/stm32/stm32_crc32.c b/drivers/crypto/stm32/stm32_crc32.c new file mode 100644 index 000000000000..765282262d44 --- /dev/null +++ b/drivers/crypto/stm32/stm32_crc32.c @@ -0,0 +1,324 @@ +/* + * Copyright (C) STMicroelectronics SA 2017 + * Author: Fabien Dessenne + * License terms: GNU General Public License (GPL), version 2 + */ + +#include +#include +#include +#include + +#include + +#include + +#define DRIVER_NAME "stm32-crc32" +#define CHKSUM_DIGEST_SIZE 4 +#define CHKSUM_BLOCK_SIZE 1 + +/* Registers */ +#define CRC_DR 0x00000000 +#define CRC_CR 0x00000008 +#define CRC_INIT 0x00000010 +#define CRC_POL 0x00000014 + +/* Registers values */ +#define CRC_CR_RESET BIT(0) +#define CRC_CR_REVERSE (BIT(7) | BIT(6) | BIT(5)) +#define CRC_INIT_DEFAULT 0xFFFFFFFF + +/* Polynomial reversed */ +#define POLY_CRC32 0xEDB88320 +#define POLY_CRC32C 0x82F63B78 + +struct stm32_crc { + struct list_head list; + struct device *dev; + void __iomem *regs; + struct clk *clk; + u8 pending_data[sizeof(u32)]; + size_t nb_pending_bytes; +}; + +struct stm32_crc_list { + struct list_head dev_list; + spinlock_t lock; /* protect dev_list */ +}; + +static struct stm32_crc_list crc_list = { + .dev_list = LIST_HEAD_INIT(crc_list.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(crc_list.lock), +}; + +struct stm32_crc_ctx { + u32 key; + u32 poly; +}; + +struct stm32_crc_desc_ctx { + u32 partial; /* crc32c: partial in first 4 bytes of that struct */ + struct stm32_crc *crc; +}; + +static int stm32_crc32_cra_init(struct crypto_tfm *tfm) +{ + struct stm32_crc_ctx *mctx = crypto_tfm_ctx(tfm); + + mctx->key = CRC_INIT_DEFAULT; + mctx->poly = POLY_CRC32; + return 0; +} + +static int stm32_crc32c_cra_init(struct crypto_tfm *tfm) +{ + struct stm32_crc_ctx *mctx = crypto_tfm_ctx(tfm); + + mctx->key = CRC_INIT_DEFAULT; + mctx->poly = POLY_CRC32C; + return 0; +} + +static int stm32_crc_setkey(struct crypto_shash *tfm, const u8 *key, + unsigned int keylen) +{ + struct stm32_crc_ctx *mctx = crypto_shash_ctx(tfm); + + if (keylen != sizeof(u32)) { + crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + mctx->key = get_unaligned_le32(key); + return 0; +} + +static int stm32_crc_init(struct shash_desc *desc) +{ + struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc); + struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm); + struct stm32_crc *crc; + + spin_lock_bh(&crc_list.lock); + list_for_each_entry(crc, &crc_list.dev_list, list) { + ctx->crc = crc; + break; + } + spin_unlock_bh(&crc_list.lock); + + /* Reset, set key, poly and configure in bit reverse mode */ + writel(bitrev32(mctx->key), ctx->crc->regs + CRC_INIT); + writel(bitrev32(mctx->poly), ctx->crc->regs + CRC_POL); + writel(CRC_CR_RESET | CRC_CR_REVERSE, ctx->crc->regs + CRC_CR); + + /* Store partial result */ + ctx->partial = readl(ctx->crc->regs + CRC_DR); + ctx->crc->nb_pending_bytes = 0; + + return 0; +} + +static int stm32_crc_update(struct shash_desc *desc, const u8 *d8, + unsigned int length) +{ + struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc); + struct stm32_crc *crc = ctx->crc; + u32 *d32; + unsigned int i; + + if (unlikely(crc->nb_pending_bytes)) { + while (crc->nb_pending_bytes != sizeof(u32) && length) { + /* Fill in pending data */ + crc->pending_data[crc->nb_pending_bytes++] = *(d8++); + length--; + } + + if (crc->nb_pending_bytes == sizeof(u32)) { + /* Process completed pending data */ + writel(*(u32 *)crc->pending_data, crc->regs + CRC_DR); + crc->nb_pending_bytes = 0; + } + } + + d32 = (u32 *)d8; + for (i = 0; i < length >> 2; i++) + /* Process 32 bits data */ + writel(*(d32++), crc->regs + CRC_DR); + + /* Store partial result */ + ctx->partial = readl(crc->regs + CRC_DR); + + /* Check for pending data (non 32 bits) */ + length &= 3; + if (likely(!length)) + return 0; + + if ((crc->nb_pending_bytes + length) >= sizeof(u32)) { + /* Shall not happen */ + dev_err(crc->dev, "Pending data overflow\n"); + return -EINVAL; + } + + d8 = (const u8 *)d32; + for (i = 0; i < length; i++) + /* Store pending data */ + crc->pending_data[crc->nb_pending_bytes++] = *(d8++); + + return 0; +} + +static int stm32_crc_final(struct shash_desc *desc, u8 *out) +{ + struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc); + struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm); + + /* Send computed CRC */ + put_unaligned_le32(mctx->poly == POLY_CRC32C ? + ~ctx->partial : ctx->partial, out); + + return 0; +} + +static int stm32_crc_finup(struct shash_desc *desc, const u8 *data, + unsigned int length, u8 *out) +{ + return stm32_crc_update(desc, data, length) ?: + stm32_crc_final(desc, out); +} + +static int stm32_crc_digest(struct shash_desc *desc, const u8 *data, + unsigned int length, u8 *out) +{ + return stm32_crc_init(desc) ?: stm32_crc_finup(desc, data, length, out); +} + +static struct shash_alg algs[] = { + /* CRC-32 */ + { + .setkey = stm32_crc_setkey, + .init = stm32_crc_init, + .update = stm32_crc_update, + .final = stm32_crc_final, + .finup = stm32_crc_finup, + .digest = stm32_crc_digest, + .descsize = sizeof(struct stm32_crc_desc_ctx), + .digestsize = CHKSUM_DIGEST_SIZE, + .base = { + .cra_name = "crc32", + .cra_driver_name = DRIVER_NAME, + .cra_priority = 200, + .cra_blocksize = CHKSUM_BLOCK_SIZE, + .cra_alignmask = 3, + .cra_ctxsize = sizeof(struct stm32_crc_ctx), + .cra_module = THIS_MODULE, + .cra_init = stm32_crc32_cra_init, + } + }, + /* CRC-32Castagnoli */ + { + .setkey = stm32_crc_setkey, + .init = stm32_crc_init, + .update = stm32_crc_update, + .final = stm32_crc_final, + .finup = stm32_crc_finup, + .digest = stm32_crc_digest, + .descsize = sizeof(struct stm32_crc_desc_ctx), + .digestsize = CHKSUM_DIGEST_SIZE, + .base = { + .cra_name = "crc32c", + .cra_driver_name = DRIVER_NAME, + .cra_priority = 200, + .cra_blocksize = CHKSUM_BLOCK_SIZE, + .cra_alignmask = 3, + .cra_ctxsize = sizeof(struct stm32_crc_ctx), + .cra_module = THIS_MODULE, + .cra_init = stm32_crc32c_cra_init, + } + } +}; + +static int stm32_crc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct stm32_crc *crc; + struct resource *res; + int ret; + + crc = devm_kzalloc(dev, sizeof(*crc), GFP_KERNEL); + if (!crc) + return -ENOMEM; + + crc->dev = dev; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + crc->regs = devm_ioremap_resource(dev, res); + if (IS_ERR(crc->regs)) { + dev_err(dev, "Cannot map CRC IO\n"); + return PTR_ERR(crc->regs); + } + + crc->clk = devm_clk_get(dev, NULL); + if (IS_ERR(crc->clk)) { + dev_err(dev, "Could not get clock\n"); + return PTR_ERR(crc->clk); + } + + ret = clk_prepare_enable(crc->clk); + if (ret) { + dev_err(crc->dev, "Failed to enable clock\n"); + return ret; + } + + platform_set_drvdata(pdev, crc); + + spin_lock(&crc_list.lock); + list_add(&crc->list, &crc_list.dev_list); + spin_unlock(&crc_list.lock); + + ret = crypto_register_shashes(algs, ARRAY_SIZE(algs)); + if (ret) { + dev_err(dev, "Failed to register\n"); + clk_disable_unprepare(crc->clk); + return ret; + } + + dev_info(dev, "Initialized\n"); + + return 0; +} + +static int stm32_crc_remove(struct platform_device *pdev) +{ + struct stm32_crc *crc = platform_get_drvdata(pdev); + + spin_lock(&crc_list.lock); + list_del(&crc->list); + spin_unlock(&crc_list.lock); + + crypto_unregister_shash(algs); + + clk_disable_unprepare(crc->clk); + + return 0; +} + +static const struct of_device_id stm32_dt_ids[] = { + { .compatible = "st,stm32f7-crc", }, + {}, +}; +MODULE_DEVICE_TABLE(of, sti_dt_ids); + +static struct platform_driver stm32_crc_driver = { + .probe = stm32_crc_probe, + .remove = stm32_crc_remove, + .driver = { + .name = DRIVER_NAME, + .of_match_table = stm32_dt_ids, + }, +}; + +module_platform_driver(stm32_crc_driver); + +MODULE_AUTHOR("Fabien Dessenne "); +MODULE_DESCRIPTION("STMicrolectronics STM32 CRC32 hardware driver"); +MODULE_LICENSE("GPL"); From 115d691fc3a40262003802ee841279597ceb3df2 Mon Sep 17 00:00:00 2001 From: Fabien DESSENNE Date: Tue, 21 Mar 2017 16:13:29 +0100 Subject: [PATCH 056/101] ARM: dts: stm32: Add CRC support to stm32f746 Add CRC (CRC32 crypto) support to stm32f746. Signed-off-by: Fabien Dessenne Signed-off-by: Herbert Xu --- arch/arm/boot/dts/stm32f746.dtsi | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/arm/boot/dts/stm32f746.dtsi b/arch/arm/boot/dts/stm32f746.dtsi index f321ffe87144..755fb923c07b 100644 --- a/arch/arm/boot/dts/stm32f746.dtsi +++ b/arch/arm/boot/dts/stm32f746.dtsi @@ -289,6 +289,13 @@ }; }; + crc: crc@40023000 { + compatible = "st,stm32f7-crc"; + reg = <0x40023000 0x400>; + clocks = <&rcc 0 12>; + status = "disabled"; + }; + rcc: rcc@40023800 { #clock-cells = <2>; compatible = "st,stm32f42xx-rcc", "st,stm32-rcc"; From 2e3db2931875a843db8b56158cf93edab4286cde Mon Sep 17 00:00:00 2001 From: Fabien DESSENNE Date: Tue, 21 Mar 2017 16:13:30 +0100 Subject: [PATCH 057/101] ARM: dts: stm32: enable CRC on stm32746g-eval board Enable the CRC (CRC32 crypto) on stm32746g-eval board Signed-off-by: Fabien Dessenne Signed-off-by: Herbert Xu --- arch/arm/boot/dts/stm32746g-eval.dts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm/boot/dts/stm32746g-eval.dts b/arch/arm/boot/dts/stm32746g-eval.dts index aa03fac1ec55..0dc18a0f0940 100644 --- a/arch/arm/boot/dts/stm32746g-eval.dts +++ b/arch/arm/boot/dts/stm32746g-eval.dts @@ -89,6 +89,10 @@ clock-frequency = <25000000>; }; +&crc { + status = "okay"; +}; + &usart1 { pinctrl-0 = <&usart1_pins_a>; pinctrl-names = "default"; From 8fbbcbdd1d259da9bc5a7772e8064aaf5e04ec20 Mon Sep 17 00:00:00 2001 From: Fabien DESSENNE Date: Tue, 21 Mar 2017 16:13:31 +0100 Subject: [PATCH 058/101] ARM: configs: stm32: Add crypto support Add STM32 crypto support in stm32_defconfig file. Signed-off-by: Fabien Dessenne Signed-off-by: Herbert Xu --- arch/arm/configs/stm32_defconfig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm/configs/stm32_defconfig b/arch/arm/configs/stm32_defconfig index a9d8e3c9b487..03437f8f9ad1 100644 --- a/arch/arm/configs/stm32_defconfig +++ b/arch/arm/configs/stm32_defconfig @@ -75,5 +75,7 @@ CONFIG_MAGIC_SYSRQ=y # CONFIG_SCHED_DEBUG is not set # CONFIG_DEBUG_BUGVERBOSE is not set # CONFIG_FTRACE is not set +CONFIG_CRYPTO=y +CONFIG_CRYPTO_DEV_STM32=y CONFIG_CRC_ITU_T=y CONFIG_CRC7=y From 51de7dd02d422da11b4dff6f11936c8333a870fe Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Tue, 28 Mar 2017 08:58:28 -0500 Subject: [PATCH 059/101] crypto: ccp - Remove redundant cpu-to-le32 macros Endianness is dealt with when the command descriptor is copied into the command queue. Remove any occurrences of cpu_to_le32() found elsewhere. Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dev-v5.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c index 5e08654c7226..e03d06a54d4e 100644 --- a/drivers/crypto/ccp/ccp-dev-v5.c +++ b/drivers/crypto/ccp/ccp-dev-v5.c @@ -419,22 +419,22 @@ static int ccp5_perform_des3(struct ccp_op *op) CCP_DES3_ENCRYPT(&function) = op->u.des3.action; CCP_DES3_MODE(&function) = op->u.des3.mode; CCP_DES3_TYPE(&function) = op->u.des3.type; - CCP5_CMD_FUNCTION(&desc) = cpu_to_le32(function.raw); + CCP5_CMD_FUNCTION(&desc) = function.raw; - CCP5_CMD_LEN(&desc) = cpu_to_le32(op->src.u.dma.length); + CCP5_CMD_LEN(&desc) = op->src.u.dma.length; - CCP5_CMD_SRC_LO(&desc) = cpu_to_le32(ccp_addr_lo(&op->src.u.dma)); - CCP5_CMD_SRC_HI(&desc) = cpu_to_le32(ccp_addr_hi(&op->src.u.dma)); - CCP5_CMD_SRC_MEM(&desc) = cpu_to_le32(CCP_MEMTYPE_SYSTEM); + CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); + CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); + CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; - CCP5_CMD_DST_LO(&desc) = cpu_to_le32(ccp_addr_lo(&op->dst.u.dma)); - CCP5_CMD_DST_HI(&desc) = cpu_to_le32(ccp_addr_hi(&op->dst.u.dma)); - CCP5_CMD_DST_MEM(&desc) = cpu_to_le32(CCP_MEMTYPE_SYSTEM); + CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); + CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); + CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; - CCP5_CMD_KEY_LO(&desc) = cpu_to_le32(lower_32_bits(key_addr)); + CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); CCP5_CMD_KEY_HI(&desc) = 0; - CCP5_CMD_KEY_MEM(&desc) = cpu_to_le32(CCP_MEMTYPE_SB); - CCP5_CMD_LSB_ID(&desc) = cpu_to_le32(op->sb_ctx); + CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; + CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; return ccp5_do_cmd(&desc, op->cmd_q); } From 2d158391061ec8c73898ceac148f4eddfa83efd5 Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Tue, 28 Mar 2017 10:57:26 -0500 Subject: [PATCH 060/101] crypto: ccp - Rearrange structure members to minimize size The AES GCM function (in ccp-ops) requires a fair amount of stack space, which elicits a complaint when KASAN is enabled. Rearranging and packing a few structures eliminates the warning. Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dev.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h index 3a45c2af2fbd..191274d41036 100644 --- a/drivers/crypto/ccp/ccp-dev.h +++ b/drivers/crypto/ccp/ccp-dev.h @@ -427,33 +427,33 @@ enum ccp_memtype { }; #define CCP_MEMTYPE_LSB CCP_MEMTYPE_KSB + struct ccp_dma_info { dma_addr_t address; unsigned int offset; unsigned int length; enum dma_data_direction dir; -}; +} __packed __aligned(4); struct ccp_dm_workarea { struct device *dev; struct dma_pool *dma_pool; - unsigned int length; u8 *address; struct ccp_dma_info dma; + unsigned int length; }; struct ccp_sg_workarea { struct scatterlist *sg; int nents; + unsigned int sg_used; struct scatterlist *dma_sg; struct device *dma_dev; unsigned int dma_count; enum dma_data_direction dma_dir; - unsigned int sg_used; - u64 bytes_left; }; From f275d3856cf597419293cd7d95aa628d3073f556 Mon Sep 17 00:00:00 2001 From: Haren Myneni Date: Sat, 1 Apr 2017 10:25:40 -0700 Subject: [PATCH 061/101] crypto: nx - Update MAINTAINERS entry for 842 compression Signed-off-by: Haren Myneni Acked-by: Dan Streetman Signed-off-by: Herbert Xu --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index d569516fd48a..620e63090275 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6211,7 +6211,7 @@ F: drivers/crypto/nx/nx_csbcpb.h F: drivers/crypto/nx/nx_debugfs.h IBM Power 842 compression accelerator -M: Dan Streetman +M: Haren Myneni S: Supported F: drivers/crypto/nx/Makefile F: drivers/crypto/nx/Kconfig From acb9b159c784dc0033ede0dadde876ebd93aca4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ondrej=20Mosn=C3=A1=C4=8Dek?= Date: Sun, 2 Apr 2017 21:19:13 +0200 Subject: [PATCH 062/101] crypto: gf128mul - define gf128mul_x_* in gf128mul.h The gf128mul_x_ble function is currently defined in gf128mul.c, because it depends on the gf128mul_table_be multiplication table. However, since the function is very small and only uses two values from the table, it is better for it to be defined as inline function in gf128mul.h. That way, the function can be inlined by the compiler for better performance. For consistency, the other gf128mul_x_* functions are also moved to the header file. In addition, the code is rewritten to be constant-time. After this change, the speed of the generic 'xts(aes)' implementation increased from ~225 MiB/s to ~235 MiB/s (measured using 'cryptsetup benchmark -c aes-xts-plain64' on an Intel system with CRYPTO_AES_X86_64 and CRYPTO_AES_NI_INTEL disabled). Signed-off-by: Ondrej Mosnacek Reviewd-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/gf128mul.c | 33 +---------------------- include/crypto/gf128mul.h | 55 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 34 deletions(-) diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c index 04facc0690aa..dc012129c063 100644 --- a/crypto/gf128mul.c +++ b/crypto/gf128mul.c @@ -130,43 +130,12 @@ static const u16 gf128mul_table_le[256] = gf128mul_dat(xda_le); static const u16 gf128mul_table_be[256] = gf128mul_dat(xda_be); /* - * The following functions multiply a field element by x or by x^8 in + * The following functions multiply a field element by x^8 in * the polynomial field representation. They use 64-bit word operations * to gain speed but compensate for machine endianness and hence work * correctly on both styles of machine. */ -static void gf128mul_x_lle(be128 *r, const be128 *x) -{ - u64 a = be64_to_cpu(x->a); - u64 b = be64_to_cpu(x->b); - u64 _tt = gf128mul_table_le[(b << 7) & 0xff]; - - r->b = cpu_to_be64((b >> 1) | (a << 63)); - r->a = cpu_to_be64((a >> 1) ^ (_tt << 48)); -} - -static void gf128mul_x_bbe(be128 *r, const be128 *x) -{ - u64 a = be64_to_cpu(x->a); - u64 b = be64_to_cpu(x->b); - u64 _tt = gf128mul_table_be[a >> 63]; - - r->a = cpu_to_be64((a << 1) | (b >> 63)); - r->b = cpu_to_be64((b << 1) ^ _tt); -} - -void gf128mul_x_ble(be128 *r, const be128 *x) -{ - u64 a = le64_to_cpu(x->a); - u64 b = le64_to_cpu(x->b); - u64 _tt = gf128mul_table_be[b >> 63]; - - r->a = cpu_to_le64((a << 1) ^ _tt); - r->b = cpu_to_le64((b << 1) | (a >> 63)); -} -EXPORT_SYMBOL(gf128mul_x_ble); - static void gf128mul_x8_lle(be128 *x) { u64 a = be64_to_cpu(x->a); diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h index 0bc9b5f1c45e..35ced9db70ea 100644 --- a/include/crypto/gf128mul.h +++ b/include/crypto/gf128mul.h @@ -49,6 +49,7 @@ #ifndef _CRYPTO_GF128MUL_H #define _CRYPTO_GF128MUL_H +#include #include #include @@ -163,8 +164,58 @@ void gf128mul_lle(be128 *a, const be128 *b); void gf128mul_bbe(be128 *a, const be128 *b); -/* multiply by x in ble format, needed by XTS */ -void gf128mul_x_ble(be128 *a, const be128 *b); +/* + * The following functions multiply a field element by x in + * the polynomial field representation. They use 64-bit word operations + * to gain speed but compensate for machine endianness and hence work + * correctly on both styles of machine. + * + * They are defined here for performance. + */ + +static inline u64 gf128mul_mask_from_bit(u64 x, int which) +{ + /* a constant-time version of 'x & ((u64)1 << which) ? (u64)-1 : 0' */ + return ((s64)(x << (63 - which)) >> 63); +} + +static inline void gf128mul_x_lle(be128 *r, const be128 *x) +{ + u64 a = be64_to_cpu(x->a); + u64 b = be64_to_cpu(x->b); + + /* equivalent to gf128mul_table_le[(b << 7) & 0xff] << 48 + * (see crypto/gf128mul.c): */ + u64 _tt = gf128mul_mask_from_bit(b, 0) & ((u64)0xe1 << 56); + + r->b = cpu_to_be64((b >> 1) | (a << 63)); + r->a = cpu_to_be64((a >> 1) ^ _tt); +} + +static inline void gf128mul_x_bbe(be128 *r, const be128 *x) +{ + u64 a = be64_to_cpu(x->a); + u64 b = be64_to_cpu(x->b); + + /* equivalent to gf128mul_table_be[a >> 63] (see crypto/gf128mul.c): */ + u64 _tt = gf128mul_mask_from_bit(a, 63) & 0x87; + + r->a = cpu_to_be64((a << 1) | (b >> 63)); + r->b = cpu_to_be64((b << 1) ^ _tt); +} + +/* needed by XTS */ +static inline void gf128mul_x_ble(be128 *r, const be128 *x) +{ + u64 a = le64_to_cpu(x->a); + u64 b = le64_to_cpu(x->b); + + /* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */ + u64 _tt = gf128mul_mask_from_bit(b, 63) & 0x87; + + r->a = cpu_to_le64((a << 1) ^ _tt); + r->b = cpu_to_le64((b << 1) | (a >> 63)); +} /* 4k table optimization */ From e55318c84f199d6056a0bcd98bc4612d01ccfe80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ondrej=20Mosn=C3=A1=C4=8Dek?= Date: Sun, 2 Apr 2017 21:19:14 +0200 Subject: [PATCH 063/101] crypto: gf128mul - switch gf128mul_x_ble to le128 Currently, gf128mul_x_ble works with pointers to be128, even though it actually interprets the words as little-endian. Consequently, it uses cpu_to_le64/le64_to_cpu on fields of type __be64, which is incorrect. This patch fixes that by changing the function to accept pointers to le128 and updating all users accordingly. Signed-off-by: Ondrej Mosnacek Reviewd-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/camellia_glue.c | 4 +-- arch/x86/crypto/serpent_sse2_glue.c | 4 +-- arch/x86/crypto/twofish_glue_3way.c | 4 +-- crypto/xts.c | 38 ++++++++++++++--------------- include/crypto/gf128mul.h | 8 +++--- include/crypto/xts.h | 2 +- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c index aa76cad9d262..af4840ab2a3d 100644 --- a/arch/x86/crypto/camellia_glue.c +++ b/arch/x86/crypto/camellia_glue.c @@ -1522,7 +1522,7 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[2 * 4]; + le128 buf[2 * 4]; struct xts_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), @@ -1540,7 +1540,7 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[2 * 4]; + le128 buf[2 * 4]; struct xts_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c index 644f97ab8cac..ac0e831943f5 100644 --- a/arch/x86/crypto/serpent_sse2_glue.c +++ b/arch/x86/crypto/serpent_sse2_glue.c @@ -328,7 +328,7 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[SERPENT_PARALLEL_BLOCKS]; + le128 buf[SERPENT_PARALLEL_BLOCKS]; struct crypt_priv crypt_ctx = { .ctx = &ctx->crypt_ctx, .fpu_enabled = false, @@ -355,7 +355,7 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[SERPENT_PARALLEL_BLOCKS]; + le128 buf[SERPENT_PARALLEL_BLOCKS]; struct crypt_priv crypt_ctx = { .ctx = &ctx->crypt_ctx, .fpu_enabled = false, diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c index 2ebb5e9789f3..243e90a4b5d9 100644 --- a/arch/x86/crypto/twofish_glue_3way.c +++ b/arch/x86/crypto/twofish_glue_3way.c @@ -296,7 +296,7 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[3]; + le128 buf[3]; struct xts_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), @@ -314,7 +314,7 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[3]; + le128 buf[3]; struct xts_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), diff --git a/crypto/xts.c b/crypto/xts.c index c976bfac29da..e197e64eb45c 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -39,11 +39,11 @@ struct xts_instance_ctx { }; struct rctx { - be128 buf[XTS_BUFFER_SIZE / sizeof(be128)]; + le128 buf[XTS_BUFFER_SIZE / sizeof(le128)]; - be128 t; + le128 t; - be128 *ext; + le128 *ext; struct scatterlist srcbuf[2]; struct scatterlist dstbuf[2]; @@ -99,7 +99,7 @@ static int setkey(struct crypto_skcipher *parent, const u8 *key, static int post_crypt(struct skcipher_request *req) { struct rctx *rctx = skcipher_request_ctx(req); - be128 *buf = rctx->ext ?: rctx->buf; + le128 *buf = rctx->ext ?: rctx->buf; struct skcipher_request *subreq; const int bs = XTS_BLOCK_SIZE; struct skcipher_walk w; @@ -112,12 +112,12 @@ static int post_crypt(struct skcipher_request *req) while (w.nbytes) { unsigned int avail = w.nbytes; - be128 *wdst; + le128 *wdst; wdst = w.dst.virt.addr; do { - be128_xor(wdst, buf++, wdst); + le128_xor(wdst, buf++, wdst); wdst++; } while ((avail -= bs) >= bs); @@ -150,7 +150,7 @@ out: static int pre_crypt(struct skcipher_request *req) { struct rctx *rctx = skcipher_request_ctx(req); - be128 *buf = rctx->ext ?: rctx->buf; + le128 *buf = rctx->ext ?: rctx->buf; struct skcipher_request *subreq; const int bs = XTS_BLOCK_SIZE; struct skcipher_walk w; @@ -174,15 +174,15 @@ static int pre_crypt(struct skcipher_request *req) while (w.nbytes) { unsigned int avail = w.nbytes; - be128 *wsrc; - be128 *wdst; + le128 *wsrc; + le128 *wdst; wsrc = w.src.virt.addr; wdst = w.dst.virt.addr; do { *buf++ = rctx->t; - be128_xor(wdst++, &rctx->t, wsrc++); + le128_xor(wdst++, &rctx->t, wsrc++); gf128mul_x_ble(&rctx->t, &rctx->t); } while ((avail -= bs) >= bs); @@ -353,8 +353,8 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst, const unsigned int max_blks = req->tbuflen / bsize; struct blkcipher_walk walk; unsigned int nblocks; - be128 *src, *dst, *t; - be128 *t_buf = req->tbuf; + le128 *src, *dst, *t; + le128 *t_buf = req->tbuf; int err, i; BUG_ON(max_blks < 1); @@ -367,8 +367,8 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst, return err; nblocks = min(nbytes / bsize, max_blks); - src = (be128 *)walk.src.virt.addr; - dst = (be128 *)walk.dst.virt.addr; + src = (le128 *)walk.src.virt.addr; + dst = (le128 *)walk.dst.virt.addr; /* calculate first value of T */ req->tweak_fn(req->tweak_ctx, (u8 *)&t_buf[0], walk.iv); @@ -384,7 +384,7 @@ first: t = &t_buf[i]; /* PP <- T xor P */ - be128_xor(dst + i, t, src + i); + le128_xor(dst + i, t, src + i); } /* CC <- E(Key2,PP) */ @@ -393,7 +393,7 @@ first: /* C <- T xor CC */ for (i = 0; i < nblocks; i++) - be128_xor(dst + i, dst + i, &t_buf[i]); + le128_xor(dst + i, dst + i, &t_buf[i]); src += nblocks; dst += nblocks; @@ -401,7 +401,7 @@ first: nblocks = min(nbytes / bsize, max_blks); } while (nblocks > 0); - *(be128 *)walk.iv = *t; + *(le128 *)walk.iv = *t; err = blkcipher_walk_done(desc, &walk, nbytes); nbytes = walk.nbytes; @@ -409,8 +409,8 @@ first: break; nblocks = min(nbytes / bsize, max_blks); - src = (be128 *)walk.src.virt.addr; - dst = (be128 *)walk.dst.virt.addr; + src = (le128 *)walk.src.virt.addr; + dst = (le128 *)walk.dst.virt.addr; } return err; diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h index 35ced9db70ea..0977fb18ff68 100644 --- a/include/crypto/gf128mul.h +++ b/include/crypto/gf128mul.h @@ -205,16 +205,16 @@ static inline void gf128mul_x_bbe(be128 *r, const be128 *x) } /* needed by XTS */ -static inline void gf128mul_x_ble(be128 *r, const be128 *x) +static inline void gf128mul_x_ble(le128 *r, const le128 *x) { u64 a = le64_to_cpu(x->a); u64 b = le64_to_cpu(x->b); /* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */ - u64 _tt = gf128mul_mask_from_bit(b, 63) & 0x87; + u64 _tt = gf128mul_mask_from_bit(a, 63) & 0x87; - r->a = cpu_to_le64((a << 1) ^ _tt); - r->b = cpu_to_le64((b << 1) | (a >> 63)); + r->a = cpu_to_le64((a << 1) | (b >> 63)); + r->b = cpu_to_le64((b << 1) ^ _tt); } /* 4k table optimization */ diff --git a/include/crypto/xts.h b/include/crypto/xts.h index 77b630672b2c..c0bde308b28a 100644 --- a/include/crypto/xts.h +++ b/include/crypto/xts.h @@ -11,7 +11,7 @@ struct blkcipher_desc; #define XTS_BLOCK_SIZE 16 struct xts_crypt_req { - be128 *tbuf; + le128 *tbuf; unsigned int tbuflen; void *tweak_ctx; From 692016bdf7cf33abaaa7f2b080b47f504c98810c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ondrej=20Mosn=C3=A1=C4=8Dek?= Date: Sun, 2 Apr 2017 21:19:15 +0200 Subject: [PATCH 064/101] crypto: glue_helper - remove the le128_gf128mul_x_ble function The le128_gf128mul_x_ble function in glue_helper.h is now obsolete and can be replaced with the gf128mul_x_ble function from gf128mul.h. Signed-off-by: Ondrej Mosnacek Reviewd-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/glue_helper.c | 3 ++- arch/x86/include/asm/crypto/glue_helper.h | 10 ---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c index 260a060d7275..24ac9fad832d 100644 --- a/arch/x86/crypto/glue_helper.c +++ b/arch/x86/crypto/glue_helper.c @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -457,7 +458,7 @@ void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, le128 *iv, le128 ivblk = *iv; /* generate next IV */ - le128_gf128mul_x_ble(iv, &ivblk); + gf128mul_x_ble(iv, &ivblk); /* CC <- T xor C */ u128_xor(dst, src, (u128 *)&ivblk); diff --git a/arch/x86/include/asm/crypto/glue_helper.h b/arch/x86/include/asm/crypto/glue_helper.h index 29e53ea7d764..ed8b66de541f 100644 --- a/arch/x86/include/asm/crypto/glue_helper.h +++ b/arch/x86/include/asm/crypto/glue_helper.h @@ -125,16 +125,6 @@ static inline void le128_inc(le128 *i) i->b = cpu_to_le64(b); } -static inline void le128_gf128mul_x_ble(le128 *dst, const le128 *src) -{ - u64 a = le64_to_cpu(src->a); - u64 b = le64_to_cpu(src->b); - u64 _tt = ((s64)a >> 63) & 0x87; - - dst->a = cpu_to_le64((a << 1) ^ (b >> 63)); - dst->b = cpu_to_le64((b << 1) ^ _tt); -} - extern int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx, struct blkcipher_desc *desc, struct scatterlist *dst, From ad1064cd612e11b807eb764140deb5c1875ca5dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ondrej=20Mosn=C3=A1=C4=8Dek?= Date: Sun, 2 Apr 2017 21:19:16 +0200 Subject: [PATCH 065/101] crypto: xts - drop gf128mul dependency Since the gf128mul_x_ble function used by xts.c is now defined inline in the header file, the XTS module no longer depends on gf128mul. Therefore, the 'select CRYPTO_GF128MUL' line can be safely removed. Signed-off-by: Ondrej Mosnacek Reviewd-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 6854c1fe54b7..aac4bc90a138 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -374,7 +374,6 @@ config CRYPTO_XTS tristate "XTS support" select CRYPTO_BLKCIPHER select CRYPTO_MANAGER - select CRYPTO_GF128MUL select CRYPTO_ECB help XTS: IEEE1619/D16 narrow block cipher use with aes-xts-plain, From 42ae2922a68ac8d68927ccb052b486f34e5fba71 Mon Sep 17 00:00:00 2001 From: Matthias Kaehlcke Date: Wed, 5 Apr 2017 11:34:58 -0700 Subject: [PATCH 066/101] crypto: arm64/sha - Add constant operand modifier to ASM_EXPORT The operand is an integer constant, make the constness explicit by adding the modifier. This is needed for clang to generate valid code and also works with gcc. Also change the constraint of the operand from 'I' ("Integer constant that is valid as an immediate operand in an ADD instruction", AArch64) to 'i' ("An immediate integer operand"). Based-on-patch-from: Greg Hackmann Signed-off-by: Greg Hackmann Signed-off-by: Matthias Kaehlcke Reviewed-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha1-ce-glue.c | 2 +- arch/arm64/crypto/sha2-ce-glue.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index aefda9868627..6b520e3f3ab1 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -18,7 +18,7 @@ #include #define ASM_EXPORT(sym, val) \ - asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val)); + asm(".globl " #sym "; .set " #sym ", %c0" :: "i"(val)); MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index 7cd587564a41..e3abe11de48c 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -18,7 +18,7 @@ #include #define ASM_EXPORT(sym, val) \ - asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val)); + asm(".globl " #sym "; .set " #sym ", %c0" :: "i"(val)); MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); From 7acd4de7f2427c326776d49d630bd3530dd54ea6 Mon Sep 17 00:00:00 2001 From: Rick Altherr Date: Wed, 5 Apr 2017 16:20:58 -0700 Subject: [PATCH 067/101] hwrng: timeriomem - Migrate to new API Preserves the existing behavior of only returning 32-bits per call. Signed-off-by: Rick Altherr Signed-off-by: Herbert Xu --- drivers/char/hw_random/timeriomem-rng.c | 62 ++++++++++++------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c index cf37db263ecd..17574452fd35 100644 --- a/drivers/char/hw_random/timeriomem-rng.c +++ b/drivers/char/hw_random/timeriomem-rng.c @@ -20,18 +20,16 @@ * TODO: add support for reading sizes other than 32bits and masking */ -#include -#include -#include -#include +#include #include #include +#include +#include +#include +#include #include #include -#include -#include #include -#include struct timeriomem_rng_private_data { void __iomem *io_base; @@ -45,32 +43,36 @@ struct timeriomem_rng_private_data { struct hwrng timeriomem_rng_ops; }; -#define to_rng_priv(rng) \ - ((struct timeriomem_rng_private_data *)rng->priv) - -/* - * have data return 1, however return 0 if we have nothing - */ -static int timeriomem_rng_data_present(struct hwrng *rng, int wait) +static int timeriomem_rng_read(struct hwrng *hwrng, void *data, + size_t max, bool wait) { - struct timeriomem_rng_private_data *priv = to_rng_priv(rng); - - if (!wait || priv->present) - return priv->present; - - wait_for_completion(&priv->completion); - - return 1; -} - -static int timeriomem_rng_data_read(struct hwrng *rng, u32 *data) -{ - struct timeriomem_rng_private_data *priv = to_rng_priv(rng); + struct timeriomem_rng_private_data *priv = + container_of(hwrng, struct timeriomem_rng_private_data, + timeriomem_rng_ops); unsigned long cur; s32 delay; - *data = readl(priv->io_base); + /* The RNG provides 32-bit per read. Ensure there is enough space. */ + if (max < sizeof(u32)) + return 0; + /* + * There may not have been enough time for new data to be generated + * since the last request. If the caller doesn't want to wait, let them + * bail out. Otherwise, wait for the completion. If the new data has + * already been generated, the completion should already be available. + */ + if (!wait && !priv->present) + return 0; + + wait_for_completion(&priv->completion); + + *(u32 *)data = readl(priv->io_base); + + /* + * Block any new callers until the RNG has had time to generate new + * data. + */ cur = jiffies; delay = cur - priv->expires; @@ -154,9 +156,7 @@ static int timeriomem_rng_probe(struct platform_device *pdev) setup_timer(&priv->timer, timeriomem_rng_trigger, (unsigned long)priv); priv->timeriomem_rng_ops.name = dev_name(&pdev->dev); - priv->timeriomem_rng_ops.data_present = timeriomem_rng_data_present; - priv->timeriomem_rng_ops.data_read = timeriomem_rng_data_read; - priv->timeriomem_rng_ops.priv = (unsigned long)priv; + priv->timeriomem_rng_ops.read = timeriomem_rng_read; priv->io_base = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(priv->io_base)) { From 5ab693e63a93c387171b4f7834b2f7bfe9b55f35 Mon Sep 17 00:00:00 2001 From: Rick Altherr Date: Wed, 5 Apr 2017 16:20:59 -0700 Subject: [PATCH 068/101] hwrng: timeriomem - Shorten verbose type and variable names No functional changes. Signed-off-by: Rick Altherr Signed-off-by: Herbert Xu --- drivers/char/hw_random/timeriomem-rng.c | 27 ++++++++++++------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c index 17574452fd35..024bdff7999f 100644 --- a/drivers/char/hw_random/timeriomem-rng.c +++ b/drivers/char/hw_random/timeriomem-rng.c @@ -31,7 +31,7 @@ #include #include -struct timeriomem_rng_private_data { +struct timeriomem_rng_private { void __iomem *io_base; unsigned int expires; unsigned int period; @@ -40,15 +40,14 @@ struct timeriomem_rng_private_data { struct timer_list timer; struct completion completion; - struct hwrng timeriomem_rng_ops; + struct hwrng rng_ops; }; static int timeriomem_rng_read(struct hwrng *hwrng, void *data, size_t max, bool wait) { - struct timeriomem_rng_private_data *priv = - container_of(hwrng, struct timeriomem_rng_private_data, - timeriomem_rng_ops); + struct timeriomem_rng_private *priv = + container_of(hwrng, struct timeriomem_rng_private, rng_ops); unsigned long cur; s32 delay; @@ -89,8 +88,8 @@ static int timeriomem_rng_read(struct hwrng *hwrng, void *data, static void timeriomem_rng_trigger(unsigned long data) { - struct timeriomem_rng_private_data *priv - = (struct timeriomem_rng_private_data *)data; + struct timeriomem_rng_private *priv + = (struct timeriomem_rng_private *)data; priv->present = 1; complete(&priv->completion); @@ -99,7 +98,7 @@ static void timeriomem_rng_trigger(unsigned long data) static int timeriomem_rng_probe(struct platform_device *pdev) { struct timeriomem_rng_data *pdata = pdev->dev.platform_data; - struct timeriomem_rng_private_data *priv; + struct timeriomem_rng_private *priv; struct resource *res; int err = 0; int period; @@ -121,7 +120,7 @@ static int timeriomem_rng_probe(struct platform_device *pdev) /* Allocate memory for the device structure (and zero it) */ priv = devm_kzalloc(&pdev->dev, - sizeof(struct timeriomem_rng_private_data), GFP_KERNEL); + sizeof(struct timeriomem_rng_private), GFP_KERNEL); if (!priv) return -ENOMEM; @@ -155,8 +154,8 @@ static int timeriomem_rng_probe(struct platform_device *pdev) setup_timer(&priv->timer, timeriomem_rng_trigger, (unsigned long)priv); - priv->timeriomem_rng_ops.name = dev_name(&pdev->dev); - priv->timeriomem_rng_ops.read = timeriomem_rng_read; + priv->rng_ops.name = dev_name(&pdev->dev); + priv->rng_ops.read = timeriomem_rng_read; priv->io_base = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(priv->io_base)) { @@ -164,7 +163,7 @@ static int timeriomem_rng_probe(struct platform_device *pdev) goto out_timer; } - err = hwrng_register(&priv->timeriomem_rng_ops); + err = hwrng_register(&priv->rng_ops); if (err) { dev_err(&pdev->dev, "problem registering\n"); goto out_timer; @@ -182,9 +181,9 @@ out_timer: static int timeriomem_rng_remove(struct platform_device *pdev) { - struct timeriomem_rng_private_data *priv = platform_get_drvdata(pdev); + struct timeriomem_rng_private *priv = platform_get_drvdata(pdev); - hwrng_unregister(&priv->timeriomem_rng_ops); + hwrng_unregister(&priv->rng_ops); del_timer_sync(&priv->timer); From ca3bff70ab320a9132c5524c495455526df4b078 Mon Sep 17 00:00:00 2001 From: Rick Altherr Date: Wed, 5 Apr 2017 16:21:00 -0700 Subject: [PATCH 069/101] hwrng: timeriomem - Improve performance for sub-jiffie update periods Some hardware RNGs provide a single register for obtaining random data. Instead of signaling when new data is available, the reader must wait a fixed amount of time between reads for new data to be generated. timeriomem_rng implements this scheme with the period specified in platform data or device tree. While the period is specified in microseconds, the implementation used a standard timer which has a minimum delay of 1 jiffie and caused a significant bottleneck for devices that can update at 1us. By switching to an hrtimer, 1us periods now only delay at most 2us per read. Signed-off-by: Rick Altherr Signed-off-by: Herbert Xu --- drivers/char/hw_random/timeriomem-rng.c | 86 +++++++++++++------------ 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c index 024bdff7999f..a0faa5f05deb 100644 --- a/drivers/char/hw_random/timeriomem-rng.c +++ b/drivers/char/hw_random/timeriomem-rng.c @@ -21,23 +21,24 @@ */ #include +#include +#include #include #include -#include +#include #include #include #include #include +#include #include -#include struct timeriomem_rng_private { void __iomem *io_base; - unsigned int expires; - unsigned int period; + ktime_t period; unsigned int present:1; - struct timer_list timer; + struct hrtimer timer; struct completion completion; struct hwrng rng_ops; @@ -48,10 +49,13 @@ static int timeriomem_rng_read(struct hwrng *hwrng, void *data, { struct timeriomem_rng_private *priv = container_of(hwrng, struct timeriomem_rng_private, rng_ops); - unsigned long cur; - s32 delay; + int retval = 0; + int period_us = ktime_to_us(priv->period); - /* The RNG provides 32-bit per read. Ensure there is enough space. */ + /* + * The RNG provides 32-bits per read. Ensure there is enough space for + * at minimum one read. + */ if (max < sizeof(u32)) return 0; @@ -66,33 +70,44 @@ static int timeriomem_rng_read(struct hwrng *hwrng, void *data, wait_for_completion(&priv->completion); - *(u32 *)data = readl(priv->io_base); + do { + /* + * After the first read, all additional reads will need to wait + * for the RNG to generate new data. Since the period can have + * a wide range of values (1us to 1s have been observed), allow + * for 1% tolerance in the sleep time rather than a fixed value. + */ + if (retval > 0) + usleep_range(period_us, + period_us + min(1, period_us / 100)); + + *(u32 *)data = readl(priv->io_base); + retval += sizeof(u32); + data += sizeof(u32); + max -= sizeof(u32); + } while (wait && max > sizeof(u32)); /* * Block any new callers until the RNG has had time to generate new * data. */ - cur = jiffies; - - delay = cur - priv->expires; - delay = priv->period - (delay % priv->period); - - priv->expires = cur + delay; priv->present = 0; - reinit_completion(&priv->completion); - mod_timer(&priv->timer, priv->expires); + hrtimer_forward_now(&priv->timer, priv->period); + hrtimer_restart(&priv->timer); - return 4; + return retval; } -static void timeriomem_rng_trigger(unsigned long data) +static enum hrtimer_restart timeriomem_rng_trigger(struct hrtimer *timer) { struct timeriomem_rng_private *priv - = (struct timeriomem_rng_private *)data; + = container_of(timer, struct timeriomem_rng_private, timer); priv->present = 1; complete(&priv->completion); + + return HRTIMER_NORESTART; } static int timeriomem_rng_probe(struct platform_device *pdev) @@ -140,43 +155,33 @@ static int timeriomem_rng_probe(struct platform_device *pdev) period = pdata->period; } - priv->period = usecs_to_jiffies(period); - if (priv->period < 1) { - dev_err(&pdev->dev, "period is less than one jiffy\n"); - return -EINVAL; - } - - priv->expires = jiffies; - priv->present = 1; - + priv->period = ns_to_ktime(period * NSEC_PER_USEC); init_completion(&priv->completion); - complete(&priv->completion); - - setup_timer(&priv->timer, timeriomem_rng_trigger, (unsigned long)priv); + hrtimer_init(&priv->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + priv->timer.function = timeriomem_rng_trigger; priv->rng_ops.name = dev_name(&pdev->dev); priv->rng_ops.read = timeriomem_rng_read; priv->io_base = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(priv->io_base)) { - err = PTR_ERR(priv->io_base); - goto out_timer; + return PTR_ERR(priv->io_base); } + /* Assume random data is already available. */ + priv->present = 1; + complete(&priv->completion); + err = hwrng_register(&priv->rng_ops); if (err) { dev_err(&pdev->dev, "problem registering\n"); - goto out_timer; + return err; } dev_info(&pdev->dev, "32bits from 0x%p @ %dus\n", priv->io_base, period); return 0; - -out_timer: - del_timer_sync(&priv->timer); - return err; } static int timeriomem_rng_remove(struct platform_device *pdev) @@ -184,8 +189,7 @@ static int timeriomem_rng_remove(struct platform_device *pdev) struct timeriomem_rng_private *priv = platform_get_drvdata(pdev); hwrng_unregister(&priv->rng_ops); - - del_timer_sync(&priv->timer); + hrtimer_cancel(&priv->timer); return 0; } From 4473710df1f8779c59b33737eeaa151596907761 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 6 Apr 2017 16:16:08 +0800 Subject: [PATCH 070/101] crypto: user - Prepare for CRYPTO_MAX_ALG_NAME expansion This patch hard-codes CRYPTO_MAX_NAME in the user-space API to 64, which is the current value of CRYPTO_MAX_ALG_NAME. This patch also replaces all remaining occurences of CRYPTO_MAX_ALG_NAME in the user-space API with CRYPTO_MAX_NAME. This way the user-space API will not be modified when we raise the value of CRYPTO_MAX_ALG_NAME. Furthermore, the code has been updated to handle names longer than the user-space API. They will be truncated. Signed-off-by: Herbert Xu Acked-by: Alexander Sverdlin Tested-by: Alexander Sverdlin --- crypto/crypto_user.c | 18 +++++++++--------- include/uapi/linux/cryptouser.h | 10 +++++----- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c index a90404a0c5ff..89acaab1d909 100644 --- a/crypto/crypto_user.c +++ b/crypto/crypto_user.c @@ -83,7 +83,7 @@ static int crypto_report_cipher(struct sk_buff *skb, struct crypto_alg *alg) { struct crypto_report_cipher rcipher; - strncpy(rcipher.type, "cipher", sizeof(rcipher.type)); + strlcpy(rcipher.type, "cipher", sizeof(rcipher.type)); rcipher.blocksize = alg->cra_blocksize; rcipher.min_keysize = alg->cra_cipher.cia_min_keysize; @@ -102,7 +102,7 @@ static int crypto_report_comp(struct sk_buff *skb, struct crypto_alg *alg) { struct crypto_report_comp rcomp; - strncpy(rcomp.type, "compression", sizeof(rcomp.type)); + strlcpy(rcomp.type, "compression", sizeof(rcomp.type)); if (nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS, sizeof(struct crypto_report_comp), &rcomp)) goto nla_put_failure; @@ -116,7 +116,7 @@ static int crypto_report_acomp(struct sk_buff *skb, struct crypto_alg *alg) { struct crypto_report_acomp racomp; - strncpy(racomp.type, "acomp", sizeof(racomp.type)); + strlcpy(racomp.type, "acomp", sizeof(racomp.type)); if (nla_put(skb, CRYPTOCFGA_REPORT_ACOMP, sizeof(struct crypto_report_acomp), &racomp)) @@ -131,7 +131,7 @@ static int crypto_report_akcipher(struct sk_buff *skb, struct crypto_alg *alg) { struct crypto_report_akcipher rakcipher; - strncpy(rakcipher.type, "akcipher", sizeof(rakcipher.type)); + strlcpy(rakcipher.type, "akcipher", sizeof(rakcipher.type)); if (nla_put(skb, CRYPTOCFGA_REPORT_AKCIPHER, sizeof(struct crypto_report_akcipher), &rakcipher)) @@ -146,7 +146,7 @@ static int crypto_report_kpp(struct sk_buff *skb, struct crypto_alg *alg) { struct crypto_report_kpp rkpp; - strncpy(rkpp.type, "kpp", sizeof(rkpp.type)); + strlcpy(rkpp.type, "kpp", sizeof(rkpp.type)); if (nla_put(skb, CRYPTOCFGA_REPORT_KPP, sizeof(struct crypto_report_kpp), &rkpp)) @@ -160,10 +160,10 @@ nla_put_failure: static int crypto_report_one(struct crypto_alg *alg, struct crypto_user_alg *ualg, struct sk_buff *skb) { - strncpy(ualg->cru_name, alg->cra_name, sizeof(ualg->cru_name)); - strncpy(ualg->cru_driver_name, alg->cra_driver_name, + strlcpy(ualg->cru_name, alg->cra_name, sizeof(ualg->cru_name)); + strlcpy(ualg->cru_driver_name, alg->cra_driver_name, sizeof(ualg->cru_driver_name)); - strncpy(ualg->cru_module_name, module_name(alg->cra_module), + strlcpy(ualg->cru_module_name, module_name(alg->cra_module), sizeof(ualg->cru_module_name)); ualg->cru_type = 0; @@ -176,7 +176,7 @@ static int crypto_report_one(struct crypto_alg *alg, if (alg->cra_flags & CRYPTO_ALG_LARVAL) { struct crypto_report_larval rl; - strncpy(rl.type, "larval", sizeof(rl.type)); + strlcpy(rl.type, "larval", sizeof(rl.type)); if (nla_put(skb, CRYPTOCFGA_REPORT_LARVAL, sizeof(struct crypto_report_larval), &rl)) goto nla_put_failure; diff --git a/include/uapi/linux/cryptouser.h b/include/uapi/linux/cryptouser.h index 11d21fce14d6..b4def5c630e7 100644 --- a/include/uapi/linux/cryptouser.h +++ b/include/uapi/linux/cryptouser.h @@ -31,7 +31,7 @@ enum { #define CRYPTO_MSG_MAX (__CRYPTO_MSG_MAX - 1) #define CRYPTO_NR_MSGTYPES (CRYPTO_MSG_MAX + 1 - CRYPTO_MSG_BASE) -#define CRYPTO_MAX_NAME CRYPTO_MAX_ALG_NAME +#define CRYPTO_MAX_NAME 64 /* Netlink message attributes. */ enum crypto_attr_type_t { @@ -53,9 +53,9 @@ enum crypto_attr_type_t { }; struct crypto_user_alg { - char cru_name[CRYPTO_MAX_ALG_NAME]; - char cru_driver_name[CRYPTO_MAX_ALG_NAME]; - char cru_module_name[CRYPTO_MAX_ALG_NAME]; + char cru_name[CRYPTO_MAX_NAME]; + char cru_driver_name[CRYPTO_MAX_NAME]; + char cru_module_name[CRYPTO_MAX_NAME]; __u32 cru_type; __u32 cru_mask; __u32 cru_refcnt; @@ -73,7 +73,7 @@ struct crypto_report_hash { }; struct crypto_report_cipher { - char type[CRYPTO_MAX_ALG_NAME]; + char type[CRYPTO_MAX_NAME]; unsigned int blocksize; unsigned int min_keysize; unsigned int max_keysize; From 3f69cc60768b2cd15e5cbd3a4b91c46535138382 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 6 Apr 2017 16:16:09 +0800 Subject: [PATCH 071/101] crypto: af_alg - Allow arbitrarily long algorithm names This patch removes the hard-coded 64-byte limit on the length of the algorithm name through bind(2). The address length can now exceed that. The user-space structure remains unchanged. In order to use a longer name simply extend the salg_name array beyond its defined 64 bytes length. Signed-off-by: Herbert Xu --- crypto/af_alg.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 690deca17c35..3556d8eb54a7 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -160,11 +160,11 @@ static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) if (sock->state == SS_CONNECTED) return -EINVAL; - if (addr_len != sizeof(*sa)) + if (addr_len < sizeof(*sa)) return -EINVAL; sa->salg_type[sizeof(sa->salg_type) - 1] = 0; - sa->salg_name[sizeof(sa->salg_name) - 1] = 0; + sa->salg_name[sizeof(sa->salg_name) + addr_len - sizeof(*sa) - 1] = 0; type = alg_get_type(sa->salg_type); if (IS_ERR(type) && PTR_ERR(type) == -ENOENT) { From 633439f5b7e9e2a88fb2ead38bc12c5061897cf2 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 6 Apr 2017 16:16:10 +0800 Subject: [PATCH 072/101] xfrm: Prepare for CRYPTO_MAX_ALG_NAME expansion This patch fixes the xfrm_user code to use the actual array size rather than the hard-coded CRYPTO_MAX_ALG_NAME length. This is because the array size is fixed at 64 bytes while we want to increase the in-kernel CRYPTO_MAX_ALG_NAME value. Signed-off-by: Herbert Xu Acked-by: Alexander Sverdlin Tested-by: Alexander Sverdlin Acked-by: Steffen Klassert --- net/xfrm/xfrm_user.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index 9705c279494b..96557cf6389e 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -55,7 +55,7 @@ static int verify_one_alg(struct nlattr **attrs, enum xfrm_attr_type_t type) return -EINVAL; } - algp->alg_name[CRYPTO_MAX_ALG_NAME - 1] = '\0'; + algp->alg_name[sizeof(algp->alg_name) - 1] = '\0'; return 0; } @@ -71,7 +71,7 @@ static int verify_auth_trunc(struct nlattr **attrs) if (nla_len(rt) < xfrm_alg_auth_len(algp)) return -EINVAL; - algp->alg_name[CRYPTO_MAX_ALG_NAME - 1] = '\0'; + algp->alg_name[sizeof(algp->alg_name) - 1] = '\0'; return 0; } @@ -87,7 +87,7 @@ static int verify_aead(struct nlattr **attrs) if (nla_len(rt) < aead_len(algp)) return -EINVAL; - algp->alg_name[CRYPTO_MAX_ALG_NAME - 1] = '\0'; + algp->alg_name[sizeof(algp->alg_name) - 1] = '\0'; return 0; } From f437a3f477cce402dbec6537b29e9e33962c9f73 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 6 Apr 2017 16:16:11 +0800 Subject: [PATCH 073/101] crypto: api - Extend algorithm name limit to 128 bytes With the new explicit IV generators, we may now exceed the 64-byte length limit on the algorithm name, e.g., with echainiv(authencesn(hmac(sha256-generic),cbc(des3_ede-generic))) This patch extends the length limit to 128 bytes. Reported-by: Alexander Sverdlin Signed-off-by: Herbert Xu Acked-by: Alexander Sverdlin Tested-by: Alexander Sverdlin --- include/linux/crypto.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/crypto.h b/include/linux/crypto.h index c0b0cf3d2d2f..84da9978e951 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -123,7 +123,7 @@ /* * Miscellaneous stuff. */ -#define CRYPTO_MAX_ALG_NAME 64 +#define CRYPTO_MAX_ALG_NAME 128 /* * The macro CRYPTO_MINALIGN_ATTR (along with the void * type in the actual From 07a77929ba672d93642a56dc2255dd21e6e2290b Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Fri, 7 Apr 2017 02:33:30 +0200 Subject: [PATCH 074/101] padata: free correct variable The author meant to free the variable that was just allocated, instead of the one that failed to be allocated, but made a simple typo. This patch rectifies that. Signed-off-by: Jason A. Donenfeld Cc: stable@vger.kernel.org Signed-off-by: Herbert Xu --- kernel/padata.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/padata.c b/kernel/padata.c index 3202aa17492c..f1aef1639204 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -354,7 +354,7 @@ static int padata_setup_cpumasks(struct parallel_data *pd, cpumask_and(pd->cpumask.pcpu, pcpumask, cpu_online_mask); if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL)) { - free_cpumask_var(pd->cpumask.cbcpu); + free_cpumask_var(pd->cpumask.pcpu); return -ENOMEM; } From cd15f1020fd627d795e60a142d1f00f7fc1fe7f3 Mon Sep 17 00:00:00 2001 From: Myungho Jung Date: Sun, 9 Apr 2017 17:34:22 -0700 Subject: [PATCH 075/101] crypto: lz4 - fixed decompress function to return error code Decompress function in LZ4 library is supposed to return an error code or negative result. But, it returns -1 when any error is detected. Return error code when the library returns negative value. Signed-off-by: Myungho Jung Signed-off-by: Herbert Xu --- crypto/lz4.c | 2 +- crypto/lz4hc.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/lz4.c b/crypto/lz4.c index 71eff9b01b12..2ce2660d3519 100644 --- a/crypto/lz4.c +++ b/crypto/lz4.c @@ -97,7 +97,7 @@ static int __lz4_decompress_crypto(const u8 *src, unsigned int slen, int out_len = LZ4_decompress_safe(src, dst, slen, *dlen); if (out_len < 0) - return out_len; + return -EINVAL; *dlen = out_len; return 0; diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c index 03a34a8109c0..2be14f054daf 100644 --- a/crypto/lz4hc.c +++ b/crypto/lz4hc.c @@ -98,7 +98,7 @@ static int __lz4hc_decompress_crypto(const u8 *src, unsigned int slen, int out_len = LZ4_decompress_safe(src, dst, slen, *dlen); if (out_len < 0) - return out_len; + return -EINVAL; *dlen = out_len; return 0; From e29abda591b517be47df7479962ce6baee302a85 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Mon, 10 Apr 2017 18:23:58 +0530 Subject: [PATCH 076/101] crypto: chcr - Increase priority of AEAD algos. templates(gcm,ccm etc) inherit priority value of driver to calculate its priority. In some cases template priority becomes more than driver priority for same algo. Without this patch we will not be able to use driver authenc algos. It will be good if it pushed in stable kernel. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 12 ++++++++++-- drivers/crypto/chelsio/chcr_crypto.h | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 41bc7f4f58cd..7d595912909c 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -2673,6 +2673,7 @@ static struct chcr_alg_template driver_algs[] = { .cra_name = "gcm(aes)", .cra_driver_name = "gcm-aes-chcr", .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, .cra_ctxsize = sizeof(struct chcr_context) + sizeof(struct chcr_aead_ctx) + sizeof(struct chcr_gcm_ctx), @@ -2691,6 +2692,7 @@ static struct chcr_alg_template driver_algs[] = { .cra_name = "rfc4106(gcm(aes))", .cra_driver_name = "rfc4106-gcm-aes-chcr", .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY + 1, .cra_ctxsize = sizeof(struct chcr_context) + sizeof(struct chcr_aead_ctx) + sizeof(struct chcr_gcm_ctx), @@ -2710,6 +2712,7 @@ static struct chcr_alg_template driver_algs[] = { .cra_name = "ccm(aes)", .cra_driver_name = "ccm-aes-chcr", .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, .cra_ctxsize = sizeof(struct chcr_context) + sizeof(struct chcr_aead_ctx), @@ -2728,6 +2731,7 @@ static struct chcr_alg_template driver_algs[] = { .cra_name = "rfc4309(ccm(aes))", .cra_driver_name = "rfc4309-ccm-aes-chcr", .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY + 1, .cra_ctxsize = sizeof(struct chcr_context) + sizeof(struct chcr_aead_ctx), @@ -2747,6 +2751,7 @@ static struct chcr_alg_template driver_algs[] = { .cra_driver_name = "authenc-hmac-sha1-cbc-aes-chcr", .cra_blocksize = AES_BLOCK_SIZE, + .cra_priority = CHCR_AEAD_PRIORITY, .cra_ctxsize = sizeof(struct chcr_context) + sizeof(struct chcr_aead_ctx) + sizeof(struct chcr_authenc_ctx), @@ -2768,6 +2773,7 @@ static struct chcr_alg_template driver_algs[] = { .cra_driver_name = "authenc-hmac-sha256-cbc-aes-chcr", .cra_blocksize = AES_BLOCK_SIZE, + .cra_priority = CHCR_AEAD_PRIORITY, .cra_ctxsize = sizeof(struct chcr_context) + sizeof(struct chcr_aead_ctx) + sizeof(struct chcr_authenc_ctx), @@ -2788,6 +2794,7 @@ static struct chcr_alg_template driver_algs[] = { .cra_driver_name = "authenc-hmac-sha224-cbc-aes-chcr", .cra_blocksize = AES_BLOCK_SIZE, + .cra_priority = CHCR_AEAD_PRIORITY, .cra_ctxsize = sizeof(struct chcr_context) + sizeof(struct chcr_aead_ctx) + sizeof(struct chcr_authenc_ctx), @@ -2807,6 +2814,7 @@ static struct chcr_alg_template driver_algs[] = { .cra_driver_name = "authenc-hmac-sha384-cbc-aes-chcr", .cra_blocksize = AES_BLOCK_SIZE, + .cra_priority = CHCR_AEAD_PRIORITY, .cra_ctxsize = sizeof(struct chcr_context) + sizeof(struct chcr_aead_ctx) + sizeof(struct chcr_authenc_ctx), @@ -2827,6 +2835,7 @@ static struct chcr_alg_template driver_algs[] = { .cra_driver_name = "authenc-hmac-sha512-cbc-aes-chcr", .cra_blocksize = AES_BLOCK_SIZE, + .cra_priority = CHCR_AEAD_PRIORITY, .cra_ctxsize = sizeof(struct chcr_context) + sizeof(struct chcr_aead_ctx) + sizeof(struct chcr_authenc_ctx), @@ -2847,6 +2856,7 @@ static struct chcr_alg_template driver_algs[] = { .cra_driver_name = "authenc-digest_null-cbc-aes-chcr", .cra_blocksize = AES_BLOCK_SIZE, + .cra_priority = CHCR_AEAD_PRIORITY, .cra_ctxsize = sizeof(struct chcr_context) + sizeof(struct chcr_aead_ctx) + sizeof(struct chcr_authenc_ctx), @@ -2915,8 +2925,6 @@ static int chcr_register_alg(void) name = driver_algs[i].alg.crypto.cra_driver_name; break; case CRYPTO_ALG_TYPE_AEAD: - driver_algs[i].alg.aead.base.cra_priority = - CHCR_CRA_PRIORITY; driver_algs[i].alg.aead.base.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC; driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt; diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 81cfd0ba132e..4469feae84a2 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -48,8 +48,8 @@ * giving the processed data */ -#define CHCR_CRA_PRIORITY 3000 - +#define CHCR_CRA_PRIORITY 500 +#define CHCR_AEAD_PRIORITY 6000 #define CHCR_AES_MAX_KEY_LEN (2 * (AES_MAX_KEY_SIZE)) /* consider xts */ #define CHCR_MAX_CRYPTO_IV_LEN 16 /* AES IV len */ From 0a7bd30c1b7fca5f1443f1bc2dfdc2681d315359 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Mon, 10 Apr 2017 18:23:59 +0530 Subject: [PATCH 077/101] crypto: chcr - Set hmac_ctrl bit to use HW register HMAC_CFG[456] Use hmac_ctrl bit value saved in setauthsize callback. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 7d595912909c..2d610437bf6e 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -1335,19 +1335,6 @@ static int chcr_copy_assoc(struct aead_request *req, return crypto_skcipher_encrypt(skreq); } -static unsigned char get_hmac(unsigned int authsize) -{ - switch (authsize) { - case ICV_8: - return CHCR_SCMD_HMAC_CTRL_PL1; - case ICV_10: - return CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; - case ICV_12: - return CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; - } - return CHCR_SCMD_HMAC_CTRL_NO_TRUNC; -} - static struct sk_buff *create_authenc_wr(struct aead_request *req, unsigned short qid, @@ -1600,13 +1587,13 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, struct chcr_context *chcrctx) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); unsigned int ivsize = AES_BLOCK_SIZE; unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; unsigned int c_id = chcrctx->dev->rx_channel_id; unsigned int ccm_xtra; unsigned char tag_offset = 0, auth_offset = 0; - unsigned char hmac_ctrl = get_hmac(crypto_aead_authsize(tfm)); unsigned int assoclen; if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) @@ -1642,8 +1629,8 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, crypto_aead_authsize(tfm)); sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 0 : 1, - cipher_mode, mac_mode, hmac_ctrl, - ivsize >> 1); + cipher_mode, mac_mode, + aeadctx->hmac_ctrl, ivsize >> 1); sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, 1, dst_size); @@ -1820,7 +1807,6 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, unsigned char tag_offset = 0; unsigned int crypt_len = 0; unsigned int authsize = crypto_aead_authsize(tfm); - unsigned char hmac_ctrl = get_hmac(authsize); int err = 0; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -1893,8 +1879,8 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, CHCR_SCMD_CIPHER_MODE_AES_GCM, - CHCR_SCMD_AUTH_MODE_GHASH, hmac_ctrl, - ivsize >> 1); + CHCR_SCMD_AUTH_MODE_GHASH, + aeadctx->hmac_ctrl, ivsize >> 1); } else { chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); From 72a56ca97dc163a7a10a654aab6635e30e781070 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Mon, 10 Apr 2017 18:24:00 +0530 Subject: [PATCH 078/101] crypto: chcr - Fix txq ids. The patch fixes a critical issue to map txqid with flows on the hardware appropriately, if tx queues created are more than flows configured then txqid shall map within the range of hardware flows configured. This ensure that un-mapped txqid does not remain un-handled. The patch also segregated the rxqid and txqid for clarity. Signed-off-by: Atul Gupta Reviewed-by: Ganesh Goudar Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 47 +++++++++++-------- drivers/crypto/chelsio/chcr_core.h | 2 + drivers/crypto/chelsio/chcr_crypto.h | 3 +- .../net/ethernet/chelsio/cxgb4/cxgb4_main.c | 9 ++++ .../net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 1 + drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 3 +- 6 files changed, 44 insertions(+), 21 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 2d610437bf6e..5470e4ec816e 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -522,7 +522,7 @@ static inline void create_wreq(struct chcr_context *ctx, { struct uld_ctx *u_ctx = ULD_CTX(ctx); int iv_loc = IV_DSGL; - int qid = u_ctx->lldi.rxq_ids[ctx->tx_channel_id]; + int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; unsigned int immdatalen = 0, nr_frags = 0; if (is_ofld_imm(skb)) { @@ -543,7 +543,7 @@ static inline void create_wreq(struct chcr_context *ctx, chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid, - is_iv ? iv_loc : IV_NOP, ctx->tx_channel_id); + is_iv ? iv_loc : IV_NOP, ctx->tx_qidx); chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id, qid); @@ -721,19 +721,19 @@ static int chcr_aes_encrypt(struct ablkcipher_request *req) struct sk_buff *skb; if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_channel_id))) { + ctx->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } - skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[ctx->tx_channel_id], + skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], CHCR_ENCRYPT_OP); if (IS_ERR(skb)) { pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); return PTR_ERR(skb); } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); + set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -746,19 +746,19 @@ static int chcr_aes_decrypt(struct ablkcipher_request *req) struct sk_buff *skb; if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_channel_id))) { + ctx->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } - skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[0], + skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], CHCR_DECRYPT_OP); if (IS_ERR(skb)) { pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); return PTR_ERR(skb); } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); + set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -766,7 +766,9 @@ static int chcr_aes_decrypt(struct ablkcipher_request *req) static int chcr_device_init(struct chcr_context *ctx) { struct uld_ctx *u_ctx; + struct adapter *adap; unsigned int id; + int txq_perchan, txq_idx, ntxq; int err = 0, rxq_perchan, rxq_idx; id = smp_processor_id(); @@ -777,11 +779,18 @@ static int chcr_device_init(struct chcr_context *ctx) goto out; } u_ctx = ULD_CTX(ctx); + adap = padap(ctx->dev); + ntxq = min_not_zero((unsigned int)u_ctx->lldi.nrxq, + adap->vres.ncrypto_fc); rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan; + txq_perchan = ntxq / u_ctx->lldi.nchan; rxq_idx = ctx->dev->tx_channel_id * rxq_perchan; rxq_idx += id % rxq_perchan; + txq_idx = ctx->dev->tx_channel_id * txq_perchan; + txq_idx += id % txq_perchan; spin_lock(&ctx->dev->lock_chcr_dev); - ctx->tx_channel_id = rxq_idx; + ctx->rx_qidx = rxq_idx; + ctx->tx_qidx = txq_idx; ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id; ctx->dev->rx_channel_id = 0; spin_unlock(&ctx->dev->lock_chcr_dev); @@ -935,7 +944,7 @@ static int chcr_ahash_update(struct ahash_request *req) u_ctx = ULD_CTX(ctx); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_channel_id))) { + ctx->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -975,7 +984,7 @@ static int chcr_ahash_update(struct ahash_request *req) } req_ctx->reqlen = remainder; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); + set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; @@ -1028,7 +1037,7 @@ static int chcr_ahash_final(struct ahash_request *req) return -ENOMEM; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); + set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -1047,7 +1056,7 @@ static int chcr_ahash_finup(struct ahash_request *req) u_ctx = ULD_CTX(ctx); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_channel_id))) { + ctx->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -1079,7 +1088,7 @@ static int chcr_ahash_finup(struct ahash_request *req) return -ENOMEM; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); + set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; @@ -1100,7 +1109,7 @@ static int chcr_ahash_digest(struct ahash_request *req) u_ctx = ULD_CTX(ctx); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_channel_id))) { + ctx->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -1130,7 +1139,7 @@ static int chcr_ahash_digest(struct ahash_request *req) return -ENOMEM; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); + set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -2451,13 +2460,13 @@ static int chcr_aead_op(struct aead_request *req, } u_ctx = ULD_CTX(ctx); if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_channel_id)) { + ctx->tx_qidx)) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } /* Form a WR from req */ - skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->tx_channel_id], size, + skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], size, op_type); if (IS_ERR(skb) || skb == NULL) { @@ -2466,7 +2475,7 @@ static int chcr_aead_op(struct aead_request *req, } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); + set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h index 79da22b5cdc9..cd0c35a18d92 100644 --- a/drivers/crypto/chelsio/chcr_core.h +++ b/drivers/crypto/chelsio/chcr_core.h @@ -54,6 +54,8 @@ #define CHK_MAC_ERR_BIT(x) (((x) >> MAC_ERROR_BIT) & 1) #define MAX_SALT 4 +#define padap(dev) pci_get_drvdata(dev->u_ctx->lldi.pdev) + struct uld_ctx; struct _key_ctx { diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 4469feae84a2..c5673f097ce9 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -211,7 +211,8 @@ struct __crypto_ctx { struct chcr_context { struct chcr_dev *dev; - unsigned char tx_channel_id; + unsigned char tx_qidx; + unsigned char rx_qidx; struct __crypto_ctx crypto_ctx[0]; }; diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c index afb0967d2ce6..6faaca1b48b3 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c @@ -3809,6 +3809,15 @@ static int adap_init0(struct adapter *adap) } if (caps_cmd.cryptocaps) { /* Should query params here...TODO */ + params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, + params, val); + if (ret < 0) { + if (ret != -EINVAL) + goto bye; + } else { + adap->vres.ncrypto_fc = val[0]; + } adap->params.crypto |= ULP_CRYPTO_LOOKASIDE; adap->num_uld += 1; } diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h index 4c856605fdfa..6e74040af49a 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h @@ -272,6 +272,7 @@ struct cxgb4_virt_res { /* virtualized HW resources */ struct cxgb4_range qp; struct cxgb4_range cq; struct cxgb4_range ocq; + unsigned int ncrypto_fc; }; #define OCQ_WIN_OFFSET(pdev, vres) \ diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h index ccc05f874419..8f8c079d0d2b 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h +++ b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h @@ -1167,7 +1167,8 @@ enum fw_params_param_pfvf { FW_PARAMS_PARAM_PFVF_ACTIVE_FILTER_START = 0x2D, FW_PARAMS_PARAM_PFVF_ACTIVE_FILTER_END = 0x2E, FW_PARAMS_PARAM_PFVF_ETHOFLD_END = 0x30, - FW_PARAMS_PARAM_PFVF_CPLFW4MSG_ENCAP = 0x31 + FW_PARAMS_PARAM_PFVF_CPLFW4MSG_ENCAP = 0x31, + FW_PARAMS_PARAM_PFVF_NCRYPTO_LOOKASIDE = 0x32 }; /* From 0e93708dabc0e999c7ffcf1e8661b68d96de24fd Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Mon, 10 Apr 2017 18:24:01 +0530 Subject: [PATCH 079/101] crypto: chcr - Add fallback for AEAD algos Fallback to sw when I AAD length greater than 511 II Zero length payload II No of sg entries exceeds Request size. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 217 ++++++++++++++++++--------- drivers/crypto/chelsio/chcr_algo.h | 4 + drivers/crypto/chelsio/chcr_crypto.h | 3 +- 3 files changed, 150 insertions(+), 74 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 5470e4ec816e..53d9ce436795 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -1343,7 +1343,36 @@ static int chcr_copy_assoc(struct aead_request *req, return crypto_skcipher_encrypt(skreq); } +static int chcr_aead_need_fallback(struct aead_request *req, int src_nent, + int aadmax, int wrlen, + unsigned short op_type) +{ + unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); + if (((req->cryptlen - (op_type ? authsize : 0)) == 0) || + (req->assoclen > aadmax) || + (src_nent > MAX_SKB_FRAGS) || + (wrlen > MAX_WR_SIZE)) + return 1; + return 0; +} + +static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_context *ctx = crypto_aead_ctx(tfm); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct aead_request *subreq = aead_request_ctx(req); + + aead_request_set_tfm(subreq, aeadctx->sw_cipher); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); + return op_type ? crypto_aead_decrypt(subreq) : + crypto_aead_encrypt(subreq); +} static struct sk_buff *create_authenc_wr(struct aead_request *req, unsigned short qid, @@ -1367,7 +1396,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, unsigned short stop_offset = 0; unsigned int assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); - int err = 0; + int err = -EINVAL, src_nent; int null = 0; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -1377,8 +1406,8 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) goto err; - - if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0) + src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); + if (src_nent < 0) goto err; src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); reqctx->dst = src; @@ -1396,7 +1425,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, } reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + (op_type ? -authsize : authsize)); - if (reqctx->dst_nents <= 0) { + if (reqctx->dst_nents < 0) { pr_err("AUTHENC:Invalid Destination sg entries\n"); goto err; } @@ -1404,6 +1433,12 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) - sizeof(chcr_req->key_ctx); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); + if (chcr_aead_need_fallback(req, src_nent + MIN_AUTH_SG, + T6_MAX_AAD_SIZE, + transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8), + op_type)) { + return ERR_PTR(chcr_aead_fallback(req, op_type)); + } skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); if (!skb) goto err; @@ -1485,24 +1520,6 @@ err: return ERR_PTR(-EINVAL); } -static void aes_gcm_empty_pld_pad(struct scatterlist *sg, - unsigned short offset) -{ - struct page *spage; - unsigned char *addr; - - spage = sg_page(sg); - get_page(spage); /* so that it is not freed by NIC */ -#ifdef KMAP_ATOMIC_ARGS - addr = kmap_atomic(spage, KM_SOFTIRQ0); -#else - addr = kmap_atomic(spage); -#endif - memset(addr + sg->offset, 0, offset + 1); - - kunmap_atomic(addr); -} - static int set_msg_len(u8 *block, unsigned int msglen, int csize) { __be32 data; @@ -1566,11 +1583,6 @@ static int ccm_format_packet(struct aead_request *req, struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); int rc = 0; - if (req->assoclen > T5_MAX_AAD_SIZE) { - pr_err("CCM: Unsupported AAD data. It should be < %d\n", - T5_MAX_AAD_SIZE); - return -EINVAL; - } if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { reqctx->iv[0] = 3; memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3); @@ -1715,16 +1727,17 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, unsigned int dst_size = 0, kctx_len; unsigned int sub_type; unsigned int authsize = crypto_aead_authsize(tfm); - int err = 0; + int err = -EINVAL, src_nent; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) goto err; - - if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0) + src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); + if (src_nent < 0) goto err; + sub_type = get_aead_subtype(tfm); src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); reqctx->dst = src; @@ -1740,7 +1753,7 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, } reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + (op_type ? -authsize : authsize)); - if (reqctx->dst_nents <= 0) { + if (reqctx->dst_nents < 0) { pr_err("CCM:Invalid Destination sg entries\n"); goto err; } @@ -1752,6 +1765,13 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); + if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG, + T6_MAX_AAD_SIZE - 18, + transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8), + op_type)) { + return ERR_PTR(chcr_aead_fallback(req, op_type)); + } + skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); if (!skb) @@ -1816,7 +1836,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, unsigned char tag_offset = 0; unsigned int crypt_len = 0; unsigned int authsize = crypto_aead_authsize(tfm); - int err = 0; + int err = -EINVAL, src_nent; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -1826,8 +1846,8 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) goto err; - - if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0) + src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); + if (src_nent < 0) goto err; src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); @@ -1849,7 +1869,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, crypt_len = req->cryptlen; reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + (op_type ? -authsize : authsize)); - if (reqctx->dst_nents <= 0) { + if (reqctx->dst_nents < 0) { pr_err("GCM:Invalid Destination sg entries\n"); goto err; } @@ -1859,6 +1879,12 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) + AEAD_H_SIZE; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); + if (chcr_aead_need_fallback(req, src_nent + MIN_GCM_SG, + T6_MAX_AAD_SIZE, + transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8), + op_type)) { + return ERR_PTR(chcr_aead_fallback(req, op_type)); + } skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); if (!skb) goto err; @@ -1876,11 +1902,11 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( ctx->dev->rx_channel_id, 2, (ivsize ? (req->assoclen + 1) : 0)); - chcr_req->sec_cpl.pldlen = htonl(req->assoclen + ivsize + crypt_len); + chcr_req->sec_cpl.pldlen = + htonl(req->assoclen + ivsize + req->cryptlen); chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( req->assoclen ? 1 : 0, req->assoclen, req->assoclen + ivsize + 1, 0); - if (req->cryptlen) { chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + ivsize + 1, tag_offset, tag_offset); @@ -1890,15 +1916,6 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, CHCR_SCMD_CIPHER_MODE_AES_GCM, CHCR_SCMD_AUTH_MODE_GHASH, aeadctx->hmac_ctrl, ivsize >> 1); - } else { - chcr_req->sec_cpl.cipherstop_lo_authinsert = - FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); - chcr_req->sec_cpl.seqno_numivs = - FILL_SEC_CPL_SCMD0_SEQNO(op_type, - (op_type == CHCR_ENCRYPT_OP) ? - 1 : 0, CHCR_SCMD_CIPHER_MODE_AES_CBC, - 0, 0, ivsize >> 1); - } chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, 1, dst_size); chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; @@ -1931,15 +1948,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, write_sg_to_skb(skb, &frags, req->src, req->assoclen); write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); - - if (req->cryptlen) { - write_sg_to_skb(skb, &frags, src, req->cryptlen); - } else { - aes_gcm_empty_pld_pad(req->dst, authsize - 1); - write_sg_to_skb(skb, &frags, reqctx->dst, crypt_len); - - } - + write_sg_to_skb(skb, &frags, src, req->cryptlen); create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1, sizeof(struct cpl_rx_phys_dsgl) + dst_size); reqctx->skb = skb; @@ -1960,8 +1969,15 @@ static int chcr_aead_cra_init(struct crypto_aead *tfm) { struct chcr_context *ctx = crypto_aead_ctx(tfm); struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct aead_alg *alg = crypto_aead_alg(tfm); - crypto_aead_set_reqsize(tfm, sizeof(struct chcr_aead_reqctx)); + aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(aeadctx->sw_cipher)) + return PTR_ERR(aeadctx->sw_cipher); + crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx), + sizeof(struct aead_request) + + crypto_aead_reqsize(aeadctx->sw_cipher))); aeadctx->null = crypto_get_default_null_skcipher(); if (IS_ERR(aeadctx->null)) return PTR_ERR(aeadctx->null); @@ -1970,7 +1986,11 @@ static int chcr_aead_cra_init(struct crypto_aead *tfm) static void chcr_aead_cra_exit(struct crypto_aead *tfm) { + struct chcr_context *ctx = crypto_aead_ctx(tfm); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + crypto_put_default_null_skcipher(); + crypto_free_aead(aeadctx->sw_cipher); } static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, @@ -1980,7 +2000,7 @@ static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; aeadctx->mayverify = VERIFY_HW; - return 0; + return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); } static int chcr_authenc_setauthsize(struct crypto_aead *tfm, unsigned int authsize) @@ -2017,7 +2037,7 @@ static int chcr_authenc_setauthsize(struct crypto_aead *tfm, aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; aeadctx->mayverify = VERIFY_SW; } - return 0; + return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); } @@ -2057,7 +2077,7 @@ static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } - return 0; + return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); } static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, @@ -2083,7 +2103,7 @@ static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } - return 0; + return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); } static int chcr_ccm_setauthsize(struct crypto_aead *tfm, @@ -2125,10 +2145,10 @@ static int chcr_ccm_setauthsize(struct crypto_aead *tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } - return 0; + return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); } -static int chcr_aead_ccm_setkey(struct crypto_aead *aead, +static int chcr_ccm_common_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { @@ -2137,8 +2157,6 @@ static int chcr_aead_ccm_setkey(struct crypto_aead *aead, unsigned char ck_size, mk_size; int key_ctx_size = 0; - memcpy(aeadctx->key, key, keylen); - aeadctx->enckey_len = keylen; key_ctx_size = sizeof(struct _key_ctx) + ((DIV_ROUND_UP(keylen, 16)) << 4) * 2; if (keylen == AES_KEYSIZE_128) { @@ -2158,9 +2176,32 @@ static int chcr_aead_ccm_setkey(struct crypto_aead *aead, } aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0, key_ctx_size >> 4); + memcpy(aeadctx->key, key, keylen); + aeadctx->enckey_len = keylen; + return 0; } +static int chcr_aead_ccm_setkey(struct crypto_aead *aead, + const u8 *key, + unsigned int keylen) +{ + struct chcr_context *ctx = crypto_aead_ctx(aead); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + int error; + + crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & + CRYPTO_TFM_REQ_MASK); + error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); + crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); + crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & + CRYPTO_TFM_RES_MASK); + if (error) + return error; + return chcr_ccm_common_setkey(aead, key, keylen); +} + static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { @@ -2175,7 +2216,7 @@ static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, } keylen -= 3; memcpy(aeadctx->salt, key + keylen, 3); - return chcr_aead_ccm_setkey(aead, key, keylen); + return chcr_ccm_common_setkey(aead, key, keylen); } static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, @@ -2188,6 +2229,17 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int ck_size; int ret = 0, key_ctx_size = 0; + aeadctx->enckey_len = 0; + crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) + & CRYPTO_TFM_REQ_MASK); + ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); + crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); + crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & + CRYPTO_TFM_RES_MASK); + if (ret) + goto out; + if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 && keylen > 3) { keylen -= 4; /* nonce/salt is present in the last 4 bytes */ @@ -2202,8 +2254,7 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, } else { crypto_tfm_set_flags((struct crypto_tfm *)aead, CRYPTO_TFM_RES_BAD_KEY_LEN); - aeadctx->enckey_len = 0; - pr_err("GCM: Invalid key length %d", keylen); + pr_err("GCM: Invalid key length %d\n", keylen); ret = -EINVAL; goto out; } @@ -2259,6 +2310,16 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, int align; u8 *o_ptr = NULL; + crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) + & CRYPTO_TFM_REQ_MASK); + err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); + crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK); + crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher) + & CRYPTO_TFM_RES_MASK); + if (err) + goto out; + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) { crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); goto out; @@ -2291,7 +2352,8 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, base_hash = chcr_alloc_shash(max_authsize); if (IS_ERR(base_hash)) { pr_err("chcr : Base driver cannot be loaded\n"); - goto out; + aeadctx->enckey_len = 0; + return -EINVAL; } { SHASH_DESC_ON_STACK(shash, base_hash); @@ -2358,11 +2420,21 @@ static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); struct crypto_authenc_keys keys; - + int err; /* it contains auth and cipher key both*/ int key_ctx_len = 0; unsigned char ck_size = 0; + crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) + & CRYPTO_TFM_REQ_MASK); + err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); + crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK); + crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher) + & CRYPTO_TFM_RES_MASK); + if (err) + goto out; + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) { crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); goto out; @@ -2469,10 +2541,8 @@ static int chcr_aead_op(struct aead_request *req, skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], size, op_type); - if (IS_ERR(skb) || skb == NULL) { - pr_err("chcr : %s : failed to form WR. No memory\n", __func__); + if (IS_ERR(skb) || !skb) return PTR_ERR(skb); - } skb->dev = u_ctx->lldi.ports[0]; set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); @@ -2921,7 +2991,8 @@ static int chcr_register_alg(void) break; case CRYPTO_ALG_TYPE_AEAD: driver_algs[i].alg.aead.base.cra_flags = - CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC; + CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK; driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt; driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt; driver_algs[i].alg.aead.init = chcr_aead_cra_init; diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index ba38bae7ce80..751d06a58101 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -218,6 +218,10 @@ #define MAX_NK 8 #define CRYPTO_MAX_IMM_TX_PKT_LEN 256 +#define MAX_WR_SIZE 512 +#define MIN_AUTH_SG 2 /*IV + AAD*/ +#define MIN_GCM_SG 2 /* IV + AAD*/ +#define MIN_CCM_SG 3 /*IV+AAD+B0*/ struct algo_param { unsigned int auth_mode; diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index c5673f097ce9..5b2fabb14229 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -41,7 +41,7 @@ #define CCM_B0_SIZE 16 #define CCM_AAD_FIELD_SIZE 2 -#define T5_MAX_AAD_SIZE 512 +#define T6_MAX_AAD_SIZE 511 /* Define following if h/w is not dropping the AAD and IV data before @@ -188,6 +188,7 @@ struct chcr_aead_ctx { __be32 key_ctx_hdr; unsigned int enckey_len; struct crypto_skcipher *null; + struct crypto_aead *sw_cipher; u8 salt[MAX_SALT]; u8 key[CHCR_AES_MAX_KEY_LEN]; u16 hmac_ctrl; From 7e207d8550644c1076ceb070f40abf52701253ec Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Tue, 11 Apr 2017 16:04:09 +0000 Subject: [PATCH 080/101] crypto: caam - fix error return code in caam_qi_init() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix to return error code -ENOMEM from the kmem_cache_create() error handling case instead of 0(err is 0 here), as done elsewhere in this function. Fixes: 67c2315def06 ("crypto: caam - add Queue Interface (QI) backend support") Signed-off-by: Wei Yongjun Acked-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/qi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index 45de8fd87774..1990ed460c46 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -789,7 +789,7 @@ int caam_qi_init(struct platform_device *caam_pdev) dev_err(qidev, "Can't allocate CAAM cache\n"); free_rsp_fqs(); platform_device_unregister(qi_pdev); - return err; + return -ENOMEM; } /* Done with the CGRs; restore the cpus allowed mask */ From ed067d4a859ff696373324c5061392e013a7561a Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 11 Apr 2017 20:08:34 +0200 Subject: [PATCH 081/101] linux/kernel.h: Add ALIGN_DOWN macro Few parts of kernel define their own macro for aligning down so provide a common define for this, with the same usage and assumptions as existing ALIGN. Convert also three existing implementations to this one. Signed-off-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- arch/metag/kernel/stacktrace.c | 2 -- drivers/gpu/drm/udl/udl_fb.c | 2 +- include/linux/kernel.h | 1 + include/video/udlfb.h | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/metag/kernel/stacktrace.c b/arch/metag/kernel/stacktrace.c index 91ffc4b75c33..09d67b7f51ca 100644 --- a/arch/metag/kernel/stacktrace.c +++ b/arch/metag/kernel/stacktrace.c @@ -31,8 +31,6 @@ static void tbi_boing_init(void) } #endif -#define ALIGN_DOWN(addr, size) ((addr)&(~((size)-1))) - /* * Unwind the current stack frame and store the new register values in the * structure passed as argument. Unwinding is equivalent to a function return, diff --git a/drivers/gpu/drm/udl/udl_fb.c b/drivers/gpu/drm/udl/udl_fb.c index 8e8d60e9a1a2..b1f0d523dff9 100644 --- a/drivers/gpu/drm/udl/udl_fb.c +++ b/drivers/gpu/drm/udl/udl_fb.c @@ -37,7 +37,7 @@ struct udl_fbdev { }; #define DL_ALIGN_UP(x, a) ALIGN(x, a) -#define DL_ALIGN_DOWN(x, a) ALIGN(x-(a-1), a) +#define DL_ALIGN_DOWN(x, a) ALIGN_DOWN(x, a) /** Read the red component (0..255) of a 32 bpp colour. */ #define DLO_RGB_GETRED(col) (uint8_t)((col) & 0xFF) diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 4c26dc3a8295..3d9f8420f973 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -47,6 +47,7 @@ /* @a is a power of 2 value */ #define ALIGN(x, a) __ALIGN_KERNEL((x), (a)) +#define ALIGN_DOWN(x, a) __ALIGN_KERNEL((x) - ((a) - 1), (a)) #define __ALIGN_MASK(x, mask) __ALIGN_KERNEL_MASK((x), (mask)) #define PTR_ALIGN(p, a) ((typeof(p))ALIGN((unsigned long)(p), (a))) #define IS_ALIGNED(x, a) (((x) & ((typeof(x))(a) - 1)) == 0) diff --git a/include/video/udlfb.h b/include/video/udlfb.h index f9466fa54ba4..3ea90aea5617 100644 --- a/include/video/udlfb.h +++ b/include/video/udlfb.h @@ -92,6 +92,6 @@ struct dlfb_data { /* remove these once align.h patch is taken into kernel */ #define DL_ALIGN_UP(x, a) ALIGN(x, a) -#define DL_ALIGN_DOWN(x, a) ALIGN(x-(a-1), a) +#define DL_ALIGN_DOWN(x, a) ALIGN_DOWN(x, a) #endif From c46ea13f55b629a26d5dd4a22688a5f88cff0906 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 11 Apr 2017 20:08:35 +0200 Subject: [PATCH 082/101] crypto: exynos - Add new Exynos RNG driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace existing hw_ranndom/exynos-rng driver with a new, reworked one. This is a driver for pseudo random number generator block which on Exynos4 chipsets must be seeded with some value. On newer Exynos5420 chipsets it might seed itself from true random number generator block but this is not implemented yet. New driver is a complete rework to use the crypto ALGAPI instead of hw_random API. Rationale for the change: 1. hw_random interface is for true RNG devices. 2. The old driver was seeding itself with jiffies which is not a reliable source for randomness. 3. Device generates five random 32-bit numbers in each pass but old driver was returning only one 32-bit number thus its performance was reduced. Compatibility with DeviceTree bindings is preserved. New driver does not use runtime power management but manually enables and disables the clock when needed. This is preferred approach because using runtime PM just to toggle clock is huge overhead. Another difference is reseeding itself with generated random data periodically and during resuming from system suspend (previously driver was re-seeding itself again with jiffies). Signed-off-by: Krzysztof Kozlowski Reviewed-by: Stephan Müller Reviewed-by: PrasannaKumar Muralidharan Reviewed-by: Bartlomiej Zolnierkiewicz Signed-off-by: Herbert Xu --- MAINTAINERS | 8 + drivers/char/hw_random/Kconfig | 14 - drivers/char/hw_random/Makefile | 1 - drivers/char/hw_random/exynos-rng.c | 231 ----------------- drivers/crypto/Kconfig | 15 ++ drivers/crypto/Makefile | 1 + drivers/crypto/exynos-rng.c | 389 ++++++++++++++++++++++++++++ 7 files changed, 413 insertions(+), 246 deletions(-) delete mode 100644 drivers/char/hw_random/exynos-rng.c create mode 100644 drivers/crypto/exynos-rng.c diff --git a/MAINTAINERS b/MAINTAINERS index 620e63090275..8b938779ad39 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10906,6 +10906,14 @@ L: alsa-devel@alsa-project.org (moderated for non-subscribers) S: Supported F: sound/soc/samsung/ +SAMSUNG EXYNOS PSEUDO RANDOM NUMBER GENERATOR (RNG) DRIVER +M: Krzysztof Kozlowski +L: linux-crypto@vger.kernel.org +L: linux-samsung-soc@vger.kernel.org +S: Maintained +F: drivers/crypto/exynos-rng.c +F: Documentation/devicetree/bindings/rng/samsung,exynos-rng4.txt + SAMSUNG FRAMEBUFFER DRIVER M: Jingoo Han L: linux-fbdev@vger.kernel.org diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index 0cafe08919c9..bdae802e7154 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -294,20 +294,6 @@ config HW_RANDOM_POWERNV If unsure, say Y. -config HW_RANDOM_EXYNOS - tristate "EXYNOS HW random number generator support" - depends on ARCH_EXYNOS || COMPILE_TEST - depends on HAS_IOMEM - default HW_RANDOM - ---help--- - This driver provides kernel-side support for the Random Number - Generator hardware found on EXYNOS SOCs. - - To compile this driver as a module, choose M here: the - module will be called exynos-rng. - - If unsure, say Y. - config HW_RANDOM_TPM tristate "TPM HW Random Number Generator support" depends on TCG_TPM diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index 5f52b1e4e7be..6f1eecc2045c 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -24,7 +24,6 @@ obj-$(CONFIG_HW_RANDOM_OCTEON) += octeon-rng.o obj-$(CONFIG_HW_RANDOM_NOMADIK) += nomadik-rng.o obj-$(CONFIG_HW_RANDOM_PSERIES) += pseries-rng.o obj-$(CONFIG_HW_RANDOM_POWERNV) += powernv-rng.o -obj-$(CONFIG_HW_RANDOM_EXYNOS) += exynos-rng.o obj-$(CONFIG_HW_RANDOM_HISI) += hisi-rng.o obj-$(CONFIG_HW_RANDOM_TPM) += tpm-rng.o obj-$(CONFIG_HW_RANDOM_BCM2835) += bcm2835-rng.o diff --git a/drivers/char/hw_random/exynos-rng.c b/drivers/char/hw_random/exynos-rng.c deleted file mode 100644 index 23d358553b21..000000000000 --- a/drivers/char/hw_random/exynos-rng.c +++ /dev/null @@ -1,231 +0,0 @@ -/* - * exynos-rng.c - Random Number Generator driver for the exynos - * - * Copyright (C) 2012 Samsung Electronics - * Jonghwa Lee - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#define EXYNOS_PRNG_STATUS_OFFSET 0x10 -#define EXYNOS_PRNG_SEED_OFFSET 0x140 -#define EXYNOS_PRNG_OUT1_OFFSET 0x160 -#define SEED_SETTING_DONE BIT(1) -#define PRNG_START 0x18 -#define PRNG_DONE BIT(5) -#define EXYNOS_AUTOSUSPEND_DELAY 100 - -struct exynos_rng { - struct device *dev; - struct hwrng rng; - void __iomem *mem; - struct clk *clk; -}; - -static u32 exynos_rng_readl(struct exynos_rng *rng, u32 offset) -{ - return readl_relaxed(rng->mem + offset); -} - -static void exynos_rng_writel(struct exynos_rng *rng, u32 val, u32 offset) -{ - writel_relaxed(val, rng->mem + offset); -} - -static int exynos_rng_configure(struct exynos_rng *exynos_rng) -{ - int i; - int ret = 0; - - for (i = 0 ; i < 5 ; i++) - exynos_rng_writel(exynos_rng, jiffies, - EXYNOS_PRNG_SEED_OFFSET + 4*i); - - if (!(exynos_rng_readl(exynos_rng, EXYNOS_PRNG_STATUS_OFFSET) - & SEED_SETTING_DONE)) - ret = -EIO; - - return ret; -} - -static int exynos_init(struct hwrng *rng) -{ - struct exynos_rng *exynos_rng = container_of(rng, - struct exynos_rng, rng); - int ret = 0; - - pm_runtime_get_sync(exynos_rng->dev); - ret = exynos_rng_configure(exynos_rng); - pm_runtime_mark_last_busy(exynos_rng->dev); - pm_runtime_put_autosuspend(exynos_rng->dev); - - return ret; -} - -static int exynos_read(struct hwrng *rng, void *buf, - size_t max, bool wait) -{ - struct exynos_rng *exynos_rng = container_of(rng, - struct exynos_rng, rng); - u32 *data = buf; - int retry = 100; - int ret = 4; - - pm_runtime_get_sync(exynos_rng->dev); - - exynos_rng_writel(exynos_rng, PRNG_START, 0); - - while (!(exynos_rng_readl(exynos_rng, - EXYNOS_PRNG_STATUS_OFFSET) & PRNG_DONE) && --retry) - cpu_relax(); - if (!retry) { - ret = -ETIMEDOUT; - goto out; - } - - exynos_rng_writel(exynos_rng, PRNG_DONE, EXYNOS_PRNG_STATUS_OFFSET); - - *data = exynos_rng_readl(exynos_rng, EXYNOS_PRNG_OUT1_OFFSET); - -out: - pm_runtime_mark_last_busy(exynos_rng->dev); - pm_runtime_put_sync_autosuspend(exynos_rng->dev); - - return ret; -} - -static int exynos_rng_probe(struct platform_device *pdev) -{ - struct exynos_rng *exynos_rng; - struct resource *res; - int ret; - - exynos_rng = devm_kzalloc(&pdev->dev, sizeof(struct exynos_rng), - GFP_KERNEL); - if (!exynos_rng) - return -ENOMEM; - - exynos_rng->dev = &pdev->dev; - exynos_rng->rng.name = "exynos"; - exynos_rng->rng.init = exynos_init; - exynos_rng->rng.read = exynos_read; - exynos_rng->clk = devm_clk_get(&pdev->dev, "secss"); - if (IS_ERR(exynos_rng->clk)) { - dev_err(&pdev->dev, "Couldn't get clock.\n"); - return -ENOENT; - } - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - exynos_rng->mem = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(exynos_rng->mem)) - return PTR_ERR(exynos_rng->mem); - - platform_set_drvdata(pdev, exynos_rng); - - pm_runtime_set_autosuspend_delay(&pdev->dev, EXYNOS_AUTOSUSPEND_DELAY); - pm_runtime_use_autosuspend(&pdev->dev); - pm_runtime_enable(&pdev->dev); - - ret = devm_hwrng_register(&pdev->dev, &exynos_rng->rng); - if (ret) { - pm_runtime_dont_use_autosuspend(&pdev->dev); - pm_runtime_disable(&pdev->dev); - } - - return ret; -} - -static int exynos_rng_remove(struct platform_device *pdev) -{ - pm_runtime_dont_use_autosuspend(&pdev->dev); - pm_runtime_disable(&pdev->dev); - - return 0; -} - -static int __maybe_unused exynos_rng_runtime_suspend(struct device *dev) -{ - struct platform_device *pdev = to_platform_device(dev); - struct exynos_rng *exynos_rng = platform_get_drvdata(pdev); - - clk_disable_unprepare(exynos_rng->clk); - - return 0; -} - -static int __maybe_unused exynos_rng_runtime_resume(struct device *dev) -{ - struct platform_device *pdev = to_platform_device(dev); - struct exynos_rng *exynos_rng = platform_get_drvdata(pdev); - - return clk_prepare_enable(exynos_rng->clk); -} - -static int __maybe_unused exynos_rng_suspend(struct device *dev) -{ - return pm_runtime_force_suspend(dev); -} - -static int __maybe_unused exynos_rng_resume(struct device *dev) -{ - struct platform_device *pdev = to_platform_device(dev); - struct exynos_rng *exynos_rng = platform_get_drvdata(pdev); - int ret; - - ret = pm_runtime_force_resume(dev); - if (ret) - return ret; - - return exynos_rng_configure(exynos_rng); -} - -static const struct dev_pm_ops exynos_rng_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(exynos_rng_suspend, exynos_rng_resume) - SET_RUNTIME_PM_OPS(exynos_rng_runtime_suspend, - exynos_rng_runtime_resume, NULL) -}; - -static const struct of_device_id exynos_rng_dt_match[] = { - { - .compatible = "samsung,exynos4-rng", - }, - { }, -}; -MODULE_DEVICE_TABLE(of, exynos_rng_dt_match); - -static struct platform_driver exynos_rng_driver = { - .driver = { - .name = "exynos-rng", - .pm = &exynos_rng_pm_ops, - .of_match_table = exynos_rng_dt_match, - }, - .probe = exynos_rng_probe, - .remove = exynos_rng_remove, -}; - -module_platform_driver(exynos_rng_driver); - -MODULE_DESCRIPTION("EXYNOS 4 H/W Random Number Generator driver"); -MODULE_AUTHOR("Jonghwa Lee "); -MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 3cd234052775..fb1e60f5002e 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -388,6 +388,21 @@ config CRYPTO_DEV_MXC_SCC This option enables support for the Security Controller (SCC) found in Freescale i.MX25 chips. +config CRYPTO_DEV_EXYNOS_RNG + tristate "EXYNOS HW pseudo random number generator support" + depends on ARCH_EXYNOS || COMPILE_TEST + depends on HAS_IOMEM + select CRYPTO_RNG + ---help--- + This driver provides kernel-side support through the + cryptographic API for the pseudo random number generator hardware + found on Exynos SoCs. + + To compile this driver as a module, choose M here: the + module will be called exynos-rng. + + If unsure, say Y. + config CRYPTO_DEV_S5P tristate "Support for Samsung S5PV210/Exynos crypto accelerator" depends on ARCH_S5PV210 || ARCH_EXYNOS || COMPILE_TEST diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index ea26c8dd5e14..463f33592d93 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -6,6 +6,7 @@ obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/ obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/ obj-$(CONFIG_CRYPTO_DEV_CPT) += cavium/cpt/ +obj-$(CONFIG_CRYPTO_DEV_EXYNOS_RNG) += exynos-rng.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/ obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o diff --git a/drivers/crypto/exynos-rng.c b/drivers/crypto/exynos-rng.c new file mode 100644 index 000000000000..451620b475a0 --- /dev/null +++ b/drivers/crypto/exynos-rng.c @@ -0,0 +1,389 @@ +/* + * exynos-rng.c - Random Number Generator driver for the Exynos + * + * Copyright (c) 2017 Krzysztof Kozlowski + * + * Loosely based on old driver from drivers/char/hw_random/exynos-rng.c: + * Copyright (C) 2012 Samsung Electronics + * Jonghwa Lee + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include + +#include + +#define EXYNOS_RNG_CONTROL 0x0 +#define EXYNOS_RNG_STATUS 0x10 +#define EXYNOS_RNG_SEED_BASE 0x140 +#define EXYNOS_RNG_SEED(n) (EXYNOS_RNG_SEED_BASE + (n * 0x4)) +#define EXYNOS_RNG_OUT_BASE 0x160 +#define EXYNOS_RNG_OUT(n) (EXYNOS_RNG_OUT_BASE + (n * 0x4)) + +/* EXYNOS_RNG_CONTROL bit fields */ +#define EXYNOS_RNG_CONTROL_START 0x18 +/* EXYNOS_RNG_STATUS bit fields */ +#define EXYNOS_RNG_STATUS_SEED_SETTING_DONE BIT(1) +#define EXYNOS_RNG_STATUS_RNG_DONE BIT(5) + +/* Five seed and output registers, each 4 bytes */ +#define EXYNOS_RNG_SEED_REGS 5 +#define EXYNOS_RNG_SEED_SIZE (EXYNOS_RNG_SEED_REGS * 4) + +/* + * Driver re-seeds itself with generated random numbers to increase + * the randomness. + * + * Time for next re-seed in ms. + */ +#define EXYNOS_RNG_RESEED_TIME 100 +/* + * In polling mode, do not wait infinitely for the engine to finish the work. + */ +#define EXYNOS_RNG_WAIT_RETRIES 100 + +/* Context for crypto */ +struct exynos_rng_ctx { + struct exynos_rng_dev *rng; +}; + +/* Device associated memory */ +struct exynos_rng_dev { + struct device *dev; + void __iomem *mem; + struct clk *clk; + /* Generated numbers stored for seeding during resume */ + u8 seed_save[EXYNOS_RNG_SEED_SIZE]; + unsigned int seed_save_len; + /* Time of last seeding in jiffies */ + unsigned long last_seeding; +}; + +static struct exynos_rng_dev *exynos_rng_dev; + +static u32 exynos_rng_readl(struct exynos_rng_dev *rng, u32 offset) +{ + return readl_relaxed(rng->mem + offset); +} + +static void exynos_rng_writel(struct exynos_rng_dev *rng, u32 val, u32 offset) +{ + writel_relaxed(val, rng->mem + offset); +} + +static int exynos_rng_set_seed(struct exynos_rng_dev *rng, + const u8 *seed, unsigned int slen) +{ + u32 val; + int i; + + /* Round seed length because loop iterates over full register size */ + slen = ALIGN_DOWN(slen, 4); + + if (slen < EXYNOS_RNG_SEED_SIZE) + return -EINVAL; + + for (i = 0; i < slen ; i += 4) { + unsigned int seed_reg = (i / 4) % EXYNOS_RNG_SEED_REGS; + + val = seed[i] << 24; + val |= seed[i + 1] << 16; + val |= seed[i + 2] << 8; + val |= seed[i + 3] << 0; + + exynos_rng_writel(rng, val, EXYNOS_RNG_SEED(seed_reg)); + } + + val = exynos_rng_readl(rng, EXYNOS_RNG_STATUS); + if (!(val & EXYNOS_RNG_STATUS_SEED_SETTING_DONE)) { + dev_warn(rng->dev, "Seed setting not finished\n"); + return -EIO; + } + + rng->last_seeding = jiffies; + + return 0; +} + +/* + * Read from output registers and put the data under 'dst' array, + * up to dlen bytes. + * + * Returns number of bytes actually stored in 'dst' (dlen + * or EXYNOS_RNG_SEED_SIZE). + */ +static unsigned int exynos_rng_copy_random(struct exynos_rng_dev *rng, + u8 *dst, unsigned int dlen) +{ + unsigned int cnt = 0; + int i, j; + u32 val; + + for (j = 0; j < EXYNOS_RNG_SEED_REGS; j++) { + val = exynos_rng_readl(rng, EXYNOS_RNG_OUT(j)); + + for (i = 0; i < 4; i++) { + dst[cnt] = val & 0xff; + val >>= 8; + if (++cnt >= dlen) + return cnt; + } + } + + return cnt; +} + +/* + * Start the engine and poll for finish. Then read from output registers + * filling the 'dst' buffer up to 'dlen' bytes or up to size of generated + * random data (EXYNOS_RNG_SEED_SIZE). + * + * On success: return 0 and store number of read bytes under 'read' address. + * On error: return -ERRNO. + */ +static int exynos_rng_get_random(struct exynos_rng_dev *rng, + u8 *dst, unsigned int dlen, + unsigned int *read) +{ + int retry = EXYNOS_RNG_WAIT_RETRIES; + + exynos_rng_writel(rng, EXYNOS_RNG_CONTROL_START, + EXYNOS_RNG_CONTROL); + + while (!(exynos_rng_readl(rng, + EXYNOS_RNG_STATUS) & EXYNOS_RNG_STATUS_RNG_DONE) && --retry) + cpu_relax(); + + if (!retry) + return -ETIMEDOUT; + + /* Clear status bit */ + exynos_rng_writel(rng, EXYNOS_RNG_STATUS_RNG_DONE, + EXYNOS_RNG_STATUS); + *read = exynos_rng_copy_random(rng, dst, dlen); + + return 0; +} + +/* Re-seed itself from time to time */ +static void exynos_rng_reseed(struct exynos_rng_dev *rng) +{ + unsigned long next_seeding = rng->last_seeding + \ + msecs_to_jiffies(EXYNOS_RNG_RESEED_TIME); + unsigned long now = jiffies; + unsigned int read = 0; + u8 seed[EXYNOS_RNG_SEED_SIZE]; + + if (time_before(now, next_seeding)) + return; + + if (exynos_rng_get_random(rng, seed, sizeof(seed), &read)) + return; + + exynos_rng_set_seed(rng, seed, read); +} + +static int exynos_rng_generate(struct crypto_rng *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int dlen) +{ + struct exynos_rng_ctx *ctx = crypto_rng_ctx(tfm); + struct exynos_rng_dev *rng = ctx->rng; + unsigned int read = 0; + int ret; + + ret = clk_prepare_enable(rng->clk); + if (ret) + return ret; + + do { + ret = exynos_rng_get_random(rng, dst, dlen, &read); + if (ret) + break; + + dlen -= read; + dst += read; + + exynos_rng_reseed(rng); + } while (dlen > 0); + + clk_disable_unprepare(rng->clk); + + return ret; +} + +static int exynos_rng_seed(struct crypto_rng *tfm, const u8 *seed, + unsigned int slen) +{ + struct exynos_rng_ctx *ctx = crypto_rng_ctx(tfm); + struct exynos_rng_dev *rng = ctx->rng; + int ret; + + ret = clk_prepare_enable(rng->clk); + if (ret) + return ret; + + ret = exynos_rng_set_seed(ctx->rng, seed, slen); + + clk_disable_unprepare(rng->clk); + + return ret; +} + +static int exynos_rng_kcapi_init(struct crypto_tfm *tfm) +{ + struct exynos_rng_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->rng = exynos_rng_dev; + + return 0; +} + +static struct rng_alg exynos_rng_alg = { + .generate = exynos_rng_generate, + .seed = exynos_rng_seed, + .seedsize = EXYNOS_RNG_SEED_SIZE, + .base = { + .cra_name = "stdrng", + .cra_driver_name = "exynos_rng", + .cra_priority = 100, + .cra_ctxsize = sizeof(struct exynos_rng_ctx), + .cra_module = THIS_MODULE, + .cra_init = exynos_rng_kcapi_init, + } +}; + +static int exynos_rng_probe(struct platform_device *pdev) +{ + struct exynos_rng_dev *rng; + struct resource *res; + int ret; + + if (exynos_rng_dev) + return -EEXIST; + + rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL); + if (!rng) + return -ENOMEM; + + rng->dev = &pdev->dev; + rng->clk = devm_clk_get(&pdev->dev, "secss"); + if (IS_ERR(rng->clk)) { + dev_err(&pdev->dev, "Couldn't get clock.\n"); + return PTR_ERR(rng->clk); + } + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + rng->mem = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(rng->mem)) + return PTR_ERR(rng->mem); + + platform_set_drvdata(pdev, rng); + + exynos_rng_dev = rng; + + ret = crypto_register_rng(&exynos_rng_alg); + if (ret) { + dev_err(&pdev->dev, + "Couldn't register rng crypto alg: %d\n", ret); + exynos_rng_dev = NULL; + } + + return ret; +} + +static int exynos_rng_remove(struct platform_device *pdev) +{ + crypto_unregister_rng(&exynos_rng_alg); + + exynos_rng_dev = NULL; + + return 0; +} + +static int __maybe_unused exynos_rng_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct exynos_rng_dev *rng = platform_get_drvdata(pdev); + int ret; + + /* If we were never seeded then after resume it will be the same */ + if (!rng->last_seeding) + return 0; + + rng->seed_save_len = 0; + ret = clk_prepare_enable(rng->clk); + if (ret) + return ret; + + /* Get new random numbers and store them for seeding on resume. */ + exynos_rng_get_random(rng, rng->seed_save, sizeof(rng->seed_save), + &(rng->seed_save_len)); + dev_dbg(rng->dev, "Stored %u bytes for seeding on system resume\n", + rng->seed_save_len); + + clk_disable_unprepare(rng->clk); + + return 0; +} + +static int __maybe_unused exynos_rng_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct exynos_rng_dev *rng = platform_get_drvdata(pdev); + int ret; + + /* Never seeded so nothing to do */ + if (!rng->last_seeding) + return 0; + + ret = clk_prepare_enable(rng->clk); + if (ret) + return ret; + + ret = exynos_rng_set_seed(rng, rng->seed_save, rng->seed_save_len); + + clk_disable_unprepare(rng->clk); + + return ret; +} + +static SIMPLE_DEV_PM_OPS(exynos_rng_pm_ops, exynos_rng_suspend, + exynos_rng_resume); + +static const struct of_device_id exynos_rng_dt_match[] = { + { + .compatible = "samsung,exynos4-rng", + }, + { }, +}; +MODULE_DEVICE_TABLE(of, exynos_rng_dt_match); + +static struct platform_driver exynos_rng_driver = { + .driver = { + .name = "exynos-rng", + .pm = &exynos_rng_pm_ops, + .of_match_table = exynos_rng_dt_match, + }, + .probe = exynos_rng_probe, + .remove = exynos_rng_remove, +}; + +module_platform_driver(exynos_rng_driver); + +MODULE_DESCRIPTION("Exynos H/W Random Number Generator driver"); +MODULE_AUTHOR("Krzysztof Kozlowski "); +MODULE_LICENSE("GPL"); From 69b348449bda0f9588737539cfe135774c9939a7 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Wed, 12 Apr 2017 10:40:19 +0200 Subject: [PATCH 083/101] padata: get_next is never NULL Per Dan's static checker warning, the code that returns NULL was removed in 2010, so this patch updates the comments and fixes the code assumptions. Signed-off-by: Jason A. Donenfeld Reported-by: Dan Carpenter Acked-by: Steffen Klassert Signed-off-by: Herbert Xu --- kernel/padata.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/kernel/padata.c b/kernel/padata.c index f1aef1639204..ac8f1e524836 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -154,8 +154,6 @@ EXPORT_SYMBOL(padata_do_parallel); * A pointer to the control struct of the next object that needs * serialization, if present in one of the percpu reorder queues. * - * NULL, if all percpu reorder queues are empty. - * * -EINPROGRESS, if the next object that needs serialization will * be parallel processed by another cpu and is not yet present in * the cpu's reorder queue. @@ -182,8 +180,6 @@ static struct padata_priv *padata_get_next(struct parallel_data *pd) cpu = padata_index_to_cpu(pd, next_index); next_queue = per_cpu_ptr(pd->pqueue, cpu); - padata = NULL; - reorder = &next_queue->reorder; spin_lock(&reorder->lock); @@ -235,12 +231,11 @@ static void padata_reorder(struct parallel_data *pd) padata = padata_get_next(pd); /* - * All reorder queues are empty, or the next object that needs - * serialization is parallel processed by another cpu and is - * still on it's way to the cpu's reorder queue, nothing to - * do for now. + * If the next object that needs serialization is parallel + * processed by another cpu and is still on it's way to the + * cpu's reorder queue, nothing to do for now. */ - if (!padata || PTR_ERR(padata) == -EINPROGRESS) + if (PTR_ERR(padata) == -EINPROGRESS) break; /* From ec1bca941a36961933484be0fce3e71981815bad Mon Sep 17 00:00:00 2001 From: Christophe Jaillet Date: Thu, 13 Apr 2017 20:25:15 +0200 Subject: [PATCH 084/101] crypto: chcr - Fix error handling related to 'chcr_alloc_shash' Up to now, 'crypto_alloc_shash()' may return a valid pointer, an error pointer or NULL (in case of invalid parameter) Update it to always return an error pointer in case of error. It now returns ERR_PTR(-EINVAL) instead of NULL in case of invalid parameter. This simplifies error handling. Also fix a crash in 'chcr_authenc_setkey()' if 'chcr_alloc_shash()' returns an error pointer and the "goto out" path is taken. Signed-off-by: Christophe JAILLET Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 53d9ce436795..f00e0d8bd039 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -294,7 +294,7 @@ static inline void get_aes_decrypt_key(unsigned char *dec_key, static struct crypto_shash *chcr_alloc_shash(unsigned int ds) { - struct crypto_shash *base_hash = NULL; + struct crypto_shash *base_hash = ERR_PTR(-EINVAL); switch (ds) { case SHA1_DIGEST_SIZE: @@ -2305,7 +2305,7 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, int err = 0, i, key_ctx_len = 0; unsigned char ck_size = 0; unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 }; - struct crypto_shash *base_hash = NULL; + struct crypto_shash *base_hash = ERR_PTR(-EINVAL); struct algo_param param; int align; u8 *o_ptr = NULL; @@ -2408,7 +2408,7 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, } out: aeadctx->enckey_len = 0; - if (base_hash) + if (!IS_ERR(base_hash)) chcr_free_shash(base_hash); return -EINVAL; } From b2a1d2717910452d002e958019546cdf8c0bf065 Mon Sep 17 00:00:00 2001 From: Markus Elfring Date: Wed, 19 Apr 2017 10:30:47 +0200 Subject: [PATCH 085/101] hwrng: n2 - Use devm_kcalloc() in n2rng_probe() * A multiplication for the size determination of a memory allocation indicated that an array data structure should be processed. Thus use the corresponding function "devm_kcalloc". * Replace the specification of a data structure by a pointer dereference to make the corresponding size determination a bit safer according to the Linux coding style convention. Signed-off-by: Markus Elfring Acked-by: Shannon Nelson Signed-off-by: Herbert Xu --- drivers/char/hw_random/n2-drv.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/char/hw_random/n2-drv.c b/drivers/char/hw_random/n2-drv.c index 31cbdbbaebfc..92dd4e925315 100644 --- a/drivers/char/hw_random/n2-drv.c +++ b/drivers/char/hw_random/n2-drv.c @@ -748,9 +748,7 @@ static int n2rng_probe(struct platform_device *op) dev_info(&op->dev, "Registered RNG HVAPI major %lu minor %lu\n", np->hvapi_major, np->hvapi_minor); - - np->units = devm_kzalloc(&op->dev, - sizeof(struct n2rng_unit) * np->num_units, + np->units = devm_kcalloc(&op->dev, np->num_units, sizeof(*np->units), GFP_KERNEL); err = -ENOMEM; if (!np->units) From 3ce5bc72eb88c02b23374c0e4f619ada27e47552 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 19 Apr 2017 14:23:05 +0100 Subject: [PATCH 086/101] crypto: acomp - allow registration of multiple acomps Add crypto_register_acomps and crypto_unregister_acomps to allow the registration of multiple implementations with one call. Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- crypto/acompress.c | 29 +++++++++++++++++++++++++++++ include/crypto/internal/acompress.h | 3 +++ 2 files changed, 32 insertions(+) diff --git a/crypto/acompress.c b/crypto/acompress.c index 47d11627cd20..1544b7c057fb 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -166,5 +166,34 @@ int crypto_unregister_acomp(struct acomp_alg *alg) } EXPORT_SYMBOL_GPL(crypto_unregister_acomp); +int crypto_register_acomps(struct acomp_alg *algs, int count) +{ + int i, ret; + + for (i = 0; i < count; i++) { + ret = crypto_register_acomp(&algs[i]); + if (ret) + goto err; + } + + return 0; + +err: + for (--i; i >= 0; --i) + crypto_unregister_acomp(&algs[i]); + + return ret; +} +EXPORT_SYMBOL_GPL(crypto_register_acomps); + +void crypto_unregister_acomps(struct acomp_alg *algs, int count) +{ + int i; + + for (i = count - 1; i >= 0; --i) + crypto_unregister_acomp(&algs[i]); +} +EXPORT_SYMBOL_GPL(crypto_unregister_acomps); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous compression type"); diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 1de2b5af12d7..51052f65cefc 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -78,4 +78,7 @@ int crypto_register_acomp(struct acomp_alg *alg); */ int crypto_unregister_acomp(struct acomp_alg *alg); +int crypto_register_acomps(struct acomp_alg *algs, int count); +void crypto_unregister_acomps(struct acomp_alg *algs, int count); + #endif From a9943a0ad19f4a23bb5c4217df0fb37feb7ac339 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 19 Apr 2017 14:27:18 +0100 Subject: [PATCH 087/101] crypto: testmgr - replace compression known answer test Compression implementations might return valid outputs that do not match what specified in the test vectors. For this reason, the testmgr might report that a compression implementation failed the test even if the data produced by the compressor is correct. This implements a decompress-and-verify test for acomp compression tests rather than a known answer test. Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- crypto/testmgr.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index cd075c7d8ee1..8373c727752a 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1458,7 +1458,7 @@ static int test_acomp(struct crypto_acomp *tfm, { const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)); unsigned int i; - char *output; + char *output, *decomp_out; int ret; struct scatterlist src, dst; struct acomp_req *req; @@ -1468,6 +1468,12 @@ static int test_acomp(struct crypto_acomp *tfm, if (!output) return -ENOMEM; + decomp_out = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); + if (!decomp_out) { + kfree(output); + return -ENOMEM; + } + for (i = 0; i < ctcount; i++) { unsigned int dlen = COMP_BUF_SIZE; int ilen = ctemplate[i].inlen; @@ -1506,7 +1512,23 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } - if (req->dlen != ctemplate[i].outlen) { + ilen = req->dlen; + dlen = COMP_BUF_SIZE; + sg_init_one(&src, output, ilen); + sg_init_one(&dst, decomp_out, dlen); + init_completion(&result.completion); + acomp_request_set_params(req, &src, &dst, ilen, dlen); + + ret = wait_async_op(&result, crypto_acomp_decompress(req)); + if (ret) { + pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n", + i + 1, algo, -ret); + kfree(input_vec); + acomp_request_free(req); + goto out; + } + + if (req->dlen != ctemplate[i].inlen) { pr_err("alg: acomp: Compression test %d failed for %s: output len = %d\n", i + 1, algo, req->dlen); ret = -EINVAL; @@ -1515,7 +1537,7 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } - if (memcmp(output, ctemplate[i].output, req->dlen)) { + if (memcmp(input_vec, decomp_out, req->dlen)) { pr_err("alg: acomp: Compression test %d failed for %s\n", i + 1, algo); hexdump(output, req->dlen); @@ -1593,6 +1615,7 @@ static int test_acomp(struct crypto_acomp *tfm, ret = 0; out: + kfree(decomp_out); kfree(output); return ret; } From 0f89f6e188fa929b6de49c00c73f5731a6bd6bac Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Thu, 20 Apr 2017 15:35:09 +1000 Subject: [PATCH 088/101] crypto: crct10dif-vpmsum - Fix missing preempt_disable() In crct10dif_vpmsum() we call enable_kernel_altivec() without first disabling preemption, which is not allowed. It used to be sufficient just to call pagefault_disable(), because that also disabled preemption. But the two were decoupled in commit 8222dbe21e79 ("sched/preempt, mm/fault: Decouple preemption from the page fault logic") in mid 2015. The crct10dif-vpmsum code inherited this bug from the crc32c-vpmsum code on which it was modelled. So add the missing preempt_disable/enable(). We should also call disable_kernel_fp(), although it does nothing by default, there is a debug switch to make it active and all enables should be paired with disables. Fixes: b01df1c16c9a ("crypto: powerpc - Add CRC-T10DIF acceleration") Acked-by: Daniel Axtens Signed-off-by: Michael Ellerman Signed-off-by: Herbert Xu --- arch/powerpc/crypto/crct10dif-vpmsum_glue.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/powerpc/crypto/crct10dif-vpmsum_glue.c b/arch/powerpc/crypto/crct10dif-vpmsum_glue.c index bebfc329f746..02ea277863d1 100644 --- a/arch/powerpc/crypto/crct10dif-vpmsum_glue.c +++ b/arch/powerpc/crypto/crct10dif-vpmsum_glue.c @@ -44,10 +44,13 @@ static u16 crct10dif_vpmsum(u16 crci, unsigned char const *p, size_t len) if (len & ~VMX_ALIGN_MASK) { crc <<= 16; + preempt_disable(); pagefault_disable(); enable_kernel_altivec(); crc = __crct10dif_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); + disable_kernel_altivec(); pagefault_enable(); + preempt_enable(); crc >>= 16; } From c2b06bcd4b5f8e9a64b5e266441cc61ebfb2a4a5 Mon Sep 17 00:00:00 2001 From: Sean Wang Date: Fri, 21 Apr 2017 00:24:25 +0800 Subject: [PATCH 089/101] dt-bindings: hwrng: Add Mediatek hardware random generator bindings Document the devicetree bindings for Mediatek random number generator which could be found on MT7623 SoC or other similar Mediatek SoCs. Signed-off-by: Sean Wang Acked-by: Rob Herring Signed-off-by: Herbert Xu --- .../devicetree/bindings/rng/mtk-rng.txt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 Documentation/devicetree/bindings/rng/mtk-rng.txt diff --git a/Documentation/devicetree/bindings/rng/mtk-rng.txt b/Documentation/devicetree/bindings/rng/mtk-rng.txt new file mode 100644 index 000000000000..a6d62a2abd39 --- /dev/null +++ b/Documentation/devicetree/bindings/rng/mtk-rng.txt @@ -0,0 +1,18 @@ +Device-Tree bindings for Mediatek random number generator +found in Mediatek SoC family + +Required properties: +- compatible : Should be "mediatek,mt7623-rng" +- clocks : list of clock specifiers, corresponding to + entries in clock-names property; +- clock-names : Should contain "rng" entries; +- reg : Specifies base physical address and size of the registers + +Example: + +rng: rng@1020f000 { + compatible = "mediatek,mt7623-rng"; + reg = <0 0x1020f000 0 0x1000>; + clocks = <&infracfg CLK_INFRA_TRNG>; + clock-names = "rng"; +}; From 7701d1ff8ed1b8d71dcccf1ae47470fe22942373 Mon Sep 17 00:00:00 2001 From: Sean Wang Date: Fri, 21 Apr 2017 00:24:26 +0800 Subject: [PATCH 090/101] hwrng: mtk - Add driver for hardware random generator on MT7623 SoC This patch adds support for hardware random generator on MT7623 SoC and should also work on other similar Mediatek SoCs. Currently, the driver is already tested successfully with rng-tools. Signed-off-by: Sean Wang Reviewed-by: PrasannaKumar Muralidharan Signed-off-by: Herbert Xu --- drivers/char/hw_random/Kconfig | 14 +++ drivers/char/hw_random/Makefile | 1 + drivers/char/hw_random/mtk-rng.c | 168 +++++++++++++++++++++++++++++++ 3 files changed, 183 insertions(+) create mode 100644 drivers/char/hw_random/mtk-rng.c diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index bdae802e7154..fd34121934e5 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -409,6 +409,20 @@ config HW_RANDOM_CAVIUM If unsure, say Y. +config HW_RANDOM_MTK + tristate "Mediatek Random Number Generator support" + depends on HW_RANDOM + depends on ARCH_MEDIATEK || COMPILE_TEST + default y + ---help--- + This driver provides kernel-side support for the Random Number + Generator hardware found on Mediatek SoCs. + + To compile this driver as a module, choose M here. the + module will be called mtk-rng. + + If unsure, say Y. + endif # HW_RANDOM config UML_RANDOM diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index 6f1eecc2045c..de6ea808bd3c 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -35,3 +35,4 @@ obj-$(CONFIG_HW_RANDOM_STM32) += stm32-rng.o obj-$(CONFIG_HW_RANDOM_PIC32) += pic32-rng.o obj-$(CONFIG_HW_RANDOM_MESON) += meson-rng.o obj-$(CONFIG_HW_RANDOM_CAVIUM) += cavium-rng.o cavium-rng-vf.o +obj-$(CONFIG_HW_RANDOM_MTK) += mtk-rng.o diff --git a/drivers/char/hw_random/mtk-rng.c b/drivers/char/hw_random/mtk-rng.c new file mode 100644 index 000000000000..df8eb54fd5a3 --- /dev/null +++ b/drivers/char/hw_random/mtk-rng.c @@ -0,0 +1,168 @@ +/* + * Driver for Mediatek Hardware Random Number Generator + * + * Copyright (C) 2017 Sean Wang + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#define MTK_RNG_DEV KBUILD_MODNAME + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define USEC_POLL 2 +#define TIMEOUT_POLL 20 + +#define RNG_CTRL 0x00 +#define RNG_EN BIT(0) +#define RNG_READY BIT(31) + +#define RNG_DATA 0x08 + +#define to_mtk_rng(p) container_of(p, struct mtk_rng, rng) + +struct mtk_rng { + void __iomem *base; + struct clk *clk; + struct hwrng rng; +}; + +static int mtk_rng_init(struct hwrng *rng) +{ + struct mtk_rng *priv = to_mtk_rng(rng); + u32 val; + int err; + + err = clk_prepare_enable(priv->clk); + if (err) + return err; + + val = readl(priv->base + RNG_CTRL); + val |= RNG_EN; + writel(val, priv->base + RNG_CTRL); + + return 0; +} + +static void mtk_rng_cleanup(struct hwrng *rng) +{ + struct mtk_rng *priv = to_mtk_rng(rng); + u32 val; + + val = readl(priv->base + RNG_CTRL); + val &= ~RNG_EN; + writel(val, priv->base + RNG_CTRL); + + clk_disable_unprepare(priv->clk); +} + +static bool mtk_rng_wait_ready(struct hwrng *rng, bool wait) +{ + struct mtk_rng *priv = to_mtk_rng(rng); + int ready; + + ready = readl(priv->base + RNG_CTRL) & RNG_READY; + if (!ready && wait) + readl_poll_timeout_atomic(priv->base + RNG_CTRL, ready, + ready & RNG_READY, USEC_POLL, + TIMEOUT_POLL); + return !!ready; +} + +static int mtk_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) +{ + struct mtk_rng *priv = to_mtk_rng(rng); + int retval = 0; + + while (max >= sizeof(u32)) { + if (!mtk_rng_wait_ready(rng, wait)) + break; + + *(u32 *)buf = readl(priv->base + RNG_DATA); + retval += sizeof(u32); + buf += sizeof(u32); + max -= sizeof(u32); + } + + return retval || !wait ? retval : -EIO; +} + +static int mtk_rng_probe(struct platform_device *pdev) +{ + struct resource *res; + int ret; + struct mtk_rng *priv; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(&pdev->dev, "no iomem resource\n"); + return -ENXIO; + } + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->rng.name = pdev->name; + priv->rng.init = mtk_rng_init; + priv->rng.cleanup = mtk_rng_cleanup; + priv->rng.read = mtk_rng_read; + + priv->clk = devm_clk_get(&pdev->dev, "rng"); + if (IS_ERR(priv->clk)) { + ret = PTR_ERR(priv->clk); + dev_err(&pdev->dev, "no clock for device: %d\n", ret); + return ret; + } + + priv->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(priv->base)) + return PTR_ERR(priv->base); + + ret = devm_hwrng_register(&pdev->dev, &priv->rng); + if (ret) { + dev_err(&pdev->dev, "failed to register rng device: %d\n", + ret); + return ret; + } + + dev_info(&pdev->dev, "registered RNG driver\n"); + + return 0; +} + +static const struct of_device_id mtk_rng_match[] = { + { .compatible = "mediatek,mt7623-rng" }, + {}, +}; +MODULE_DEVICE_TABLE(of, mtk_rng_match); + +static struct platform_driver mtk_rng_driver = { + .probe = mtk_rng_probe, + .driver = { + .name = MTK_RNG_DEV, + .of_match_table = mtk_rng_match, + }, +}; + +module_platform_driver(mtk_rng_driver); + +MODULE_DESCRIPTION("Mediatek Random Number Generator Driver"); +MODULE_AUTHOR("Sean Wang "); +MODULE_LICENSE("GPL"); From 56467cb11cf8ae4db9003f54b3d3425b5f07a10a Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Thu, 20 Apr 2017 15:24:09 -0500 Subject: [PATCH 091/101] crypto: ccp - Use only the relevant interrupt bits Each CCP queue can product interrupts for 4 conditions: operation complete, queue empty, error, and queue stopped. This driver only works with completion and error events. Cc: # 4.9.x+ Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dev-v5.c | 9 +++++---- drivers/crypto/ccp/ccp-dev.h | 5 ++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c index e03d06a54d4e..c7972e733e43 100644 --- a/drivers/crypto/ccp/ccp-dev-v5.c +++ b/drivers/crypto/ccp/ccp-dev-v5.c @@ -801,7 +801,7 @@ static int ccp5_init(struct ccp_device *ccp) ioread32(cmd_q->reg_status); /* Clear the interrupts */ - iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status); + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); } dev_dbg(dev, "Requesting an IRQ...\n"); @@ -882,7 +882,7 @@ static int ccp5_init(struct ccp_device *ccp) /* Enable interrupts */ for (i = 0; i < ccp->cmd_q_count; i++) { cmd_q = &ccp->cmd_q[i]; - iowrite32(ALL_INTERRUPTS, cmd_q->reg_int_enable); + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_int_enable); } dev_dbg(dev, "Registering device...\n"); @@ -942,7 +942,7 @@ static void ccp5_destroy(struct ccp_device *ccp) iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control); /* Disable the interrupts */ - iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status); + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); /* Clear the interrupt status */ iowrite32(0x00, cmd_q->reg_int_enable); @@ -1002,7 +1002,8 @@ static irqreturn_t ccp5_irq_handler(int irq, void *data) cmd_q->int_rcvd = 1; /* Acknowledge the interrupt and wake the kthread */ - iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status); + iowrite32(SUPPORTED_INTERRUPTS, + cmd_q->reg_interrupt_status); wake_up_interruptible(&cmd_q->int_queue); } } diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h index 191274d41036..2dfec019a832 100644 --- a/drivers/crypto/ccp/ccp-dev.h +++ b/drivers/crypto/ccp/ccp-dev.h @@ -109,9 +109,8 @@ #define INT_COMPLETION 0x1 #define INT_ERROR 0x2 #define INT_QUEUE_STOPPED 0x4 -#define ALL_INTERRUPTS (INT_COMPLETION| \ - INT_ERROR| \ - INT_QUEUE_STOPPED) +#define INT_EMPTY_QUEUE 0x8 +#define SUPPORTED_INTERRUPTS (INT_COMPLETION | INT_ERROR) #define LSB_REGION_WIDTH 5 #define MAX_LSB_CNT 8 From 116591fe3eef11c6f06b662c9176385f13891183 Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Thu, 20 Apr 2017 15:24:22 -0500 Subject: [PATCH 092/101] crypto: ccp - Disable interrupts early on unload Ensure that we disable interrupts first when shutting down the driver. Cc: # 4.9.x+ Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dev-v5.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c index c7972e733e43..13b81a1c1184 100644 --- a/drivers/crypto/ccp/ccp-dev-v5.c +++ b/drivers/crypto/ccp/ccp-dev-v5.c @@ -942,10 +942,10 @@ static void ccp5_destroy(struct ccp_device *ccp) iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control); /* Disable the interrupts */ - iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); + iowrite32(0x00, cmd_q->reg_int_enable); /* Clear the interrupt status */ - iowrite32(0x00, cmd_q->reg_int_enable); + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); ioread32(cmd_q->reg_int_status); ioread32(cmd_q->reg_status); } From 899f35fabeea0d51fbfa146aaeaa9fc8f3b00292 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 24 Apr 2017 16:09:50 +0800 Subject: [PATCH 093/101] Revert "crypto: arm64/sha - Add constant operand modifier to ASM_EXPORT" This reverts commit 42ae2922a68ac8d68927ccb052b486f34e5fba71. It causes a regression with older versions of gcc. The consensus is that this should instead be fixed in clang. Reported-by: Paul Gortmaker Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha1-ce-glue.c | 2 +- arch/arm64/crypto/sha2-ce-glue.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index 6b520e3f3ab1..aefda9868627 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -18,7 +18,7 @@ #include #define ASM_EXPORT(sym, val) \ - asm(".globl " #sym "; .set " #sym ", %c0" :: "i"(val)); + asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val)); MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index e3abe11de48c..7cd587564a41 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -18,7 +18,7 @@ #include #define ASM_EXPORT(sym, val) \ - asm(".globl " #sym "; .set " #sym ", %c0" :: "i"(val)); + asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val)); MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); From 6175ca2ba7d2c4265d6d5590d699fbd907b24708 Mon Sep 17 00:00:00 2001 From: Milan Broz Date: Fri, 21 Apr 2017 13:03:06 +0200 Subject: [PATCH 094/101] crypto: testmgr - Allow ecb(cipher_null) in FIPS mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cipher_null is not a real cipher, FIPS mode should not restrict its use. It is used for several tests (for example in cryptsetup testsuite) and also temporarily for reencryption of not yet encrypted device in cryptsetup-reencrypt tool. Problem is easily reproducible with cryptsetup benchmark -c null Signed-off-by: Milan Broz Acked-by: Stephan Müller Signed-off-by: Herbert Xu --- crypto/testmgr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 8373c727752a..6b8661ee8dc0 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -2910,6 +2910,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, { .alg = "ecb(cipher_null)", .test = alg_test_null, + .fips_allowed = 1, }, { .alg = "ecb(des)", .test = alg_test_skcipher, From 7c6c0dc7bb45be9931e432b9c525f72aecdc908a Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 21 Apr 2017 12:13:49 +0100 Subject: [PATCH 095/101] crypto: crypto4xx - rename ce_ring_contol to ce_ring_control trivial spelling mistake, missing r, rename to ce_ring_control Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 2 +- drivers/crypto/amcc/crypto4xx_reg_def.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index d10b4ae5e0da..fdc83a2281ca 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -50,7 +50,7 @@ static void crypto4xx_hw_init(struct crypto4xx_device *dev) { union ce_ring_size ring_size; - union ce_ring_contol ring_ctrl; + union ce_ring_control ring_ctrl; union ce_part_ring_size part_ring_size; union ce_io_threshold io_threshold; u32 rand_num; diff --git a/drivers/crypto/amcc/crypto4xx_reg_def.h b/drivers/crypto/amcc/crypto4xx_reg_def.h index 46fe57c8f6eb..279b8725559f 100644 --- a/drivers/crypto/amcc/crypto4xx_reg_def.h +++ b/drivers/crypto/amcc/crypto4xx_reg_def.h @@ -180,7 +180,7 @@ union ce_ring_size { } __attribute__((packed)); #define CRYPTO4XX_RING_CONTROL_OFFSET 0x54 -union ce_ring_contol { +union ce_ring_control { struct { u32 continuous:1; u32 rsv:5; From 7b537b24e76a1e8e6d7ea91483a45d5b1426809b Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Fri, 21 Apr 2017 10:50:05 -0500 Subject: [PATCH 096/101] crypto: ccp - Change ISR handler method for a v3 CCP The CCP has the ability to perform several operations simultaneously, but only one interrupt. When implemented as a PCI device and using MSI-X/MSI interrupts, use a tasklet model to service interrupts. By disabling and enabling interrupts from the CCP, coupled with the queuing that tasklets provide, we can ensure that all events (occurring on the device) are recognized and serviced. This change fixes a problem wherein 2 or more busy queues can cause notification bits to change state while a (CCP) interrupt is being serviced, but after the queue state has been evaluated. This results in the event being 'lost' and the queue hanging, waiting to be serviced. Since the status bits are never fully de-asserted, the CCP never generates another interrupt (all bits zero -> one or more bits one), and no further CCP operations will be executed. Cc: # 4.9.x+ Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dev-v3.c | 120 +++++++++++++++++++------------- drivers/crypto/ccp/ccp-dev.h | 3 + drivers/crypto/ccp/ccp-pci.c | 2 + 3 files changed, 75 insertions(+), 50 deletions(-) diff --git a/drivers/crypto/ccp/ccp-dev-v3.c b/drivers/crypto/ccp/ccp-dev-v3.c index a3689a66cb1d..367c2e30656f 100644 --- a/drivers/crypto/ccp/ccp-dev-v3.c +++ b/drivers/crypto/ccp/ccp-dev-v3.c @@ -315,17 +315,73 @@ static int ccp_perform_ecc(struct ccp_op *op) return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); } +static void ccp_disable_queue_interrupts(struct ccp_device *ccp) +{ + iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); +} + +static void ccp_enable_queue_interrupts(struct ccp_device *ccp) +{ + iowrite32(ccp->qim, ccp->io_regs + IRQ_MASK_REG); +} + +static void ccp_irq_bh(unsigned long data) +{ + struct ccp_device *ccp = (struct ccp_device *)data; + struct ccp_cmd_queue *cmd_q; + u32 q_int, status; + unsigned int i; + + status = ioread32(ccp->io_regs + IRQ_STATUS_REG); + + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + q_int = status & (cmd_q->int_ok | cmd_q->int_err); + if (q_int) { + cmd_q->int_status = status; + cmd_q->q_status = ioread32(cmd_q->reg_status); + cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); + + /* On error, only save the first error value */ + if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error) + cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); + + cmd_q->int_rcvd = 1; + + /* Acknowledge the interrupt and wake the kthread */ + iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG); + wake_up_interruptible(&cmd_q->int_queue); + } + } + ccp_enable_queue_interrupts(ccp); +} + +static irqreturn_t ccp_irq_handler(int irq, void *data) +{ + struct device *dev = data; + struct ccp_device *ccp = dev_get_drvdata(dev); + + ccp_disable_queue_interrupts(ccp); + if (ccp->use_tasklet) + tasklet_schedule(&ccp->irq_tasklet); + else + ccp_irq_bh((unsigned long)ccp); + + return IRQ_HANDLED; +} + static int ccp_init(struct ccp_device *ccp) { struct device *dev = ccp->dev; struct ccp_cmd_queue *cmd_q; struct dma_pool *dma_pool; char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; - unsigned int qmr, qim, i; + unsigned int qmr, i; int ret; /* Find available queues */ - qim = 0; + ccp->qim = 0; qmr = ioread32(ccp->io_regs + Q_MASK_REG); for (i = 0; i < MAX_HW_QUEUES; i++) { if (!(qmr & (1 << i))) @@ -370,7 +426,7 @@ static int ccp_init(struct ccp_device *ccp) init_waitqueue_head(&cmd_q->int_queue); /* Build queue interrupt mask (two interrupts per queue) */ - qim |= cmd_q->int_ok | cmd_q->int_err; + ccp->qim |= cmd_q->int_ok | cmd_q->int_err; #ifdef CONFIG_ARM64 /* For arm64 set the recommended queue cache settings */ @@ -388,14 +444,14 @@ static int ccp_init(struct ccp_device *ccp) dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count); /* Disable and clear interrupts until ready */ - iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); + ccp_disable_queue_interrupts(ccp); for (i = 0; i < ccp->cmd_q_count; i++) { cmd_q = &ccp->cmd_q[i]; ioread32(cmd_q->reg_int_status); ioread32(cmd_q->reg_status); } - iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG); + iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); /* Request an irq */ ret = ccp->get_irq(ccp); @@ -404,6 +460,11 @@ static int ccp_init(struct ccp_device *ccp) goto e_pool; } + /* Initialize the ISR tasklet? */ + if (ccp->use_tasklet) + tasklet_init(&ccp->irq_tasklet, ccp_irq_bh, + (unsigned long)ccp); + dev_dbg(dev, "Starting threads...\n"); /* Create a kthread for each queue */ for (i = 0; i < ccp->cmd_q_count; i++) { @@ -426,7 +487,7 @@ static int ccp_init(struct ccp_device *ccp) dev_dbg(dev, "Enabling interrupts...\n"); /* Enable interrupts */ - iowrite32(qim, ccp->io_regs + IRQ_MASK_REG); + ccp_enable_queue_interrupts(ccp); dev_dbg(dev, "Registering device...\n"); ccp_add_device(ccp); @@ -463,7 +524,7 @@ static void ccp_destroy(struct ccp_device *ccp) { struct ccp_cmd_queue *cmd_q; struct ccp_cmd *cmd; - unsigned int qim, i; + unsigned int i; /* Unregister the DMA engine */ ccp_dmaengine_unregister(ccp); @@ -474,22 +535,15 @@ static void ccp_destroy(struct ccp_device *ccp) /* Remove this device from the list of available units */ ccp_del_device(ccp); - /* Build queue interrupt mask (two interrupt masks per queue) */ - qim = 0; - for (i = 0; i < ccp->cmd_q_count; i++) { - cmd_q = &ccp->cmd_q[i]; - qim |= cmd_q->int_ok | cmd_q->int_err; - } - /* Disable and clear interrupts */ - iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); + ccp_disable_queue_interrupts(ccp); for (i = 0; i < ccp->cmd_q_count; i++) { cmd_q = &ccp->cmd_q[i]; ioread32(cmd_q->reg_int_status); ioread32(cmd_q->reg_status); } - iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG); + iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); /* Stop the queue kthreads */ for (i = 0; i < ccp->cmd_q_count; i++) @@ -516,40 +570,6 @@ static void ccp_destroy(struct ccp_device *ccp) } } -static irqreturn_t ccp_irq_handler(int irq, void *data) -{ - struct device *dev = data; - struct ccp_device *ccp = dev_get_drvdata(dev); - struct ccp_cmd_queue *cmd_q; - u32 q_int, status; - unsigned int i; - - status = ioread32(ccp->io_regs + IRQ_STATUS_REG); - - for (i = 0; i < ccp->cmd_q_count; i++) { - cmd_q = &ccp->cmd_q[i]; - - q_int = status & (cmd_q->int_ok | cmd_q->int_err); - if (q_int) { - cmd_q->int_status = status; - cmd_q->q_status = ioread32(cmd_q->reg_status); - cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); - - /* On error, only save the first error value */ - if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error) - cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); - - cmd_q->int_rcvd = 1; - - /* Acknowledge the interrupt and wake the kthread */ - iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG); - wake_up_interruptible(&cmd_q->int_queue); - } - } - - return IRQ_HANDLED; -} - static const struct ccp_actions ccp3_actions = { .aes = ccp_perform_aes, .xts_aes = ccp_perform_xts_aes, diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h index 2dfec019a832..0cb09d0feeaf 100644 --- a/drivers/crypto/ccp/ccp-dev.h +++ b/drivers/crypto/ccp/ccp-dev.h @@ -339,7 +339,10 @@ struct ccp_device { void *dev_specific; int (*get_irq)(struct ccp_device *ccp); void (*free_irq)(struct ccp_device *ccp); + unsigned int qim; unsigned int irq; + bool use_tasklet; + struct tasklet_struct irq_tasklet; /* I/O area used for device communication. The register mapping * starts at an offset into the mapped bar. diff --git a/drivers/crypto/ccp/ccp-pci.c b/drivers/crypto/ccp/ccp-pci.c index 28a9996c1085..e880d4cf4ada 100644 --- a/drivers/crypto/ccp/ccp-pci.c +++ b/drivers/crypto/ccp/ccp-pci.c @@ -69,6 +69,7 @@ static int ccp_get_msix_irqs(struct ccp_device *ccp) goto e_irq; } } + ccp->use_tasklet = true; return 0; @@ -100,6 +101,7 @@ static int ccp_get_msi_irq(struct ccp_device *ccp) dev_notice(dev, "unable to allocate MSI IRQ (%d)\n", ret); goto e_msi; } + ccp->use_tasklet = true; return 0; From 6263b51eb3190d30351360fd168959af7e3a49a9 Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Fri, 21 Apr 2017 10:50:14 -0500 Subject: [PATCH 097/101] crypto: ccp - Change ISR handler method for a v5 CCP The CCP has the ability to perform several operations simultaneously, but only one interrupt. When implemented as a PCI device and using MSI-X/MSI interrupts, use a tasklet model to service interrupts. By disabling and enabling interrupts from the CCP, coupled with the queuing that tasklets provide, we can ensure that all events (occurring on the device) are recognized and serviced. This change fixes a problem wherein 2 or more busy queues can cause notification bits to change state while a (CCP) interrupt is being serviced, but after the queue state has been evaluated. This results in the event being 'lost' and the queue hanging, waiting to be serviced. Since the status bits are never fully de-asserted, the CCP never generates another interrupt (all bits zero -> one or more bits one), and no further CCP operations will be executed. Cc: # 4.9.x+ Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dev-v5.c | 111 +++++++++++++++++++------------- 1 file changed, 67 insertions(+), 44 deletions(-) diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c index 13b81a1c1184..ccbe32d5dd1c 100644 --- a/drivers/crypto/ccp/ccp-dev-v5.c +++ b/drivers/crypto/ccp/ccp-dev-v5.c @@ -705,6 +705,65 @@ static int ccp_assign_lsbs(struct ccp_device *ccp) return rc; } +static void ccp5_disable_queue_interrupts(struct ccp_device *ccp) +{ + unsigned int i; + + for (i = 0; i < ccp->cmd_q_count; i++) + iowrite32(0x0, ccp->cmd_q[i].reg_int_enable); +} + +static void ccp5_enable_queue_interrupts(struct ccp_device *ccp) +{ + unsigned int i; + + for (i = 0; i < ccp->cmd_q_count; i++) + iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable); +} + +static void ccp5_irq_bh(unsigned long data) +{ + struct ccp_device *ccp = (struct ccp_device *)data; + u32 status; + unsigned int i; + + for (i = 0; i < ccp->cmd_q_count; i++) { + struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; + + status = ioread32(cmd_q->reg_interrupt_status); + + if (status) { + cmd_q->int_status = status; + cmd_q->q_status = ioread32(cmd_q->reg_status); + cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); + + /* On error, only save the first error value */ + if ((status & INT_ERROR) && !cmd_q->cmd_error) + cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); + + cmd_q->int_rcvd = 1; + + /* Acknowledge the interrupt and wake the kthread */ + iowrite32(status, cmd_q->reg_interrupt_status); + wake_up_interruptible(&cmd_q->int_queue); + } + } + ccp5_enable_queue_interrupts(ccp); +} + +static irqreturn_t ccp5_irq_handler(int irq, void *data) +{ + struct device *dev = data; + struct ccp_device *ccp = dev_get_drvdata(dev); + + ccp5_disable_queue_interrupts(ccp); + if (ccp->use_tasklet) + tasklet_schedule(&ccp->irq_tasklet); + else + ccp5_irq_bh((unsigned long)ccp); + return IRQ_HANDLED; +} + static int ccp5_init(struct ccp_device *ccp) { struct device *dev = ccp->dev; @@ -789,18 +848,17 @@ static int ccp5_init(struct ccp_device *ccp) } /* Turn off the queues and disable interrupts until ready */ + ccp5_disable_queue_interrupts(ccp); for (i = 0; i < ccp->cmd_q_count; i++) { cmd_q = &ccp->cmd_q[i]; cmd_q->qcontrol = 0; /* Start with nothing */ iowrite32(cmd_q->qcontrol, cmd_q->reg_control); - /* Disable the interrupts */ - iowrite32(0x00, cmd_q->reg_int_enable); ioread32(cmd_q->reg_int_status); ioread32(cmd_q->reg_status); - /* Clear the interrupts */ + /* Clear the interrupt status */ iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); } @@ -811,6 +869,10 @@ static int ccp5_init(struct ccp_device *ccp) dev_err(dev, "unable to allocate an IRQ\n"); goto e_pool; } + /* Initialize the ISR tasklet */ + if (ccp->use_tasklet) + tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh, + (unsigned long)ccp); dev_dbg(dev, "Loading LSB map...\n"); /* Copy the private LSB mask to the public registers */ @@ -879,11 +941,7 @@ static int ccp5_init(struct ccp_device *ccp) } dev_dbg(dev, "Enabling interrupts...\n"); - /* Enable interrupts */ - for (i = 0; i < ccp->cmd_q_count; i++) { - cmd_q = &ccp->cmd_q[i]; - iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_int_enable); - } + ccp5_enable_queue_interrupts(ccp); dev_dbg(dev, "Registering device...\n"); /* Put this on the unit list to make it available */ @@ -935,15 +993,13 @@ static void ccp5_destroy(struct ccp_device *ccp) ccp_del_device(ccp); /* Disable and clear interrupts */ + ccp5_disable_queue_interrupts(ccp); for (i = 0; i < ccp->cmd_q_count; i++) { cmd_q = &ccp->cmd_q[i]; /* Turn off the run bit */ iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control); - /* Disable the interrupts */ - iowrite32(0x00, cmd_q->reg_int_enable); - /* Clear the interrupt status */ iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); ioread32(cmd_q->reg_int_status); @@ -978,39 +1034,6 @@ static void ccp5_destroy(struct ccp_device *ccp) } } -static irqreturn_t ccp5_irq_handler(int irq, void *data) -{ - struct device *dev = data; - struct ccp_device *ccp = dev_get_drvdata(dev); - u32 status; - unsigned int i; - - for (i = 0; i < ccp->cmd_q_count; i++) { - struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; - - status = ioread32(cmd_q->reg_interrupt_status); - - if (status) { - cmd_q->int_status = status; - cmd_q->q_status = ioread32(cmd_q->reg_status); - cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); - - /* On error, only save the first error value */ - if ((status & INT_ERROR) && !cmd_q->cmd_error) - cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); - - cmd_q->int_rcvd = 1; - - /* Acknowledge the interrupt and wake the kthread */ - iowrite32(SUPPORTED_INTERRUPTS, - cmd_q->reg_interrupt_status); - wake_up_interruptible(&cmd_q->int_queue); - } - } - - return IRQ_HANDLED; -} - static void ccp5_config(struct ccp_device *ccp) { /* Public side */ From 3de4f5e1a5dbe1a36d1e8a08ee1978f44c4b739b Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Fri, 21 Apr 2017 21:54:29 +0100 Subject: [PATCH 098/101] crypto: scomp - allow registration of multiple scomps Add crypto_register_scomps and crypto_unregister_scomps to allow the registration of multiple implementations with one call. Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- crypto/scompress.c | 29 +++++++++++++++++++++++++++++ include/crypto/internal/scompress.h | 3 +++ 2 files changed, 32 insertions(+) diff --git a/crypto/scompress.c b/crypto/scompress.c index 6b048b36312d..ae1d3cf209e4 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -353,5 +353,34 @@ int crypto_unregister_scomp(struct scomp_alg *alg) } EXPORT_SYMBOL_GPL(crypto_unregister_scomp); +int crypto_register_scomps(struct scomp_alg *algs, int count) +{ + int i, ret; + + for (i = 0; i < count; i++) { + ret = crypto_register_scomp(&algs[i]); + if (ret) + goto err; + } + + return 0; + +err: + for (--i; i >= 0; --i) + crypto_unregister_scomp(&algs[i]); + + return ret; +} +EXPORT_SYMBOL_GPL(crypto_register_scomps); + +void crypto_unregister_scomps(struct scomp_alg *algs, int count) +{ + int i; + + for (i = count - 1; i >= 0; --i) + crypto_unregister_scomp(&algs[i]); +} +EXPORT_SYMBOL_GPL(crypto_unregister_scomps); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Synchronous compression type"); diff --git a/include/crypto/internal/scompress.h b/include/crypto/internal/scompress.h index 3fda3c5655a0..ccad9b2c9bd6 100644 --- a/include/crypto/internal/scompress.h +++ b/include/crypto/internal/scompress.h @@ -133,4 +133,7 @@ int crypto_register_scomp(struct scomp_alg *alg); */ int crypto_unregister_scomp(struct scomp_alg *alg); +int crypto_register_scomps(struct scomp_alg *algs, int count); +void crypto_unregister_scomps(struct scomp_alg *algs, int count); + #endif From a368f43d6e3a001e684e9191a27df384fbff12f5 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Fri, 21 Apr 2017 21:54:30 +0100 Subject: [PATCH 099/101] crypto: scomp - add support for deflate rfc1950 (zlib) Add scomp backend for zlib-deflate compression algorithm. This backend outputs data using the format defined in rfc1950 (raw deflate surrounded by zlib header and footer). Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- crypto/deflate.c | 61 ++++++++++++++++++++++++++++----------- crypto/testmgr.c | 10 +++++++ crypto/testmgr.h | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 17 deletions(-) diff --git a/crypto/deflate.c b/crypto/deflate.c index f942cb391890..94ec3b36a8e8 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -43,20 +43,24 @@ struct deflate_ctx { struct z_stream_s decomp_stream; }; -static int deflate_comp_init(struct deflate_ctx *ctx) +static int deflate_comp_init(struct deflate_ctx *ctx, int format) { int ret = 0; struct z_stream_s *stream = &ctx->comp_stream; stream->workspace = vzalloc(zlib_deflate_workspacesize( - -DEFLATE_DEF_WINBITS, DEFLATE_DEF_MEMLEVEL)); + MAX_WBITS, MAX_MEM_LEVEL)); if (!stream->workspace) { ret = -ENOMEM; goto out; } - ret = zlib_deflateInit2(stream, DEFLATE_DEF_LEVEL, Z_DEFLATED, - -DEFLATE_DEF_WINBITS, DEFLATE_DEF_MEMLEVEL, - Z_DEFAULT_STRATEGY); + if (format) + ret = zlib_deflateInit(stream, 3); + else + ret = zlib_deflateInit2(stream, DEFLATE_DEF_LEVEL, Z_DEFLATED, + -DEFLATE_DEF_WINBITS, + DEFLATE_DEF_MEMLEVEL, + Z_DEFAULT_STRATEGY); if (ret != Z_OK) { ret = -EINVAL; goto out_free; @@ -68,7 +72,7 @@ out_free: goto out; } -static int deflate_decomp_init(struct deflate_ctx *ctx) +static int deflate_decomp_init(struct deflate_ctx *ctx, int format) { int ret = 0; struct z_stream_s *stream = &ctx->decomp_stream; @@ -78,7 +82,10 @@ static int deflate_decomp_init(struct deflate_ctx *ctx) ret = -ENOMEM; goto out; } - ret = zlib_inflateInit2(stream, -DEFLATE_DEF_WINBITS); + if (format) + ret = zlib_inflateInit(stream); + else + ret = zlib_inflateInit2(stream, -DEFLATE_DEF_WINBITS); if (ret != Z_OK) { ret = -EINVAL; goto out_free; @@ -102,21 +109,21 @@ static void deflate_decomp_exit(struct deflate_ctx *ctx) vfree(ctx->decomp_stream.workspace); } -static int __deflate_init(void *ctx) +static int __deflate_init(void *ctx, int format) { int ret; - ret = deflate_comp_init(ctx); + ret = deflate_comp_init(ctx, format); if (ret) goto out; - ret = deflate_decomp_init(ctx); + ret = deflate_decomp_init(ctx, format); if (ret) deflate_comp_exit(ctx); out: return ret; } -static void *deflate_alloc_ctx(struct crypto_scomp *tfm) +static void *gen_deflate_alloc_ctx(struct crypto_scomp *tfm, int format) { struct deflate_ctx *ctx; int ret; @@ -125,7 +132,7 @@ static void *deflate_alloc_ctx(struct crypto_scomp *tfm) if (!ctx) return ERR_PTR(-ENOMEM); - ret = __deflate_init(ctx); + ret = __deflate_init(ctx, format); if (ret) { kfree(ctx); return ERR_PTR(ret); @@ -134,11 +141,21 @@ static void *deflate_alloc_ctx(struct crypto_scomp *tfm) return ctx; } +static void *deflate_alloc_ctx(struct crypto_scomp *tfm) +{ + return gen_deflate_alloc_ctx(tfm, 0); +} + +static void *zlib_deflate_alloc_ctx(struct crypto_scomp *tfm) +{ + return gen_deflate_alloc_ctx(tfm, 1); +} + static int deflate_init(struct crypto_tfm *tfm) { struct deflate_ctx *ctx = crypto_tfm_ctx(tfm); - return __deflate_init(ctx); + return __deflate_init(ctx, 0); } static void __deflate_exit(void *ctx) @@ -272,7 +289,7 @@ static struct crypto_alg alg = { .coa_decompress = deflate_decompress } } }; -static struct scomp_alg scomp = { +static struct scomp_alg scomp[] = { { .alloc_ctx = deflate_alloc_ctx, .free_ctx = deflate_free_ctx, .compress = deflate_scompress, @@ -282,7 +299,17 @@ static struct scomp_alg scomp = { .cra_driver_name = "deflate-scomp", .cra_module = THIS_MODULE, } -}; +}, { + .alloc_ctx = zlib_deflate_alloc_ctx, + .free_ctx = deflate_free_ctx, + .compress = deflate_scompress, + .decompress = deflate_sdecompress, + .base = { + .cra_name = "zlib-deflate", + .cra_driver_name = "zlib-deflate-scomp", + .cra_module = THIS_MODULE, + } +} }; static int __init deflate_mod_init(void) { @@ -292,7 +319,7 @@ static int __init deflate_mod_init(void) if (ret) return ret; - ret = crypto_register_scomp(&scomp); + ret = crypto_register_scomps(scomp, ARRAY_SIZE(scomp)); if (ret) { crypto_unregister_alg(&alg); return ret; @@ -304,7 +331,7 @@ static int __init deflate_mod_init(void) static void __exit deflate_mod_fini(void) { crypto_unregister_alg(&alg); - crypto_unregister_scomp(&scomp); + crypto_unregister_scomps(scomp, ARRAY_SIZE(scomp)); } module_init(deflate_mod_init); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 6b8661ee8dc0..6f5f3ed8376c 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -3513,6 +3513,16 @@ static const struct alg_test_desc alg_test_descs[] = { .dec = __VECS(tf_xts_dec_tv_template) } } + }, { + .alg = "zlib-deflate", + .test = alg_test_comp, + .fips_allowed = 1, + .suite = { + .comp = { + .comp = __VECS(zlib_deflate_comp_tv_template), + .decomp = __VECS(zlib_deflate_decomp_tv_template) + } + } } }; diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 15c043f74b18..429357339dcc 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -33204,6 +33204,81 @@ static const struct comp_testvec deflate_decomp_tv_template[] = { }, }; +static const struct comp_testvec zlib_deflate_comp_tv_template[] = { + { + .inlen = 70, + .outlen = 44, + .input = "Join us now and share the software " + "Join us now and share the software ", + .output = "\x78\x5e\xf3\xca\xcf\xcc\x53\x28" + "\x2d\x56\xc8\xcb\x2f\x57\x48\xcc" + "\x4b\x51\x28\xce\x48\x2c\x4a\x55" + "\x28\xc9\x48\x55\x28\xce\x4f\x2b" + "\x29\x07\x71\xbc\x08\x2b\x01\x00" + "\x7c\x65\x19\x3d", + }, { + .inlen = 191, + .outlen = 129, + .input = "This document describes a compression method based on the DEFLATE" + "compression algorithm. This document defines the application of " + "the DEFLATE algorithm to the IP Payload Compression Protocol.", + .output = "\x78\x5e\x5d\xce\x41\x0a\xc3\x30" + "\x0c\x04\xc0\xaf\xec\x0b\xf2\x87" + "\xd2\xa6\x50\xe8\xc1\x07\x7f\x40" + "\xb1\x95\x5a\x60\x5b\xc6\x56\x0f" + "\xfd\x7d\x93\x1e\x42\xe8\x51\xec" + "\xee\x20\x9f\x64\x20\x6a\x78\x17" + "\xae\x86\xc8\x23\x74\x59\x78\x80" + "\x10\xb4\xb4\xce\x63\x88\x56\x14" + "\xb6\xa4\x11\x0b\x0d\x8e\xd8\x6e" + "\x4b\x8c\xdb\x7c\x7f\x5e\xfc\x7c" + "\xae\x51\x7e\x69\x17\x4b\x65\x02" + "\xfc\x1f\xbc\x4a\xdd\xd8\x7d\x48" + "\xad\x65\x09\x64\x3b\xac\xeb\xd9" + "\xc2\x01\xc0\xf4\x17\x3c\x1c\x1c" + "\x7d\xb2\x52\xc4\xf5\xf4\x8f\xeb" + "\x6a\x1a\x34\x4f\x5f\x2e\x32\x45" + "\x4e", + }, +}; + +static const struct comp_testvec zlib_deflate_decomp_tv_template[] = { + { + .inlen = 128, + .outlen = 191, + .input = "\x78\x9c\x5d\x8d\x31\x0e\xc2\x30" + "\x10\x04\xbf\xb2\x2f\xc8\x1f\x10" + "\x04\x09\x89\xc2\x85\x3f\x70\xb1" + "\x2f\xf8\x24\xdb\x67\xd9\x47\xc1" + "\xef\x49\x68\x12\x51\xae\x76\x67" + "\xd6\x27\x19\x88\x1a\xde\x85\xab" + "\x21\xf2\x08\x5d\x16\x1e\x20\x04" + "\x2d\xad\xf3\x18\xa2\x15\x85\x2d" + "\x69\xc4\x42\x83\x23\xb6\x6c\x89" + "\x71\x9b\xef\xcf\x8b\x9f\xcf\x33" + "\xca\x2f\xed\x62\xa9\x4c\x80\xff" + "\x13\xaf\x52\x37\xed\x0e\x52\x6b" + "\x59\x02\xd9\x4e\xe8\x7a\x76\x1d" + "\x02\x98\xfe\x8a\x87\x83\xa3\x4f" + "\x56\x8a\xb8\x9e\x8e\x5c\x57\xd3" + "\xa0\x79\xfa\x02\x2e\x32\x45\x4e", + .output = "This document describes a compression method based on the DEFLATE" + "compression algorithm. This document defines the application of " + "the DEFLATE algorithm to the IP Payload Compression Protocol.", + }, { + .inlen = 44, + .outlen = 70, + .input = "\x78\x9c\xf3\xca\xcf\xcc\x53\x28" + "\x2d\x56\xc8\xcb\x2f\x57\x48\xcc" + "\x4b\x51\x28\xce\x48\x2c\x4a\x55" + "\x28\xc9\x48\x55\x28\xce\x4f\x2b" + "\x29\x07\x71\xbc\x08\x2b\x01\x00" + "\x7c\x65\x19\x3d", + .output = "Join us now and share the software " + "Join us now and share the software ", + }, +}; + /* * LZO test vectors (null-terminated strings). */ From 2a2a251f110576b1d89efbd0662677d7e7db21a8 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Mon, 24 Apr 2017 11:15:23 +0200 Subject: [PATCH 100/101] crypto: algif_aead - Require setkey before accept(2) Some cipher implementations will crash if you try to use them without calling setkey first. This patch adds a check so that the accept(2) call will fail with -ENOKEY if setkey hasn't been done on the socket yet. Fixes: 400c40cf78da ("crypto: algif - add AEAD support") Cc: Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/algif_aead.c | 157 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 149 insertions(+), 8 deletions(-) diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index 5a8053758657..e0d55ea2f0eb 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -44,6 +44,11 @@ struct aead_async_req { char iv[]; }; +struct aead_tfm { + struct crypto_aead *aead; + bool has_key; +}; + struct aead_ctx { struct aead_sg_list tsgl; struct aead_async_rsgl first_rsgl; @@ -723,24 +728,146 @@ static struct proto_ops algif_aead_ops = { .poll = aead_poll, }; +static int aead_check_key(struct socket *sock) +{ + int err = 0; + struct sock *psk; + struct alg_sock *pask; + struct aead_tfm *tfm; + struct sock *sk = sock->sk; + struct alg_sock *ask = alg_sk(sk); + + lock_sock(sk); + if (ask->refcnt) + goto unlock_child; + + psk = ask->parent; + pask = alg_sk(ask->parent); + tfm = pask->private; + + err = -ENOKEY; + lock_sock_nested(psk, SINGLE_DEPTH_NESTING); + if (!tfm->has_key) + goto unlock; + + if (!pask->refcnt++) + sock_hold(psk); + + ask->refcnt = 1; + sock_put(psk); + + err = 0; + +unlock: + release_sock(psk); +unlock_child: + release_sock(sk); + + return err; +} + +static int aead_sendmsg_nokey(struct socket *sock, struct msghdr *msg, + size_t size) +{ + int err; + + err = aead_check_key(sock); + if (err) + return err; + + return aead_sendmsg(sock, msg, size); +} + +static ssize_t aead_sendpage_nokey(struct socket *sock, struct page *page, + int offset, size_t size, int flags) +{ + int err; + + err = aead_check_key(sock); + if (err) + return err; + + return aead_sendpage(sock, page, offset, size, flags); +} + +static int aead_recvmsg_nokey(struct socket *sock, struct msghdr *msg, + size_t ignored, int flags) +{ + int err; + + err = aead_check_key(sock); + if (err) + return err; + + return aead_recvmsg(sock, msg, ignored, flags); +} + +static struct proto_ops algif_aead_ops_nokey = { + .family = PF_ALG, + + .connect = sock_no_connect, + .socketpair = sock_no_socketpair, + .getname = sock_no_getname, + .ioctl = sock_no_ioctl, + .listen = sock_no_listen, + .shutdown = sock_no_shutdown, + .getsockopt = sock_no_getsockopt, + .mmap = sock_no_mmap, + .bind = sock_no_bind, + .accept = sock_no_accept, + .setsockopt = sock_no_setsockopt, + + .release = af_alg_release, + .sendmsg = aead_sendmsg_nokey, + .sendpage = aead_sendpage_nokey, + .recvmsg = aead_recvmsg_nokey, + .poll = aead_poll, +}; + static void *aead_bind(const char *name, u32 type, u32 mask) { - return crypto_alloc_aead(name, type, mask); + struct aead_tfm *tfm; + struct crypto_aead *aead; + + tfm = kzalloc(sizeof(*tfm), GFP_KERNEL); + if (!tfm) + return ERR_PTR(-ENOMEM); + + aead = crypto_alloc_aead(name, type, mask); + if (IS_ERR(aead)) { + kfree(tfm); + return ERR_CAST(aead); + } + + tfm->aead = aead; + + return tfm; } static void aead_release(void *private) { - crypto_free_aead(private); + struct aead_tfm *tfm = private; + + crypto_free_aead(tfm->aead); + kfree(tfm); } static int aead_setauthsize(void *private, unsigned int authsize) { - return crypto_aead_setauthsize(private, authsize); + struct aead_tfm *tfm = private; + + return crypto_aead_setauthsize(tfm->aead, authsize); } static int aead_setkey(void *private, const u8 *key, unsigned int keylen) { - return crypto_aead_setkey(private, key, keylen); + struct aead_tfm *tfm = private; + int err; + + err = crypto_aead_setkey(tfm->aead, key, keylen); + tfm->has_key = !err; + + return err; } static void aead_sock_destruct(struct sock *sk) @@ -757,12 +884,14 @@ static void aead_sock_destruct(struct sock *sk) af_alg_release_parent(sk); } -static int aead_accept_parent(void *private, struct sock *sk) +static int aead_accept_parent_nokey(void *private, struct sock *sk) { struct aead_ctx *ctx; struct alg_sock *ask = alg_sk(sk); - unsigned int len = sizeof(*ctx) + crypto_aead_reqsize(private); - unsigned int ivlen = crypto_aead_ivsize(private); + struct aead_tfm *tfm = private; + struct crypto_aead *aead = tfm->aead; + unsigned int len = sizeof(*ctx) + crypto_aead_reqsize(aead); + unsigned int ivlen = crypto_aead_ivsize(aead); ctx = sock_kmalloc(sk, len, GFP_KERNEL); if (!ctx) @@ -789,7 +918,7 @@ static int aead_accept_parent(void *private, struct sock *sk) ask->private = ctx; - aead_request_set_tfm(&ctx->aead_req, private); + aead_request_set_tfm(&ctx->aead_req, aead); aead_request_set_callback(&ctx->aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, af_alg_complete, &ctx->completion); @@ -798,13 +927,25 @@ static int aead_accept_parent(void *private, struct sock *sk) return 0; } +static int aead_accept_parent(void *private, struct sock *sk) +{ + struct aead_tfm *tfm = private; + + if (!tfm->has_key) + return -ENOKEY; + + return aead_accept_parent_nokey(private, sk); +} + static const struct af_alg_type algif_type_aead = { .bind = aead_bind, .release = aead_release, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .accept = aead_accept_parent, + .accept_nokey = aead_accept_parent_nokey, .ops = &algif_aead_ops, + .ops_nokey = &algif_aead_ops_nokey, .name = "aead", .owner = THIS_MODULE }; From 929562b144783b9212625305eadcbbd800809643 Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Tue, 25 Apr 2017 14:35:42 +0000 Subject: [PATCH 101/101] crypto: stm32 - Fix OF module alias information The module alias information passed to MODULE_DEVICE_TABLE() should use stm32_dt_ids instead of undefined sti_dt_ids. Fixes: b51dbe90912a ("crypto: stm32 - Support for STM32 CRC32 crypto module") Signed-off-by: Wei Yongjun Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32_crc32.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/stm32/stm32_crc32.c b/drivers/crypto/stm32/stm32_crc32.c index 765282262d44..ec83b1e6bfe8 100644 --- a/drivers/crypto/stm32/stm32_crc32.c +++ b/drivers/crypto/stm32/stm32_crc32.c @@ -306,7 +306,7 @@ static const struct of_device_id stm32_dt_ids[] = { { .compatible = "st,stm32f7-crc", }, {}, }; -MODULE_DEVICE_TABLE(of, sti_dt_ids); +MODULE_DEVICE_TABLE(of, stm32_dt_ids); static struct platform_driver stm32_crc_driver = { .probe = stm32_crc_probe,