From e51034bab378a22cd0b287eed754ccb16eb4a4f1 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Tue, 6 Jul 2021 07:07:18 -0700 Subject: [PATCH] Make GCM AES faster 13.22% mbedtls_aesni_gcm_mult 13.03% mbedtls_gcm_update 9.85% mbedtls_aesni_crypt_ecb Overhead improvement (perf record) 10.97% mbedtls_aesni_gcm_mult 10.59% mbedtls_aesni_crypt_ecb 2.26% mbedtls_gcm_update --- third_party/mbedtls/README.cosmo | 4 + third_party/mbedtls/aesni.c | 201 +++++++------ third_party/mbedtls/aesni.h | 97 +------ third_party/mbedtls/base64.c | 10 +- third_party/mbedtls/cipher.c | 12 +- third_party/mbedtls/error.c | 2 - third_party/mbedtls/gcm.c | 470 +++++++++++++++++++------------ third_party/mbedtls/gcm.h | 289 ++----------------- third_party/mbedtls/platform.h | 5 +- third_party/mbedtls/sha1.c | 2 +- 10 files changed, 452 insertions(+), 640 deletions(-) diff --git a/third_party/mbedtls/README.cosmo b/third_party/mbedtls/README.cosmo index 1c45cdcf3..72aaf4536 100644 --- a/third_party/mbedtls/README.cosmo +++ b/third_party/mbedtls/README.cosmo @@ -47,3 +47,7 @@ LOCAL CHANGES doesn't have large amounts of duplicated generated code. - Make chacha20 26% faster. + + - Make base64 100x faster. + + - Make gcm faster. diff --git a/third_party/mbedtls/aesni.c b/third_party/mbedtls/aesni.c index 19ac5f399..064be41c9 100644 --- a/third_party/mbedtls/aesni.c +++ b/third_party/mbedtls/aesni.c @@ -1,3 +1,4 @@ +#include "libc/bits/bits.h" #include "libc/str/str.h" #include "third_party/mbedtls/aesni.h" #include "third_party/mbedtls/common.h" @@ -62,8 +63,18 @@ asm(".include \"libc/disclaimer.inc\""); #define xmm1_xmm0 "0xC1" #define xmm1_xmm2 "0xD1" -/* - * AES-NI AES-ECB block en(de)cryption +/** + * \brief Internal AES-NI AES-ECB block encryption and decryption + * + * \note This function is only for internal use by other library + * functions; you must not call it directly. + * + * \param ctx AES context + * \param mode MBEDTLS_AES_ENCRYPT or MBEDTLS_AES_DECRYPT + * \param input 16-byte input block + * \param output 16-byte output block + * + * \return 0 on success (cannot fail) */ int mbedtls_aesni_crypt_ecb( mbedtls_aes_context *ctx, int mode, @@ -107,62 +118,64 @@ int mbedtls_aesni_crypt_ecb( mbedtls_aes_context *ctx, return( 0 ); } -/* - * GCM multiplication: c = a times b in GF(2^128) - * Based on [CLMUL-WP] algorithms 1 (with equation 27) and 5. +/** + * \brief Internal GCM multiplication: a = a * b in GF(2^128) + * Based on [CLMUL-WP] algorithms 1 (with equation 27) and 5. + * + * \note This function is intended for internal use. + * + * \param a First operand (big endian) + * \param b Second operand (host endian) + * + * \note Both operands and result are bit strings interpreted as + * elements of GF(2^128) as per the GCM spec. */ -void mbedtls_aesni_gcm_mult( unsigned char c[16], - const unsigned char a[16], - const unsigned char b[16] ) +void mbedtls_aesni_gcm_mult( unsigned char a[16], const uint64_t b[2] ) { - unsigned char aa[16], bb[16], cc[16]; size_t i; + uint64_t aa _Vector_size(16) forcealign(16); + uint64_t bb _Vector_size(16) forcealign(16); /* The inputs are in big-endian order, so byte-reverse them */ - for( i = 0; i < 16; i++ ) - { - aa[i] = a[15 - i]; - bb[i] = b[15 - i]; - } + aa[0] = READ64BE(a+8); + aa[1] = READ64BE(a+0); + memcpy(&bb, b, 16); - asm( "movdqu (%0), %%xmm0 \n\t" // a1:a0 - "movdqu (%1), %%xmm1 \n\t" // b1:b0 - - /* + asm( /* * Caryless multiplication xmm2:xmm1 = xmm0 * xmm1 * using [CLMUL-WP] algorithm 1 (p. 13). */ - "movdqa %%xmm1, %%xmm2 \n\t" // copy of b1:b0 - "movdqa %%xmm1, %%xmm3 \n\t" // same - "movdqa %%xmm1, %%xmm4 \n\t" // same - PCLMULQDQ xmm0_xmm1 ",0x00 \n\t" // a0*b0 = c1:c0 - PCLMULQDQ xmm0_xmm2 ",0x11 \n\t" // a1*b1 = d1:d0 - PCLMULQDQ xmm0_xmm3 ",0x10 \n\t" // a0*b1 = e1:e0 - PCLMULQDQ xmm0_xmm4 ",0x01 \n\t" // a1*b0 = f1:f0 - "pxor %%xmm3, %%xmm4 \n\t" // e1+f1:e0+f0 - "movdqa %%xmm4, %%xmm3 \n\t" // same - "psrldq $8, %%xmm4 \n\t" // 0:e1+f1 - "pslldq $8, %%xmm3 \n\t" // e0+f0:0 - "pxor %%xmm4, %%xmm2 \n\t" // d1:d0+e1+f1 - "pxor %%xmm3, %%xmm1 \n\t" // c1+e0+f1:c0 + "movdqa %1, %%xmm2 \n\t" // copy of b1:b0 + "movdqa %1, %%xmm3 \n\t" // same + "movdqa %1, %%xmm4 \n\t" // same + "pclmullqlqdq %0,%1 \n\t" // a0*b0 = c1:c0 + "pclmulhqhqdq %0,%%xmm2 \n\t" // a1*b1 = d1:d0 + "pclmullqhqdq %0,%%xmm3 \n\t" // a0*b1 = e1:e0 + "pclmulhqlqdq %0,%%xmm4 \n\t" // a1*b0 = f1:f0 + "pxor %%xmm3, %%xmm4 \n\t" // e1+f1:e0+f0 + "movdqa %%xmm4, %%xmm3 \n\t" // same + "psrldq $8, %%xmm4 \n\t" // 0:e1+f1 + "pslldq $8, %%xmm3 \n\t" // e0+f0:0 + "pxor %%xmm4, %%xmm2 \n\t" // d1:d0+e1+f1 + "pxor %%xmm3, %1 \n\t" // c1+e0+f1:c0 /* * Now shift the result one bit to the left, * taking advantage of [CLMUL-WP] eq 27 (p. 20) */ - "movdqa %%xmm1, %%xmm3 \n\t" // r1:r0 - "movdqa %%xmm2, %%xmm4 \n\t" // r3:r2 - "psllq $1, %%xmm1 \n\t" // r1<<1:r0<<1 - "psllq $1, %%xmm2 \n\t" // r3<<1:r2<<1 - "psrlq $63, %%xmm3 \n\t" // r1>>63:r0>>63 - "psrlq $63, %%xmm4 \n\t" // r3>>63:r2>>63 - "movdqa %%xmm3, %%xmm5 \n\t" // r1>>63:r0>>63 - "pslldq $8, %%xmm3 \n\t" // r0>>63:0 - "pslldq $8, %%xmm4 \n\t" // r2>>63:0 - "psrldq $8, %%xmm5 \n\t" // 0:r1>>63 - "por %%xmm3, %%xmm1 \n\t" // r1<<1|r0>>63:r0<<1 - "por %%xmm4, %%xmm2 \n\t" // r3<<1|r2>>62:r2<<1 - "por %%xmm5, %%xmm2 \n\t" // r3<<1|r2>>62:r2<<1|r1>>63 + "movdqa %1, %%xmm3 \n\t" // r1:r0 + "movdqa %%xmm2, %%xmm4 \n\t" // r3:r2 + "psllq $1, %1 \n\t" // r1<<1:r0<<1 + "psllq $1, %%xmm2 \n\t" // r3<<1:r2<<1 + "psrlq $63, %%xmm3 \n\t" // r1>>63:r0>>63 + "psrlq $63, %%xmm4 \n\t" // r3>>63:r2>>63 + "movdqa %%xmm3, %%xmm5 \n\t" // r1>>63:r0>>63 + "pslldq $8, %%xmm3 \n\t" // r0>>63:0 + "pslldq $8, %%xmm4 \n\t" // r2>>63:0 + "psrldq $8, %%xmm5 \n\t" // 0:r1>>63 + "por %%xmm3, %1 \n\t" // r1<<1|r0>>63:r0<<1 + "por %%xmm4, %%xmm2 \n\t" // r3<<1|r2>>62:r2<<1 + "por %%xmm5, %%xmm2 \n\t" // r3<<1|r2>>62:r2<<1|r1>>63 /* * Now reduce modulo the GCM polynomial x^128 + x^7 + x^2 + x + 1 @@ -170,57 +183,61 @@ void mbedtls_aesni_gcm_mult( unsigned char c[16], * Currently xmm2:xmm1 holds x3:x2:x1:x0 (already shifted). */ /* Step 2 (1) */ - "movdqa %%xmm1, %%xmm3 \n\t" // x1:x0 - "movdqa %%xmm1, %%xmm4 \n\t" // same - "movdqa %%xmm1, %%xmm5 \n\t" // same - "psllq $63, %%xmm3 \n\t" // x1<<63:x0<<63 = stuff:a - "psllq $62, %%xmm4 \n\t" // x1<<62:x0<<62 = stuff:b - "psllq $57, %%xmm5 \n\t" // x1<<57:x0<<57 = stuff:c + "movdqa %1, %%xmm3 \n\t" // x1:x0 + "movdqa %1, %%xmm4 \n\t" // same + "movdqa %1, %%xmm5 \n\t" // same + "psllq $63, %%xmm3 \n\t" // x1<<63:x0<<63 = stuff:a + "psllq $62, %%xmm4 \n\t" // x1<<62:x0<<62 = stuff:b + "psllq $57, %%xmm5 \n\t" // x1<<57:x0<<57 = stuff:c /* Step 2 (2) */ - "pxor %%xmm4, %%xmm3 \n\t" // stuff:a+b - "pxor %%xmm5, %%xmm3 \n\t" // stuff:a+b+c - "pslldq $8, %%xmm3 \n\t" // a+b+c:0 - "pxor %%xmm3, %%xmm1 \n\t" // x1+a+b+c:x0 = d:x0 + "pxor %%xmm4, %%xmm3 \n\t" // stuff:a+b + "pxor %%xmm5, %%xmm3 \n\t" // stuff:a+b+c + "pslldq $8, %%xmm3 \n\t" // a+b+c:0 + "pxor %%xmm3, %1 \n\t" // x1+a+b+c:x0 = d:x0 /* Steps 3 and 4 */ - "movdqa %%xmm1,%%xmm0 \n\t" // d:x0 - "movdqa %%xmm1,%%xmm4 \n\t" // same - "movdqa %%xmm1,%%xmm5 \n\t" // same - "psrlq $1, %%xmm0 \n\t" // e1:x0>>1 = e1:e0' - "psrlq $2, %%xmm4 \n\t" // f1:x0>>2 = f1:f0' - "psrlq $7, %%xmm5 \n\t" // g1:x0>>7 = g1:g0' - "pxor %%xmm4, %%xmm0 \n\t" // e1+f1:e0'+f0' - "pxor %%xmm5, %%xmm0 \n\t" // e1+f1+g1:e0'+f0'+g0' + "movdqa %1,%0 \n\t" // d:x0 + "movdqa %1,%%xmm4 \n\t" // same + "movdqa %1,%%xmm5 \n\t" // same + "psrlq $1, %0 \n\t" // e1:x0>>1 = e1:e0' + "psrlq $2, %%xmm4 \n\t" // f1:x0>>2 = f1:f0' + "psrlq $7, %%xmm5 \n\t" // g1:x0>>7 = g1:g0' + "pxor %%xmm4, %0 \n\t" // e1+f1:e0'+f0' + "pxor %%xmm5, %0 \n\t" // e1+f1+g1:e0'+f0'+g0' // e0'+f0'+g0' is almost e0+f0+g0, ex\tcept for some missing // bits carried from d. Now get those\t bits back in. - "movdqa %%xmm1,%%xmm3 \n\t" // d:x0 - "movdqa %%xmm1,%%xmm4 \n\t" // same - "movdqa %%xmm1,%%xmm5 \n\t" // same - "psllq $63, %%xmm3 \n\t" // d<<63:stuff - "psllq $62, %%xmm4 \n\t" // d<<62:stuff - "psllq $57, %%xmm5 \n\t" // d<<57:stuff - "pxor %%xmm4, %%xmm3 \n\t" // d<<63+d<<62:stuff - "pxor %%xmm5, %%xmm3 \n\t" // missing bits of d:stuff - "psrldq $8, %%xmm3 \n\t" // 0:missing bits of d - "pxor %%xmm3, %%xmm0 \n\t" // e1+f1+g1:e0+f0+g0 - "pxor %%xmm1, %%xmm0 \n\t" // h1:h0 - "pxor %%xmm2, %%xmm0 \n\t" // x3+h1:x2+h0 - - "movdqu %%xmm0, (%2) \n\t" // done - : - : "r" (aa), "r" (bb), "r" (cc) - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" ); + "movdqa %1,%%xmm3 \n\t" // d:x0 + "movdqa %1,%%xmm4 \n\t" // same + "movdqa %1,%%xmm5 \n\t" // same + "psllq $63, %%xmm3 \n\t" // d<<63:stuff + "psllq $62, %%xmm4 \n\t" // d<<62:stuff + "psllq $57, %%xmm5 \n\t" // d<<57:stuff + "pxor %%xmm4, %%xmm3 \n\t" // d<<63+d<<62:stuff + "pxor %%xmm5, %%xmm3 \n\t" // missing bits of d:stuff + "psrldq $8, %%xmm3 \n\t" // 0:missing bits of d + "pxor %%xmm3, %0 \n\t" // e1+f1+g1:e0+f0+g0 + "pxor %1, %0 \n\t" // h1:h0 + "pxor %%xmm2, %0 \n\t" // x3+h1:x2+h0 + : "+x" (aa) + : "x" (bb) + : "xmm2", "xmm3", "xmm4", "xmm5" ); /* Now byte-reverse the outputs */ - for( i = 0; i < 16; i++ ) - c[i] = cc[15 - i]; - - return; + WRITE64BE(a+0, aa[1]); + WRITE64BE(a+8, aa[0]); } -/* - * Compute decryption round keys from encryption round keys +/** + * \brief Internal round key inversion. This function computes + * decryption round keys from the encryption round keys. + * + * \note This function is only for internal use by other library + * functions; you must not call it directly. + * + * \param invkey Round keys for the equivalent inverse cipher + * \param fwdkey Original round keys (for encryption) + * \param nr Number of rounds (that is, number of round keys minus one) */ void mbedtls_aesni_inverse_key( unsigned char *invkey, const unsigned char *fwdkey, int nr ) @@ -414,8 +431,17 @@ static void aesni_setkey_enc_256( unsigned char *rk, : "memory", "cc", "0" ); } -/* - * Key expansion, wrapper +/** + * \brief Internal key expansion for encryption + * + * \note This function is only for internal use by other library + * functions; you must not call it directly. + * + * \param rk Destination buffer where the round keys are written + * \param key Encryption key + * \param bits Key size in bits (must be 128, 192 or 256) + * + * \return 0 if successful, or MBEDTLS_ERR_AES_INVALID_KEY_LENGTH */ int mbedtls_aesni_setkey_enc( unsigned char *rk, const unsigned char *key, @@ -428,7 +454,6 @@ int mbedtls_aesni_setkey_enc( unsigned char *rk, case 256: aesni_setkey_enc_256( rk, key ); break; default : return( MBEDTLS_ERR_AES_INVALID_KEY_LENGTH ); } - return( 0 ); } diff --git a/third_party/mbedtls/aesni.h b/third_party/mbedtls/aesni.h index e7bc4f69c..1ba509dde 100644 --- a/third_party/mbedtls/aesni.h +++ b/third_party/mbedtls/aesni.h @@ -1,96 +1,17 @@ -#ifndef MBEDTLS_AESNI_H -#define MBEDTLS_AESNI_H +#ifndef MBEDTLS_AESNI_H_ +#define MBEDTLS_AESNI_H_ #include "third_party/mbedtls/aes.h" #include "third_party/mbedtls/config.h" +COSMOPOLITAN_C_START_ /* clang-format off */ #define MBEDTLS_AESNI_AES 0x02000000u #define MBEDTLS_AESNI_CLMUL 0x00000002u -#ifdef __cplusplus -extern "C" { -#endif +int mbedtls_aesni_crypt_ecb( mbedtls_aes_context *, int, const unsigned char[16], unsigned char[16] ); +void mbedtls_aesni_gcm_mult( unsigned char[16], const uint64_t[2] ); +void mbedtls_aesni_inverse_key( unsigned char *, const unsigned char *, int ); +int mbedtls_aesni_setkey_enc( unsigned char *, const unsigned char *, size_t ); -/** - * \brief Internal function to detect the AES-NI feature in CPUs. - * - * \note This function is only for internal use by other library - * functions; you must not call it directly. - * - * \param what The feature to detect - * (MBEDTLS_AESNI_AES or MBEDTLS_AESNI_CLMUL) - * - * \return 1 if CPU has support for the feature, 0 otherwise - */ -int mbedtls_aesni_has_support( unsigned int what ); - -/** - * \brief Internal AES-NI AES-ECB block encryption and decryption - * - * \note This function is only for internal use by other library - * functions; you must not call it directly. - * - * \param ctx AES context - * \param mode MBEDTLS_AES_ENCRYPT or MBEDTLS_AES_DECRYPT - * \param input 16-byte input block - * \param output 16-byte output block - * - * \return 0 on success (cannot fail) - */ -int mbedtls_aesni_crypt_ecb( mbedtls_aes_context *ctx, - int mode, - const unsigned char input[16], - unsigned char output[16] ); - -/** - * \brief Internal GCM multiplication: c = a * b in GF(2^128) - * - * \note This function is only for internal use by other library - * functions; you must not call it directly. - * - * \param c Result - * \param a First operand - * \param b Second operand - * - * \note Both operands and result are bit strings interpreted as - * elements of GF(2^128) as per the GCM spec. - */ -void mbedtls_aesni_gcm_mult( unsigned char c[16], - const unsigned char a[16], - const unsigned char b[16] ); - -/** - * \brief Internal round key inversion. This function computes - * decryption round keys from the encryption round keys. - * - * \note This function is only for internal use by other library - * functions; you must not call it directly. - * - * \param invkey Round keys for the equivalent inverse cipher - * \param fwdkey Original round keys (for encryption) - * \param nr Number of rounds (that is, number of round keys minus one) - */ -void mbedtls_aesni_inverse_key( unsigned char *invkey, - const unsigned char *fwdkey, - int nr ); - -/** - * \brief Internal key expansion for encryption - * - * \note This function is only for internal use by other library - * functions; you must not call it directly. - * - * \param rk Destination buffer where the round keys are written - * \param key Encryption key - * \param bits Key size in bits (must be 128, 192 or 256) - * - * \return 0 if successful, or MBEDTLS_ERR_AES_INVALID_KEY_LENGTH - */ -int mbedtls_aesni_setkey_enc( unsigned char *rk, - const unsigned char *key, - size_t bits ); - -#ifdef __cplusplus -} -#endif -#endif /* MBEDTLS_AESNI_H */ +COSMOPOLITAN_C_END_ +#endif /* MBEDTLS_AESNI_H_ */ diff --git a/third_party/mbedtls/base64.c b/third_party/mbedtls/base64.c index 32f43dd47..f7a32becd 100644 --- a/third_party/mbedtls/base64.c +++ b/third_party/mbedtls/base64.c @@ -93,18 +93,18 @@ forceinline unsigned char mbedtls_base64_eq( size_t in_a, size_t in_b ) /* * Constant flow lookup into table. */ -static unsigned char mbedtls_base64_table_lookup( const unsigned char * const table, - const size_t table_size, - const size_t table_index ) +static inline unsigned char mbedtls_base64_table_lookup( const unsigned char * const table, + const size_t table_size, + const size_t table_index ) { + return 0 <= table_index && table_index < table_size ? table[table_index] : 127; + /* come on really? */ size_t i; unsigned char result = 0; - for( i = 0; i < table_size; ++i ) { mbedtls_base64_cond_assign_uchar( &result, &table[i], mbedtls_base64_eq( i, table_index ) ); } - return result; } diff --git a/third_party/mbedtls/cipher.c b/third_party/mbedtls/cipher.c index 5f5ee84a5..7052c819a 100644 --- a/third_party/mbedtls/cipher.c +++ b/third_party/mbedtls/cipher.c @@ -469,7 +469,7 @@ int mbedtls_cipher_update_ad( mbedtls_cipher_context_t *ctx, #endif /* MBEDTLS_GCM_C || MBEDTLS_CHACHAPOLY_C */ int mbedtls_cipher_update( mbedtls_cipher_context_t *ctx, const unsigned char *input, - size_t ilen, unsigned char *output, size_t *olen ) + size_t ilen, unsigned char *output, size_t *olen ) { int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED; size_t block_size; @@ -481,16 +481,6 @@ int mbedtls_cipher_update( mbedtls_cipher_context_t *ctx, const unsigned char *i if( ctx->cipher_info == NULL ) return( MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA ); -#if defined(MBEDTLS_USE_PSA_CRYPTO) - if( ctx->psa_enabled == 1 ) - { - /* While PSA Crypto has an API for multipart - * operations, we currently don't make it - * accessible through the cipher layer. */ - return( MBEDTLS_ERR_CIPHER_FEATURE_UNAVAILABLE ); - } -#endif /* MBEDTLS_USE_PSA_CRYPTO */ - *olen = 0; block_size = mbedtls_cipher_get_block_size( ctx ); if ( 0 == block_size ) diff --git a/third_party/mbedtls/error.c b/third_party/mbedtls/error.c index 5a52af5ee..75cc6dc09 100644 --- a/third_party/mbedtls/error.c +++ b/third_party/mbedtls/error.c @@ -551,8 +551,6 @@ const char * mbedtls_low_level_strerr( int error_code ) #if defined(MBEDTLS_GCM_C) case -(MBEDTLS_ERR_GCM_AUTH_FAILED): return( "GCM - Authenticated decryption failed" ); - case -(MBEDTLS_ERR_GCM_HW_ACCEL_FAILED): - return( "GCM - GCM hardware accelerator failed" ); case -(MBEDTLS_ERR_GCM_BAD_INPUT): return( "GCM - Bad input parameters to function" ); #endif /* MBEDTLS_GCM_C */ diff --git a/third_party/mbedtls/gcm.c b/third_party/mbedtls/gcm.c index 0ed767cc2..a28cf7308 100644 --- a/third_party/mbedtls/gcm.c +++ b/third_party/mbedtls/gcm.c @@ -1,4 +1,7 @@ +#include "libc/bits/bits.h" +#include "libc/bits/likely.h" #include "libc/nexgen32e/x86feature.h" +#include "libc/runtime/runtime.h" #include "libc/str/str.h" #include "third_party/mbedtls/aes.h" #include "third_party/mbedtls/aesni.h" @@ -52,8 +55,16 @@ asm(".include \"libc/disclaimer.inc\""); #define GCM_VALIDATE( cond ) \ MBEDTLS_INTERNAL_VALIDATE( cond ) -/* - * Initialize a context +/** + * \brief This function initializes the specified GCM context, + * to make references valid, and prepares the context + * for mbedtls_gcm_setkey() or mbedtls_gcm_free(). + * + * The function does not bind the GCM context to a particular + * cipher, nor set the key. For this purpose, use + * mbedtls_gcm_setkey(). + * + * \param ctx The GCM context to initialize. This must not be \c NULL. */ void mbedtls_gcm_init( mbedtls_gcm_context *ctx ) { @@ -76,58 +87,60 @@ static int gcm_gen_table( mbedtls_gcm_context *ctx ) uint64_t vl, vh; unsigned char h[16]; size_t olen = 0; - memset( h, 0, 16 ); if( ( ret = mbedtls_cipher_update( &ctx->cipher_ctx, h, 16, h, &olen ) ) != 0 ) return( ret ); - - /* pack h as two 64-bits ints, big-endian */ - GET_UINT32_BE( hi, h, 0 ); - GET_UINT32_BE( lo, h, 4 ); - vh = (uint64_t) hi << 32 | lo; - - GET_UINT32_BE( hi, h, 8 ); - GET_UINT32_BE( lo, h, 12 ); - vl = (uint64_t) hi << 32 | lo; - - /* 8 = 1000 corresponds to 1 in GF(2^128) */ - ctx->HL[8] = vl; - ctx->HH[8] = vh; - + vh = READ64BE( h + 0 ); + vl = READ64BE( h + 8 ); #if defined(MBEDTLS_AESNI_C) && defined(MBEDTLS_HAVE_X86_64) /* With CLMUL support, we need only h, not the rest of the table */ - if (X86_HAVE(AES) && X86_HAVE(PCLMUL)) return 0; + if (X86_HAVE(AES) && X86_HAVE(PCLMUL)) { + ctx->H8[0] = vl; + ctx->H8[1] = vh; + return 0; + } #endif - /* 0 corresponds to 0 in GF(2^128) */ ctx->HH[0] = 0; ctx->HL[0] = 0; - - for( i = 4; i > 0; i >>= 1 ) - { + /* 8 = 1000 corresponds to 1 in GF(2^128) */ + ctx->HL[8] = vl; + ctx->HH[8] = vh; + for( i = 4; i > 0; i >>= 1 ) { uint32_t T = ( vl & 1 ) * 0xe1000000U; vl = ( vh << 63 ) | ( vl >> 1 ); vh = ( vh >> 1 ) ^ ( (uint64_t) T << 32); - ctx->HL[i] = vl; ctx->HH[i] = vh; } - - for( i = 2; i <= 8; i *= 2 ) - { + for( i = 2; i <= 8; i *= 2 ) { uint64_t *HiL = ctx->HL + i, *HiH = ctx->HH + i; vh = *HiH; vl = *HiL; - for( j = 1; j < i; j++ ) - { + for( j = 1; j < i; j++ ) { HiH[j] = vh ^ ctx->HH[j]; HiL[j] = vl ^ ctx->HL[j]; } } - return( 0 ); } +/** + * \brief This function associates a GCM context with a + * cipher algorithm and a key. + * + * \param ctx The GCM context. This must be initialized. + * \param cipher The 128-bit block cipher to use. + * \param key The encryption key. This must be a readable buffer of at + * least \p keybits bits. + * \param keybits The key size in bits. Valid options are: + * + * + * \return \c 0 on success. + * \return A cipher-specific error code on failure. + */ int mbedtls_gcm_setkey( mbedtls_gcm_context *ctx, mbedtls_cipher_id_t cipher, const unsigned char *key, @@ -135,33 +148,24 @@ int mbedtls_gcm_setkey( mbedtls_gcm_context *ctx, { int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED; const mbedtls_cipher_info_t *cipher_info; - GCM_VALIDATE_RET( ctx != NULL ); GCM_VALIDATE_RET( key != NULL ); GCM_VALIDATE_RET( keybits == 128 || keybits == 192 || keybits == 256 ); - cipher_info = mbedtls_cipher_info_from_values( cipher, keybits, MBEDTLS_MODE_ECB ); if( cipher_info == NULL ) return( MBEDTLS_ERR_GCM_BAD_INPUT ); - if( cipher_info->block_size != 16 ) return( MBEDTLS_ERR_GCM_BAD_INPUT ); - mbedtls_cipher_free( &ctx->cipher_ctx ); - if( ( ret = mbedtls_cipher_setup( &ctx->cipher_ctx, cipher_info ) ) != 0 ) return( ret ); - if( ( ret = mbedtls_cipher_setkey( &ctx->cipher_ctx, key, keybits, - MBEDTLS_ENCRYPT ) ) != 0 ) - { + MBEDTLS_ENCRYPT ) ) != 0 ) { return( ret ); } - if( ( ret = gcm_gen_table( ctx ) ) != 0 ) return( ret ); - return( 0 ); } @@ -182,48 +186,31 @@ static const uint64_t last4[16] = * Sets output to x times H using the precomputed tables. * x and output are seen as elements of GF(2^128) as in [MGV]. */ -static void gcm_mult( mbedtls_gcm_context *ctx, const unsigned char x[16], - unsigned char output[16] ) +static void gcm_mult( mbedtls_gcm_context *ctx, unsigned char x[16] ) { - int i = 0; - unsigned char lo, hi, rem; + int i; uint64_t zh, zl; - + unsigned char lo, hi, rem; #if defined(MBEDTLS_AESNI_C) && defined(MBEDTLS_HAVE_X86_64) - if (X86_HAVE(AES) && X86_HAVE(PCLMUL)) { - unsigned char h[16]; - - PUT_UINT32_BE( ctx->HH[8] >> 32, h, 0 ); - PUT_UINT32_BE( ctx->HH[8], h, 4 ); - PUT_UINT32_BE( ctx->HL[8] >> 32, h, 8 ); - PUT_UINT32_BE( ctx->HL[8], h, 12 ); - - mbedtls_aesni_gcm_mult( output, x, h ); + if (LIKELY(X86_HAVE(AES) && X86_HAVE(PCLMUL))) { + mbedtls_aesni_gcm_mult( x, ctx->H8 ); return; } #endif /* MBEDTLS_AESNI_C && MBEDTLS_HAVE_X86_64 */ - lo = x[15] & 0xf; - zh = ctx->HH[lo]; zl = ctx->HL[lo]; - - for( i = 15; i >= 0; i-- ) - { + for( i = 15; i >= 0; i-- ) { lo = x[i] & 0xf; hi = ( x[i] >> 4 ) & 0xf; - - if( i != 15 ) - { + if( i != 15 ) { rem = (unsigned char) zl & 0xf; zl = ( zh << 60 ) | ( zl >> 4 ); zh = ( zh >> 4 ); zh ^= (uint64_t) last4[rem] << 48; zh ^= ctx->HH[lo]; zl ^= ctx->HL[lo]; - } - rem = (unsigned char) zl & 0xf; zl = ( zh << 60 ) | ( zl >> 4 ); zh = ( zh >> 4 ); @@ -231,13 +218,27 @@ static void gcm_mult( mbedtls_gcm_context *ctx, const unsigned char x[16], zh ^= ctx->HH[hi]; zl ^= ctx->HL[hi]; } - - PUT_UINT32_BE( zh >> 32, output, 0 ); - PUT_UINT32_BE( zh, output, 4 ); - PUT_UINT32_BE( zl >> 32, output, 8 ); - PUT_UINT32_BE( zl, output, 12 ); + PUT_UINT64_BE( zh, x, 0 ); + PUT_UINT64_BE( zl, x, 8 ); } +/** + * \brief This function starts a GCM encryption or decryption + * operation. + * + * \param ctx The GCM context. This must be initialized. + * \param mode The operation to perform: #MBEDTLS_GCM_ENCRYPT or + * #MBEDTLS_GCM_DECRYPT. + * \param iv The initialization vector. This must be a readable buffer of + * at least \p iv_len Bytes. + * \param iv_len The length of the IV. + * \param add The buffer holding the additional data, or \c NULL + * if \p add_len is \c 0. + * \param add_len The length of the additional data. If \c 0, + * \p add may be \c NULL. + * + * \return \c 0 on success. + */ int mbedtls_gcm_starts( mbedtls_gcm_context *ctx, int mode, const unsigned char *iv, @@ -245,190 +246,277 @@ int mbedtls_gcm_starts( mbedtls_gcm_context *ctx, const unsigned char *add, size_t add_len ) { - int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED; - unsigned char work_buf[16]; size_t i; const unsigned char *p; size_t use_len, olen = 0; - + unsigned char work_buf[16]; + int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED; GCM_VALIDATE_RET( ctx != NULL ); GCM_VALIDATE_RET( iv != NULL ); GCM_VALIDATE_RET( add_len == 0 || add != NULL ); - /* IV and AD are limited to 2^64 bits, so 2^61 bytes */ /* IV is not allowed to be zero length */ if( iv_len == 0 || ( (uint64_t) iv_len ) >> 61 != 0 || - ( (uint64_t) add_len ) >> 61 != 0 ) - { + ( (uint64_t) add_len ) >> 61 != 0 ) { return( MBEDTLS_ERR_GCM_BAD_INPUT ); } - memset( ctx->y, 0x00, sizeof(ctx->y) ); memset( ctx->buf, 0x00, sizeof(ctx->buf) ); - ctx->mode = mode; ctx->len = 0; ctx->add_len = 0; - - if( iv_len == 12 ) - { + if( iv_len == 12 ) { memcpy( ctx->y, iv, iv_len ); ctx->y[15] = 1; - } - else - { + } else { memset( work_buf, 0x00, 16 ); PUT_UINT32_BE( iv_len * 8, work_buf, 12 ); - p = iv; - while( iv_len > 0 ) - { + while( iv_len > 0 ) { use_len = ( iv_len < 16 ) ? iv_len : 16; - for( i = 0; i < use_len; i++ ) ctx->y[i] ^= p[i]; - - gcm_mult( ctx, ctx->y, ctx->y ); - + gcm_mult( ctx, ctx->y ); iv_len -= use_len; p += use_len; } - for( i = 0; i < 16; i++ ) ctx->y[i] ^= work_buf[i]; - - gcm_mult( ctx, ctx->y, ctx->y ); + gcm_mult( ctx, ctx->y ); } - if( ( ret = mbedtls_cipher_update( &ctx->cipher_ctx, ctx->y, 16, - ctx->base_ectr, &olen ) ) != 0 ) - { + ctx->base_ectr, &olen ) ) != 0 ) { return( ret ); } - ctx->add_len = add_len; p = add; - while( add_len > 0 ) - { + while( add_len > 0 ) { use_len = ( add_len < 16 ) ? add_len : 16; - for( i = 0; i < use_len; i++ ) ctx->buf[i] ^= p[i]; - - gcm_mult( ctx, ctx->buf, ctx->buf ); - + gcm_mult( ctx, ctx->buf ); add_len -= use_len; p += use_len; } - return( 0 ); } +/** + * \brief This function feeds an input buffer into an ongoing GCM + * encryption or decryption operation. + * + * The function expects input to be a multiple of 16 + * Bytes. Only the last call before calling + * mbedtls_gcm_finish() can be less than 16 Bytes. + * + * \note For decryption, the output buffer cannot be the same as + * input buffer. If the buffers overlap, the output buffer + * must trail at least 8 Bytes behind the input buffer. + * + * \param ctx The GCM context. This must be initialized. + * \param length The length of the input data. This must be a multiple of + * 16 except in the last call before mbedtls_gcm_finish(). + * \param input The buffer holding the input data. If \p length is greater + * than zero, this must be a readable buffer of at least that + * size in Bytes. + * \param output The buffer for holding the output data. If \p length is + * greater than zero, this must be a writable buffer of at + * least that size in Bytes. + * + * \return \c 0 on success. + * \return #MBEDTLS_ERR_GCM_BAD_INPUT on failure. + */ int mbedtls_gcm_update( mbedtls_gcm_context *ctx, size_t length, const unsigned char *input, unsigned char *output ) { + size_t i, j; + uint64_t a, b; int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED; unsigned char ectr[16]; - size_t i; const unsigned char *p; - unsigned char *out_p = output; - size_t use_len, olen = 0; - + unsigned char *q, *out_p = output; + size_t olen = 0; GCM_VALIDATE_RET( ctx != NULL ); GCM_VALIDATE_RET( length == 0 || input != NULL ); GCM_VALIDATE_RET( length == 0 || output != NULL ); - if( output > input && (size_t) ( output - input ) < length ) return( MBEDTLS_ERR_GCM_BAD_INPUT ); - /* Total length is restricted to 2^39 - 256 bits, ie 2^36 - 2^5 bytes * Also check for possible overflow */ if( ctx->len + length < ctx->len || - (uint64_t) ctx->len + length > 0xFFFFFFFE0ull ) - { + (uint64_t) ctx->len + length > 0xFFFFFFFE0ull ) { return( MBEDTLS_ERR_GCM_BAD_INPUT ); } - ctx->len += length; - p = input; - while( length > 0 ) - { - use_len = ( length < 16 ) ? length : 16; - + q = ctx->buf; + for( j = 0; j + 16 <= length; j += 16 ){ for( i = 16; i > 12; i-- ) if( ++ctx->y[i - 1] != 0 ) break; - - if( ( ret = mbedtls_cipher_update( &ctx->cipher_ctx, ctx->y, 16, ectr, - &olen ) ) != 0 ) - { + if( !( ret = mbedtls_cipher_update( &ctx->cipher_ctx, ctx->y, 16, + ectr, &olen ) ) ) { + if( ctx->mode == MBEDTLS_GCM_DECRYPT ) { + __builtin_memcpy(&a, p+j, 8); + __builtin_memcpy(&b, q, 8); + b ^= a; + __builtin_memcpy(q, &b, 8); + __builtin_memcpy(&b, ectr, 8); + b ^= a; + __builtin_memcpy(out_p+j, &b, 8); + __builtin_memcpy(&a, p+j+8, 8); + __builtin_memcpy(&b, q+8, 8); + b ^= a; + __builtin_memcpy(q+8, &b, 8); + __builtin_memcpy(&b, ectr+8, 8); + b ^= a; + __builtin_memcpy(out_p+j+8, &b, 8); + /* for( i = 0; i < 16; i++ ) ctx->buf[i] ^= p[i]; */ + /* for( i = 0; i < 16; i++ ) out_p[i] = ectr[i] ^ p[i]; */ + } else { + __builtin_memcpy(&a, ectr, 8); + __builtin_memcpy(&b, p+j, 8); + b ^= a; + __builtin_memcpy(out_p+j, &b, 8); + __builtin_memcpy(&a, q, 8); + b ^= a; + __builtin_memcpy(q, &b, 8); + __builtin_memcpy(&a, ectr+8, 8); + __builtin_memcpy(&b, p+j+8, 8); + b ^= a; + __builtin_memcpy(out_p+j+8, &b, 8); + __builtin_memcpy(&a, q+8, 8); + b ^= a; + __builtin_memcpy(q+8, &b, 8); + /* for( i = 0; i < 16; i++ ) out_p[i] = ectr[i] ^ p[i]; */ + /* for( i = 0; i < 16; i++ ) ctx->buf[i] ^= out_p[i]; */ + } + gcm_mult( ctx, q ); + } else { + return( ret ); + } + } + length -= j; + out_p += j; + p += j; + if( length ) { + for( i = 16; i > 12; i-- ) + if( ++ctx->y[i - 1] != 0 ) + break; + if( !( ret = mbedtls_cipher_update( &ctx->cipher_ctx, ctx->y, 16, ectr, + &olen ) ) ) { + if( ctx->mode == MBEDTLS_GCM_DECRYPT ) { + for( i = 0; i < length; i++ ){ + q[i] ^= p[i]; + out_p[i] = ectr[i] ^ p[i]; + } + } else { + for( i = 0; i < length; i++ ){ + out_p[i] = ectr[i] ^ p[i]; + q[i] ^= out_p[i]; + } + } + gcm_mult( ctx, q ); + } else { return( ret ); } - - for( i = 0; i < use_len; i++ ) - { - if( ctx->mode == MBEDTLS_GCM_DECRYPT ) - ctx->buf[i] ^= p[i]; - out_p[i] = ectr[i] ^ p[i]; - if( ctx->mode == MBEDTLS_GCM_ENCRYPT ) - ctx->buf[i] ^= out_p[i]; - } - - gcm_mult( ctx, ctx->buf, ctx->buf ); - - length -= use_len; - p += use_len; - out_p += use_len; } - return( 0 ); } +/** + * \brief This function finishes the GCM operation and generates + * the authentication tag. + * + * It wraps up the GCM stream, and generates the + * tag. The tag can have a maximum length of 16 Bytes. + * + * \param ctx The GCM context. This must be initialized. + * \param tag The buffer for holding the tag. This must be a writable + * buffer of at least \p tag_len Bytes. + * \param tag_len The length of the tag to generate. This must be at least + * four. + * + * \return \c 0 on success. + * \return #MBEDTLS_ERR_GCM_BAD_INPUT on failure. + */ int mbedtls_gcm_finish( mbedtls_gcm_context *ctx, unsigned char *tag, size_t tag_len ) { - unsigned char work_buf[16]; size_t i; uint64_t orig_len; uint64_t orig_add_len; - GCM_VALIDATE_RET( ctx != NULL ); GCM_VALIDATE_RET( tag != NULL ); - orig_len = ctx->len * 8; orig_add_len = ctx->add_len * 8; - if( tag_len > 16 || tag_len < 4 ) return( MBEDTLS_ERR_GCM_BAD_INPUT ); - memcpy( tag, ctx->base_ectr, tag_len ); - - if( orig_len || orig_add_len ) - { - memset( work_buf, 0x00, 16 ); - - PUT_UINT32_BE( ( orig_add_len >> 32 ), work_buf, 0 ); - PUT_UINT32_BE( ( orig_add_len ), work_buf, 4 ); - PUT_UINT32_BE( ( orig_len >> 32 ), work_buf, 8 ); - PUT_UINT32_BE( ( orig_len ), work_buf, 12 ); - - for( i = 0; i < 16; i++ ) - ctx->buf[i] ^= work_buf[i]; - - gcm_mult( ctx, ctx->buf, ctx->buf ); - - for( i = 0; i < tag_len; i++ ) - tag[i] ^= ctx->buf[i]; + if( orig_len || orig_add_len ) { + Write64be( ctx->buf + 0, READ64BE( ctx->buf + 0 ) ^ orig_add_len ); + Write64be( ctx->buf + 8, READ64BE( ctx->buf + 8 ) ^ orig_len ); + gcm_mult( ctx, ctx->buf ); + for( i = 0; i < tag_len; i++ ) tag[i] ^= ctx->buf[i]; } - return( 0 ); } +/** + * \brief This function performs GCM encryption or decryption of a buffer. + * + * \note For encryption, the output buffer can be the same as the + * input buffer. For decryption, the output buffer cannot be + * the same as input buffer. If the buffers overlap, the output + * buffer must trail at least 8 Bytes behind the input buffer. + * + * \warning When this function performs a decryption, it outputs the + * authentication tag and does not verify that the data is + * authentic. You should use this function to perform encryption + * only. For decryption, use mbedtls_gcm_auth_decrypt() instead. + * + * \param ctx The GCM context to use for encryption or decryption. This + * must be initialized. + * \param mode The operation to perform: + * - #MBEDTLS_GCM_ENCRYPT to perform authenticated encryption. + * The ciphertext is written to \p output and the + * authentication tag is written to \p tag. + * - #MBEDTLS_GCM_DECRYPT to perform decryption. + * The plaintext is written to \p output and the + * authentication tag is written to \p tag. + * Note that this mode is not recommended, because it does + * not verify the authenticity of the data. For this reason, + * you should use mbedtls_gcm_auth_decrypt() instead of + * calling this function in decryption mode. + * \param length The length of the input data, which is equal to the length + * of the output data. + * \param iv The initialization vector. This must be a readable buffer of + * at least \p iv_len Bytes. + * \param iv_len The length of the IV. + * \param add The buffer holding the additional data. This must be of at + * least that size in Bytes. + * \param add_len The length of the additional data. + * \param input The buffer holding the input data. If \p length is greater + * than zero, this must be a readable buffer of at least that + * size in Bytes. + * \param output The buffer for holding the output data. If \p length is greater + * than zero, this must be a writable buffer of at least that + * size in Bytes. + * \param tag_len The length of the tag to generate. + * \param tag The buffer for holding the tag. This must be a writable + * buffer of at least \p tag_len Bytes. + * + * \return \c 0 if the encryption or decryption was performed + * successfully. Note that in #MBEDTLS_GCM_DECRYPT mode, + * this does not indicate that the data is authentic. + * \return #MBEDTLS_ERR_GCM_BAD_INPUT if the lengths or pointers are + * not valid or a cipher-specific error code if the encryption + * or decryption failed. + */ int mbedtls_gcm_crypt_and_tag( mbedtls_gcm_context *ctx, int mode, size_t length, @@ -442,26 +530,54 @@ int mbedtls_gcm_crypt_and_tag( mbedtls_gcm_context *ctx, unsigned char *tag ) { int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED; - GCM_VALIDATE_RET( ctx != NULL ); GCM_VALIDATE_RET( iv != NULL ); GCM_VALIDATE_RET( add_len == 0 || add != NULL ); GCM_VALIDATE_RET( length == 0 || input != NULL ); GCM_VALIDATE_RET( length == 0 || output != NULL ); GCM_VALIDATE_RET( tag != NULL ); - if( ( ret = mbedtls_gcm_starts( ctx, mode, iv, iv_len, add, add_len ) ) != 0 ) return( ret ); - if( ( ret = mbedtls_gcm_update( ctx, length, input, output ) ) != 0 ) return( ret ); - if( ( ret = mbedtls_gcm_finish( ctx, tag, tag_len ) ) != 0 ) return( ret ); - return( 0 ); } +/** + * \brief This function performs a GCM authenticated decryption of a + * buffer. + * + * \note For decryption, the output buffer cannot be the same as + * input buffer. If the buffers overlap, the output buffer + * must trail at least 8 Bytes behind the input buffer. + * + * \param ctx The GCM context. This must be initialized. + * \param length The length of the ciphertext to decrypt, which is also + * the length of the decrypted plaintext. + * \param iv The initialization vector. This must be a readable buffer + * of at least \p iv_len Bytes. + * \param iv_len The length of the IV. + * \param add The buffer holding the additional data. This must be of at + * least that size in Bytes. + * \param add_len The length of the additional data. + * \param tag The buffer holding the tag to verify. This must be a + * readable buffer of at least \p tag_len Bytes. + * \param tag_len The length of the tag to verify. + * \param input The buffer holding the ciphertext. If \p length is greater + * than zero, this must be a readable buffer of at least that + * size. + * \param output The buffer for holding the decrypted plaintext. If \p length + * is greater than zero, this must be a writable buffer of at + * least that size. + * + * \return \c 0 if successful and authenticated. + * \return #MBEDTLS_ERR_GCM_AUTH_FAILED if the tag does not match. + * \return #MBEDTLS_ERR_GCM_BAD_INPUT if the lengths or pointers are + * not valid or a cipher-specific error code if the decryption + * failed. + */ int mbedtls_gcm_auth_decrypt( mbedtls_gcm_context *ctx, size_t length, const unsigned char *iv, @@ -477,34 +593,34 @@ int mbedtls_gcm_auth_decrypt( mbedtls_gcm_context *ctx, unsigned char check_tag[16]; size_t i; int diff; - GCM_VALIDATE_RET( ctx != NULL ); GCM_VALIDATE_RET( iv != NULL ); GCM_VALIDATE_RET( add_len == 0 || add != NULL ); GCM_VALIDATE_RET( tag != NULL ); GCM_VALIDATE_RET( length == 0 || input != NULL ); GCM_VALIDATE_RET( length == 0 || output != NULL ); - if( ( ret = mbedtls_gcm_crypt_and_tag( ctx, MBEDTLS_GCM_DECRYPT, length, iv, iv_len, add, add_len, - input, output, tag_len, check_tag ) ) != 0 ) - { + input, output, tag_len, check_tag ) ) != 0 ) { return( ret ); } - /* Check tag in "constant-time" */ for( diff = 0, i = 0; i < tag_len; i++ ) diff |= tag[i] ^ check_tag[i]; - - if( diff != 0 ) - { + if( diff != 0 ) { mbedtls_platform_zeroize( output, length ); return( MBEDTLS_ERR_GCM_AUTH_FAILED ); } - return( 0 ); } +/** + * \brief This function clears a GCM context and the underlying + * cipher sub-context. + * + * \param ctx The GCM context to clear. If this is \c NULL, the call has + * no effect. Otherwise, this must be initialized. + */ void mbedtls_gcm_free( mbedtls_gcm_context *ctx ) { if( ctx == NULL ) @@ -743,6 +859,12 @@ static const unsigned char tag_test_data[MAX_TESTS * 3][16] = 0xc8, 0xb5, 0xd4, 0xcf, 0x5a, 0xe9, 0xf1, 0x9a }, }; +/** + * \brief The GCM checkup routine. + * + * \return \c 0 on success. + * \return \c 1 on failure. + */ int mbedtls_gcm_self_test( int verbose ) { mbedtls_gcm_context ctx; diff --git a/third_party/mbedtls/gcm.h b/third_party/mbedtls/gcm.h index a658c1344..181051f5d 100644 --- a/third_party/mbedtls/gcm.h +++ b/third_party/mbedtls/gcm.h @@ -1,287 +1,38 @@ -#ifndef MBEDTLS_GCM_H -#define MBEDTLS_GCM_H +#ifndef MBEDTLS_GCM_H_ +#define MBEDTLS_GCM_H_ #include "third_party/mbedtls/cipher.h" #include "third_party/mbedtls/config.h" +COSMOPOLITAN_C_START_ /* clang-format off */ #define MBEDTLS_GCM_ENCRYPT 1 #define MBEDTLS_GCM_DECRYPT 0 #define MBEDTLS_ERR_GCM_AUTH_FAILED -0x0012 /**< Authenticated decryption failed. */ - -/* MBEDTLS_ERR_GCM_HW_ACCEL_FAILED is deprecated and should not be used. */ -#define MBEDTLS_ERR_GCM_HW_ACCEL_FAILED -0x0013 /**< GCM hardware accelerator failed. */ - #define MBEDTLS_ERR_GCM_BAD_INPUT -0x0014 /**< Bad input parameters to function. */ -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_GCM_ALT) - -/** - * \brief The GCM context structure. - */ -typedef struct mbedtls_gcm_context -{ +typedef struct mbedtls_gcm_context { mbedtls_cipher_context_t cipher_ctx; /*!< The cipher context used. */ - uint64_t HL[16]; /*!< Precalculated HTable low. */ - uint64_t HH[16]; /*!< Precalculated HTable high. */ uint64_t len; /*!< The total length of the encrypted data. */ uint64_t add_len; /*!< The total length of the additional data. */ unsigned char base_ectr[16]; /*!< The first ECTR for tag. */ unsigned char y[16]; /*!< The Y working value. */ unsigned char buf[16]; /*!< The buf working value. */ - int mode; /*!< The operation to perform: - #MBEDTLS_GCM_ENCRYPT or - #MBEDTLS_GCM_DECRYPT. */ -} -mbedtls_gcm_context; + int mode; /*!< The operation to perform: #MBEDTLS_GCM_ENCRYPT or #MBEDTLS_GCM_DECRYPT. */ + uint64_t H8[2]; /*!< For AES-NI. */ + uint64_t HL[16]; /*!< Precalculated HTable low. */ + uint64_t HH[16]; /*!< Precalculated HTable high. */ +} mbedtls_gcm_context; -#else /* !MBEDTLS_GCM_ALT */ -/* #include "third_party/mbedtls/gcm_alt.h" */ -#endif /* !MBEDTLS_GCM_ALT */ +void mbedtls_gcm_init( mbedtls_gcm_context * ); +int mbedtls_gcm_setkey( mbedtls_gcm_context *, mbedtls_cipher_id_t, const unsigned char *, unsigned int ); +int mbedtls_gcm_crypt_and_tag( mbedtls_gcm_context *, int, size_t, const unsigned char *, size_t, const unsigned char *, size_t, const unsigned char *, unsigned char *, size_t, unsigned char * ); +int mbedtls_gcm_auth_decrypt( mbedtls_gcm_context *, size_t, const unsigned char *, size_t, const unsigned char *, size_t, const unsigned char *, size_t, const unsigned char *, unsigned char * ); +int mbedtls_gcm_starts( mbedtls_gcm_context *, int, const unsigned char *, size_t, const unsigned char *, size_t ); +int mbedtls_gcm_update( mbedtls_gcm_context *, size_t, const unsigned char *, unsigned char * ); +int mbedtls_gcm_finish( mbedtls_gcm_context *, unsigned char *, size_t ); +void mbedtls_gcm_free( mbedtls_gcm_context * ); +int mbedtls_gcm_self_test( int ); -/** - * \brief This function initializes the specified GCM context, - * to make references valid, and prepares the context - * for mbedtls_gcm_setkey() or mbedtls_gcm_free(). - * - * The function does not bind the GCM context to a particular - * cipher, nor set the key. For this purpose, use - * mbedtls_gcm_setkey(). - * - * \param ctx The GCM context to initialize. This must not be \c NULL. - */ -void mbedtls_gcm_init( mbedtls_gcm_context *ctx ); - -/** - * \brief This function associates a GCM context with a - * cipher algorithm and a key. - * - * \param ctx The GCM context. This must be initialized. - * \param cipher The 128-bit block cipher to use. - * \param key The encryption key. This must be a readable buffer of at - * least \p keybits bits. - * \param keybits The key size in bits. Valid options are: - * - * - * \return \c 0 on success. - * \return A cipher-specific error code on failure. - */ -int mbedtls_gcm_setkey( mbedtls_gcm_context *ctx, - mbedtls_cipher_id_t cipher, - const unsigned char *key, - unsigned int keybits ); - -/** - * \brief This function performs GCM encryption or decryption of a buffer. - * - * \note For encryption, the output buffer can be the same as the - * input buffer. For decryption, the output buffer cannot be - * the same as input buffer. If the buffers overlap, the output - * buffer must trail at least 8 Bytes behind the input buffer. - * - * \warning When this function performs a decryption, it outputs the - * authentication tag and does not verify that the data is - * authentic. You should use this function to perform encryption - * only. For decryption, use mbedtls_gcm_auth_decrypt() instead. - * - * \param ctx The GCM context to use for encryption or decryption. This - * must be initialized. - * \param mode The operation to perform: - * - #MBEDTLS_GCM_ENCRYPT to perform authenticated encryption. - * The ciphertext is written to \p output and the - * authentication tag is written to \p tag. - * - #MBEDTLS_GCM_DECRYPT to perform decryption. - * The plaintext is written to \p output and the - * authentication tag is written to \p tag. - * Note that this mode is not recommended, because it does - * not verify the authenticity of the data. For this reason, - * you should use mbedtls_gcm_auth_decrypt() instead of - * calling this function in decryption mode. - * \param length The length of the input data, which is equal to the length - * of the output data. - * \param iv The initialization vector. This must be a readable buffer of - * at least \p iv_len Bytes. - * \param iv_len The length of the IV. - * \param add The buffer holding the additional data. This must be of at - * least that size in Bytes. - * \param add_len The length of the additional data. - * \param input The buffer holding the input data. If \p length is greater - * than zero, this must be a readable buffer of at least that - * size in Bytes. - * \param output The buffer for holding the output data. If \p length is greater - * than zero, this must be a writable buffer of at least that - * size in Bytes. - * \param tag_len The length of the tag to generate. - * \param tag The buffer for holding the tag. This must be a writable - * buffer of at least \p tag_len Bytes. - * - * \return \c 0 if the encryption or decryption was performed - * successfully. Note that in #MBEDTLS_GCM_DECRYPT mode, - * this does not indicate that the data is authentic. - * \return #MBEDTLS_ERR_GCM_BAD_INPUT if the lengths or pointers are - * not valid or a cipher-specific error code if the encryption - * or decryption failed. - */ -int mbedtls_gcm_crypt_and_tag( mbedtls_gcm_context *ctx, - int mode, - size_t length, - const unsigned char *iv, - size_t iv_len, - const unsigned char *add, - size_t add_len, - const unsigned char *input, - unsigned char *output, - size_t tag_len, - unsigned char *tag ); - -/** - * \brief This function performs a GCM authenticated decryption of a - * buffer. - * - * \note For decryption, the output buffer cannot be the same as - * input buffer. If the buffers overlap, the output buffer - * must trail at least 8 Bytes behind the input buffer. - * - * \param ctx The GCM context. This must be initialized. - * \param length The length of the ciphertext to decrypt, which is also - * the length of the decrypted plaintext. - * \param iv The initialization vector. This must be a readable buffer - * of at least \p iv_len Bytes. - * \param iv_len The length of the IV. - * \param add The buffer holding the additional data. This must be of at - * least that size in Bytes. - * \param add_len The length of the additional data. - * \param tag The buffer holding the tag to verify. This must be a - * readable buffer of at least \p tag_len Bytes. - * \param tag_len The length of the tag to verify. - * \param input The buffer holding the ciphertext. If \p length is greater - * than zero, this must be a readable buffer of at least that - * size. - * \param output The buffer for holding the decrypted plaintext. If \p length - * is greater than zero, this must be a writable buffer of at - * least that size. - * - * \return \c 0 if successful and authenticated. - * \return #MBEDTLS_ERR_GCM_AUTH_FAILED if the tag does not match. - * \return #MBEDTLS_ERR_GCM_BAD_INPUT if the lengths or pointers are - * not valid or a cipher-specific error code if the decryption - * failed. - */ -int mbedtls_gcm_auth_decrypt( mbedtls_gcm_context *ctx, - size_t length, - const unsigned char *iv, - size_t iv_len, - const unsigned char *add, - size_t add_len, - const unsigned char *tag, - size_t tag_len, - const unsigned char *input, - unsigned char *output ); - -/** - * \brief This function starts a GCM encryption or decryption - * operation. - * - * \param ctx The GCM context. This must be initialized. - * \param mode The operation to perform: #MBEDTLS_GCM_ENCRYPT or - * #MBEDTLS_GCM_DECRYPT. - * \param iv The initialization vector. This must be a readable buffer of - * at least \p iv_len Bytes. - * \param iv_len The length of the IV. - * \param add The buffer holding the additional data, or \c NULL - * if \p add_len is \c 0. - * \param add_len The length of the additional data. If \c 0, - * \p add may be \c NULL. - * - * \return \c 0 on success. - */ -int mbedtls_gcm_starts( mbedtls_gcm_context *ctx, - int mode, - const unsigned char *iv, - size_t iv_len, - const unsigned char *add, - size_t add_len ); - -/** - * \brief This function feeds an input buffer into an ongoing GCM - * encryption or decryption operation. - * - * ` The function expects input to be a multiple of 16 - * Bytes. Only the last call before calling - * mbedtls_gcm_finish() can be less than 16 Bytes. - * - * \note For decryption, the output buffer cannot be the same as - * input buffer. If the buffers overlap, the output buffer - * must trail at least 8 Bytes behind the input buffer. - * - * \param ctx The GCM context. This must be initialized. - * \param length The length of the input data. This must be a multiple of - * 16 except in the last call before mbedtls_gcm_finish(). - * \param input The buffer holding the input data. If \p length is greater - * than zero, this must be a readable buffer of at least that - * size in Bytes. - * \param output The buffer for holding the output data. If \p length is - * greater than zero, this must be a writable buffer of at - * least that size in Bytes. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_GCM_BAD_INPUT on failure. - */ -int mbedtls_gcm_update( mbedtls_gcm_context *ctx, - size_t length, - const unsigned char *input, - unsigned char *output ); - -/** - * \brief This function finishes the GCM operation and generates - * the authentication tag. - * - * It wraps up the GCM stream, and generates the - * tag. The tag can have a maximum length of 16 Bytes. - * - * \param ctx The GCM context. This must be initialized. - * \param tag The buffer for holding the tag. This must be a writable - * buffer of at least \p tag_len Bytes. - * \param tag_len The length of the tag to generate. This must be at least - * four. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_GCM_BAD_INPUT on failure. - */ -int mbedtls_gcm_finish( mbedtls_gcm_context *ctx, - unsigned char *tag, - size_t tag_len ); - -/** - * \brief This function clears a GCM context and the underlying - * cipher sub-context. - * - * \param ctx The GCM context to clear. If this is \c NULL, the call has - * no effect. Otherwise, this must be initialized. - */ -void mbedtls_gcm_free( mbedtls_gcm_context *ctx ); - -#if defined(MBEDTLS_SELF_TEST) - -/** - * \brief The GCM checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_gcm_self_test( int verbose ); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - - -#endif /* gcm.h */ +COSMOPOLITAN_C_END_ +#endif /* MBEDTLS_GCM_H_ */ diff --git a/third_party/mbedtls/platform.h b/third_party/mbedtls/platform.h index d778c0e34..236f3c7e9 100644 --- a/third_party/mbedtls/platform.h +++ b/third_party/mbedtls/platform.h @@ -1,6 +1,7 @@ #ifndef COSMOPOLITAN_THIRD_PARTY_MBEDTLS_PLATFORM_H_ #define COSMOPOLITAN_THIRD_PARTY_MBEDTLS_PLATFORM_H_ #include "libc/assert.h" +#include "libc/bits/likely.h" #include "libc/calls/weirdtypes.h" #include "libc/fmt/fmt.h" #include "libc/mem/mem.h" @@ -34,7 +35,7 @@ COSMOPOLITAN_C_START_ #define MBEDTLS_INTERNAL_VALIDATE_RET(cond, ret) \ do { \ - if (!(cond)) { \ + if (UNLIKELY(!(cond))) { \ MBEDTLS_PARAM_FAILED(cond); \ return ret; \ } \ @@ -42,7 +43,7 @@ COSMOPOLITAN_C_START_ #define MBEDTLS_INTERNAL_VALIDATE(cond) \ do { \ - if (!(cond)) { \ + if (UNLIKELY(!(cond))) { \ MBEDTLS_PARAM_FAILED(cond); \ return; \ } \ diff --git a/third_party/mbedtls/sha1.c b/third_party/mbedtls/sha1.c index ed9439cae..14866676a 100644 --- a/third_party/mbedtls/sha1.c +++ b/third_party/mbedtls/sha1.c @@ -149,7 +149,7 @@ int mbedtls_internal_sha1_process( mbedtls_sha1_context *ctx, SHA1_VALIDATE_RET( ctx != NULL ); SHA1_VALIDATE_RET( (const unsigned char *)data != NULL ); - if (!IsTiny() && X86_HAVE(AVX2) && X86_HAVE(BMI) && X86_HAVE(BMI2)) { + if (!IsTiny() && LIKELY(X86_HAVE(AVX2) && X86_HAVE(BMI) && X86_HAVE(BMI2))) { sha1_transform_avx2(ctx, data, 1); return 0; }