mirrors/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2024-08-23 09:19:51 +00:00
Code Issues Releases Wiki Activity

crypto: aesni - Add GCM_COMPLETE macro

Browse source 
Merge encode and decode tag calculations in GCM_COMPLETE macro.
Scatter/gather routines will call this once at the end of encryption
or decryption.

Signed-off-by: Dave Watson <davejwatson@fb.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Dave Watson 2018-02-14 09:38:57 -08:00 committed by Herbert Xu
parent 7af964c2fc
commit adcadab369
1 changed files with 63 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

172
arch/x86/crypto/aesni-intel_asm.S
View file

@ -223,6 +223,67 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff
mov %r13, %r12
.endm
# GCM_COMPLETE Finishes update of tag of last partial block
# Output: Authorization Tag (AUTH_TAG)
# Clobbers rax, r10-r12, and xmm0, xmm1, xmm5-xmm15
.macro GCM_COMPLETE
mov arg8, %r12 # %r13 = aadLen (number of bytes)
shl $3, %r12 # convert into number of bits
movd %r12d, %xmm15 # len(A) in %xmm15
shl $3, %arg4 # len(C) in bits (*128)
MOVQ_R64_XMM %arg4, %xmm1
pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000
pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C)
pxor %xmm15, %xmm8
GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
# final GHASH computation
movdqa SHUF_MASK(%rip), %xmm10
PSHUFB_XMM %xmm10, %xmm8
mov %arg5, %rax # %rax = *Y0
movdqu (%rax), %xmm0 # %xmm0 = Y0
ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Y0)
pxor %xmm8, %xmm0
_return_T_\@:
mov arg9, %r10 # %r10 = authTag
mov arg10, %r11 # %r11 = auth_tag_len
cmp $16, %r11
je _T_16_\@
cmp $8, %r11
jl _T_4_\@
_T_8_\@:
MOVQ_R64_XMM %xmm0, %rax
mov %rax, (%r10)
add $8, %r10
sub $8, %r11
psrldq $8, %xmm0
cmp $0, %r11
je _return_T_done_\@
_T_4_\@:
movd %xmm0, %eax
mov %eax, (%r10)
add $4, %r10
sub $4, %r11
psrldq $4, %xmm0
cmp $0, %r11
je _return_T_done_\@
_T_123_\@:
movd %xmm0, %eax
cmp $2, %r11
jl _T_1_\@
mov %ax, (%r10)
cmp $2, %r11
je _return_T_done_\@
add $2, %r10
sar $16, %eax
_T_1_\@:
mov %al, (%r10)
jmp _return_T_done_\@
_T_16_\@:
movdqu %xmm0, (%r10)
_return_T_done_\@:
.endm
#ifdef __x86_64__
/* GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
*
@ -1272,61 +1333,7 @@ _less_than_8_bytes_left_decrypt:
sub $1, %r13
jne _less_than_8_bytes_left_decrypt
_multiple_of_16_bytes_decrypt:
mov arg8, %r12 # %r13 = aadLen (number of bytes)
shl $3, %r12 # convert into number of bits
movd %r12d, %xmm15 # len(A) in %xmm15
shl $3, %arg4 # len(C) in bits (*128)
MOVQ_R64_XMM %arg4, %xmm1
pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000
pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C)
pxor %xmm15, %xmm8
GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
# final GHASH computation
movdqa SHUF_MASK(%rip), %xmm10
PSHUFB_XMM %xmm10, %xmm8
mov %arg5, %rax # %rax = *Y0
movdqu (%rax), %xmm0 # %xmm0 = Y0
ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Y0)
pxor %xmm8, %xmm0
_return_T_decrypt:
mov arg9, %r10 # %r10 = authTag
mov arg10, %r11 # %r11 = auth_tag_len
cmp $16, %r11
je _T_16_decrypt
cmp $8, %r11
jl _T_4_decrypt
_T_8_decrypt:
MOVQ_R64_XMM %xmm0, %rax
mov %rax, (%r10)
add $8, %r10
sub $8, %r11
psrldq $8, %xmm0
cmp $0, %r11
je _return_T_done_decrypt
_T_4_decrypt:
movd %xmm0, %eax
mov %eax, (%r10)
add $4, %r10
sub $4, %r11
psrldq $4, %xmm0
cmp $0, %r11
je _return_T_done_decrypt
_T_123_decrypt:
movd %xmm0, %eax
cmp $2, %r11
jl _T_1_decrypt
mov %ax, (%r10)
cmp $2, %r11
je _return_T_done_decrypt
add $2, %r10
sar $16, %eax
_T_1_decrypt:
mov %al, (%r10)
jmp _return_T_done_decrypt
_T_16_decrypt:
movdqu %xmm0, (%r10)
_return_T_done_decrypt:
GCM_COMPLETE
FUNC_RESTORE
ret
ENDPROC(aesni_gcm_dec)
@ -1502,61 +1509,8 @@ _less_than_8_bytes_left_encrypt:
sub $1, %r13
jne _less_than_8_bytes_left_encrypt
_multiple_of_16_bytes_encrypt:
mov arg8, %r12 # %r12 = addLen (number of bytes)
shl $3, %r12
movd %r12d, %xmm15 # len(A) in %xmm15
shl $3, %arg4 # len(C) in bits (*128)
MOVQ_R64_XMM %arg4, %xmm1
pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000
pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C)
pxor %xmm15, %xmm8
GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
# final GHASH computation
movdqa SHUF_MASK(%rip), %xmm10
PSHUFB_XMM %xmm10, %xmm8 # perform a 16 byte swap
mov %arg5, %rax # %rax = *Y0
movdqu (%rax), %xmm0 # %xmm0 = Y0
ENCRYPT_SINGLE_BLOCK %xmm0, %xmm15 # Encrypt(K, Y0)
pxor %xmm8, %xmm0
_return_T_encrypt:
mov arg9, %r10 # %r10 = authTag
mov arg10, %r11 # %r11 = auth_tag_len
cmp $16, %r11
je _T_16_encrypt
cmp $8, %r11
jl _T_4_encrypt
_T_8_encrypt:
MOVQ_R64_XMM %xmm0, %rax
mov %rax, (%r10)
add $8, %r10
sub $8, %r11
psrldq $8, %xmm0
cmp $0, %r11
je _return_T_done_encrypt
_T_4_encrypt:
movd %xmm0, %eax
mov %eax, (%r10)
add $4, %r10
sub $4, %r11
psrldq $4, %xmm0
cmp $0, %r11
je _return_T_done_encrypt
_T_123_encrypt:
movd %xmm0, %eax
cmp $2, %r11
jl _T_1_encrypt
mov %ax, (%r10)
cmp $2, %r11
je _return_T_done_encrypt
add $2, %r10
sar $16, %eax
_T_1_encrypt:
mov %al, (%r10)
jmp _return_T_done_encrypt
_T_16_encrypt:
movdqu %xmm0, (%r10)
_return_T_done_encrypt:
GCM_COMPLETE
FUNC_RESTORE
ret
ENDPROC(aesni_gcm_enc)