503 lines
17 KiB
ArmAsm
503 lines
17 KiB
ArmAsm
|
// Copyright 2016 The Go Authors. All rights reserved.
|
||
|
|
// Use of this source code is governed by a BSD-style
|
||
|
|
// license that can be found in the LICENSE file.
|
||
|
|
|
||
|
|
// +build go1.7,amd64,!gccgo,!appengine
|
||
|
|
|
||
|
|
#include "textflag.h"
|
||
|
|
|
||
|
|
DATA ·AVX2_iv0<>+0x00(SB)/8, $0x6a09e667f3bcc908
|
||
|
|
DATA ·AVX2_iv0<>+0x08(SB)/8, $0xbb67ae8584caa73b
|
||
|
|
DATA ·AVX2_iv0<>+0x10(SB)/8, $0x3c6ef372fe94f82b
|
||
|
|
DATA ·AVX2_iv0<>+0x18(SB)/8, $0xa54ff53a5f1d36f1
|
||
|
|
GLOBL ·AVX2_iv0<>(SB), (NOPTR+RODATA), $32
|
||
|
|
|
||
|
|
DATA ·AVX2_iv1<>+0x00(SB)/8, $0x510e527fade682d1
|
||
|
|
DATA ·AVX2_iv1<>+0x08(SB)/8, $0x9b05688c2b3e6c1f
|
||
|
|
DATA ·AVX2_iv1<>+0x10(SB)/8, $0x1f83d9abfb41bd6b
|
||
|
|
DATA ·AVX2_iv1<>+0x18(SB)/8, $0x5be0cd19137e2179
|
||
|
|
GLOBL ·AVX2_iv1<>(SB), (NOPTR+RODATA), $32
|
||
|
|
|
||
|
|
DATA ·AVX2_c40<>+0x00(SB)/8, $0x0201000706050403
|
||
|
|
DATA ·AVX2_c40<>+0x08(SB)/8, $0x0a09080f0e0d0c0b
|
||
|
|
DATA ·AVX2_c40<>+0x10(SB)/8, $0x0201000706050403
|
||
|
|
DATA ·AVX2_c40<>+0x18(SB)/8, $0x0a09080f0e0d0c0b
|
||
|
|
GLOBL ·AVX2_c40<>(SB), (NOPTR+RODATA), $32
|
||
|
|
|
||
|
|
DATA ·AVX2_c48<>+0x00(SB)/8, $0x0100070605040302
|
||
|
|
DATA ·AVX2_c48<>+0x08(SB)/8, $0x09080f0e0d0c0b0a
|
||
|
|
DATA ·AVX2_c48<>+0x10(SB)/8, $0x0100070605040302
|
||
|
|
DATA ·AVX2_c48<>+0x18(SB)/8, $0x09080f0e0d0c0b0a
|
||
|
|
GLOBL ·AVX2_c48<>(SB), (NOPTR+RODATA), $32
|
||
|
|
|
||
|
|
DATA ·AVX_iv0<>+0x00(SB)/8, $0x6a09e667f3bcc908
|
||
|
|
DATA ·AVX_iv0<>+0x08(SB)/8, $0xbb67ae8584caa73b
|
||
|
|
GLOBL ·AVX_iv0<>(SB), (NOPTR+RODATA), $16
|
||
|
|
|
||
|
|
DATA ·AVX_iv1<>+0x00(SB)/8, $0x3c6ef372fe94f82b
|
||
|
|
DATA ·AVX_iv1<>+0x08(SB)/8, $0xa54ff53a5f1d36f1
|
||
|
|
GLOBL ·AVX_iv1<>(SB), (NOPTR+RODATA), $16
|
||
|
|
|
||
|
|
DATA ·AVX_iv2<>+0x00(SB)/8, $0x510e527fade682d1
|
||
|
|
DATA ·AVX_iv2<>+0x08(SB)/8, $0x9b05688c2b3e6c1f
|
||
|
|
GLOBL ·AVX_iv2<>(SB), (NOPTR+RODATA), $16
|
||
|
|
|
||
|
|
DATA ·AVX_iv3<>+0x00(SB)/8, $0x1f83d9abfb41bd6b
|
||
|
|
DATA ·AVX_iv3<>+0x08(SB)/8, $0x5be0cd19137e2179
|
||
|
|
GLOBL ·AVX_iv3<>(SB), (NOPTR+RODATA), $16
|
||
|
|
|
||
|
|
DATA ·AVX_c40<>+0x00(SB)/8, $0x0201000706050403
|
||
|
|
DATA ·AVX_c40<>+0x08(SB)/8, $0x0a09080f0e0d0c0b
|
||
|
|
GLOBL ·AVX_c40<>(SB), (NOPTR+RODATA), $16
|
||
|
|
|
||
|
|
DATA ·AVX_c48<>+0x00(SB)/8, $0x0100070605040302
|
||
|
|
DATA ·AVX_c48<>+0x08(SB)/8, $0x09080f0e0d0c0b0a
|
||
|
|
GLOBL ·AVX_c48<>(SB), (NOPTR+RODATA), $16
|
||
|
|
|
||
|
|
// unfortunately the BYTE representation of VPERMQ must be used
|
||
|
|
#define ROUND_AVX2(m0, m1, m2, m3, t, c40, c48) \
|
||
|
|
VPADDQ m0, Y0, Y0; \
|
||
|
|
VPADDQ Y1, Y0, Y0; \
|
||
|
|
VPXOR Y0, Y3, Y3; \
|
||
|
|
VPSHUFD $-79, Y3, Y3; \
|
||
|
|
VPADDQ Y3, Y2, Y2; \
|
||
|
|
VPXOR Y2, Y1, Y1; \
|
||
|
|
VPSHUFB c40, Y1, Y1; \
|
||
|
|
VPADDQ m1, Y0, Y0; \
|
||
|
|
VPADDQ Y1, Y0, Y0; \
|
||
|
|
VPXOR Y0, Y3, Y3; \
|
||
|
|
VPSHUFB c48, Y3, Y3; \
|
||
|
|
VPADDQ Y3, Y2, Y2; \
|
||
|
|
VPXOR Y2, Y1, Y1; \
|
||
|
|
VPADDQ Y1, Y1, t; \
|
||
|
|
VPSRLQ $63, Y1, Y1; \
|
||
|
|
VPXOR t, Y1, Y1; \
|
||
|
|
BYTE $0xc4; BYTE $0xe3; BYTE $0xfd; BYTE $0x00; BYTE $0xc9; BYTE $0x39 \ // VPERMQ 0x39, Y1, Y1
|
||
|
|
BYTE $0xc4; BYTE $0xe3; BYTE $0xfd; BYTE $0x00; BYTE $0xd2; BYTE $0x4e \ // VPERMQ 0x4e, Y2, Y2
|
||
|
|
BYTE $0xc4; BYTE $0xe3; BYTE $0xfd; BYTE $0x00; BYTE $0xdb; BYTE $0x93 \ // VPERMQ 0x93, Y3, Y3
|
||
|
|
VPADDQ m2, Y0, Y0; \
|
||
|
|
VPADDQ Y1, Y0, Y0; \
|
||
|
|
VPXOR Y0, Y3, Y3; \
|
||
|
|
VPSHUFD $-79, Y3, Y3; \
|
||
|
|
VPADDQ Y3, Y2, Y2; \
|
||
|
|
VPXOR Y2, Y1, Y1; \
|
||
|
|
VPSHUFB c40, Y1, Y1; \
|
||
|
|
VPADDQ m3, Y0, Y0; \
|
||
|
|
VPADDQ Y1, Y0, Y0; \
|
||
|
|
VPXOR Y0, Y3, Y3; \
|
||
|
|
VPSHUFB c48, Y3, Y3; \
|
||
|
|
VPADDQ Y3, Y2, Y2; \
|
||
|
|
VPXOR Y2, Y1, Y1; \
|
||
|
|
VPADDQ Y1, Y1, t; \
|
||
|
|
VPSRLQ $63, Y1, Y1; \
|
||
|
|
VPXOR t, Y1, Y1; \
|
||
|
|
BYTE $0xc4; BYTE $0xe3; BYTE $0xfd; BYTE $0x00; BYTE $0xdb; BYTE $0x39 \ // VPERMQ 0x39, Y3, Y3
|
||
|
|
BYTE $0xc4; BYTE $0xe3; BYTE $0xfd; BYTE $0x00; BYTE $0xd2; BYTE $0x4e \ // VPERMQ 0x4e, Y2, Y2
|
||
|
|
BYTE $0xc4; BYTE $0xe3; BYTE $0xfd; BYTE $0x00; BYTE $0xc9; BYTE $0x93 \ // VPERMQ 0x93, Y1, Y1
|
||
|
|
|
||
|
|
// load msg into Y12, Y13, Y14, Y15
|
||
|
|
#define LOAD_MSG_AVX2(src, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15) \
|
||
|
|
MOVQ i0*8(src), X12; \
|
||
|
|
PINSRQ $1, i1*8(src), X12; \
|
||
|
|
MOVQ i2*8(src), X11; \
|
||
|
|
PINSRQ $1, i3*8(src), X11; \
|
||
|
|
VINSERTI128 $1, X11, Y12, Y12; \
|
||
|
|
MOVQ i4*8(src), X13; \
|
||
|
|
PINSRQ $1, i5*8(src), X13; \
|
||
|
|
MOVQ i6*8(src), X11; \
|
||
|
|
PINSRQ $1, i7*8(src), X11; \
|
||
|
|
VINSERTI128 $1, X11, Y13, Y13; \
|
||
|
|
MOVQ i8*8(src), X14; \
|
||
|
|
PINSRQ $1, i9*8(src), X14; \
|
||
|
|
MOVQ i10*8(src), X11; \
|
||
|
|
PINSRQ $1, i11*8(src), X11; \
|
||
|
|
VINSERTI128 $1, X11, Y14, Y14; \
|
||
|
|
MOVQ i12*8(src), X15; \
|
||
|
|
PINSRQ $1, i13*8(src), X15; \
|
||
|
|
MOVQ i14*8(src), X11; \
|
||
|
|
PINSRQ $1, i15*8(src), X11; \
|
||
|
|
VINSERTI128 $1, X11, Y15, Y15
|
||
|
|
|
||
|
|
// func hashBlocksAVX2(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte)
|
||
|
|
TEXT ·hashBlocksAVX2(SB), 4, $320-48 // frame size = 288 + 32 byte alignment
|
||
|
|
MOVQ h+0(FP), AX
|
||
|
|
MOVQ c+8(FP), BX
|
||
|
|
MOVQ flag+16(FP), CX
|
||
|
|
MOVQ blocks_base+24(FP), SI
|
||
|
|
MOVQ blocks_len+32(FP), DI
|
||
|
|
|
||
|
|
MOVQ SP, DX
|
||
|
|
MOVQ SP, R9
|
||
|
|
ADDQ $31, R9
|
||
|
|
ANDQ $~31, R9
|
||
|
|
MOVQ R9, SP
|
||
|
|
|
||
|
|
MOVQ CX, 16(SP)
|
||
|
|
XORQ CX, CX
|
||
|
|
MOVQ CX, 24(SP)
|
||
|
|
|
||
|
|
VMOVDQU ·AVX2_c40<>(SB), Y4
|
||
|
|
VMOVDQU ·AVX2_c48<>(SB), Y5
|
||
|
|
|
||
|
|
VMOVDQU 0(AX), Y8
|
||
|
|
VMOVDQU 32(AX), Y9
|
||
|
|
VMOVDQU ·AVX2_iv0<>(SB), Y6
|
||
|
|
VMOVDQU ·AVX2_iv1<>(SB), Y7
|
||
|
|
|
||
|
|
MOVQ 0(BX), R8
|
||
|
|
MOVQ 8(BX), R9
|
||
|
|
MOVQ R9, 8(SP)
|
||
|
|
|
||
|
|
loop:
|
||
|
|
ADDQ $128, R8
|
||
|
|
MOVQ R8, 0(SP)
|
||
|
|
CMPQ R8, $128
|
||
|
|
JGE noinc
|
||
|
|
INCQ R9
|
||
|
|
MOVQ R9, 8(SP)
|
||
|
|
|
||
|
|
noinc:
|
||
|
|
VMOVDQA Y8, Y0
|
||
|
|
VMOVDQA Y9, Y1
|
||
|
|
VMOVDQA Y6, Y2
|
||
|
|
VPXOR 0(SP), Y7, Y3
|
||
|
|
|
||
|
|
LOAD_MSG_AVX2(SI, 0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15)
|
||
|
|
VMOVDQA Y12, 32(SP)
|
||
|
|
VMOVDQA Y13, 64(SP)
|
||
|
|
VMOVDQA Y14, 96(SP)
|
||
|
|
VMOVDQA Y15, 128(SP)
|
||
|
|
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
|
||
|
|
LOAD_MSG_AVX2(SI, 14, 4, 9, 13, 10, 8, 15, 6, 1, 0, 11, 5, 12, 2, 7, 3)
|
||
|
|
VMOVDQA Y12, 160(SP)
|
||
|
|
VMOVDQA Y13, 192(SP)
|
||
|
|
VMOVDQA Y14, 224(SP)
|
||
|
|
VMOVDQA Y15, 256(SP)
|
||
|
|
|
||
|
|
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
|
||
|
|
LOAD_MSG_AVX2(SI, 11, 12, 5, 15, 8, 0, 2, 13, 10, 3, 7, 9, 14, 6, 1, 4)
|
||
|
|
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
|
||
|
|
LOAD_MSG_AVX2(SI, 7, 3, 13, 11, 9, 1, 12, 14, 2, 5, 4, 15, 6, 10, 0, 8)
|
||
|
|
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
|
||
|
|
LOAD_MSG_AVX2(SI, 9, 5, 2, 10, 0, 7, 4, 15, 14, 11, 6, 3, 1, 12, 8, 13)
|
||
|
|
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
|
||
|
|
LOAD_MSG_AVX2(SI, 2, 6, 0, 8, 12, 10, 11, 3, 4, 7, 15, 1, 13, 5, 14, 9)
|
||
|
|
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
|
||
|
|
LOAD_MSG_AVX2(SI, 12, 1, 14, 4, 5, 15, 13, 10, 0, 6, 9, 8, 7, 3, 2, 11)
|
||
|
|
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
|
||
|
|
LOAD_MSG_AVX2(SI, 13, 7, 12, 3, 11, 14, 1, 9, 5, 15, 8, 2, 0, 4, 6, 10)
|
||
|
|
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
|
||
|
|
LOAD_MSG_AVX2(SI, 6, 14, 11, 0, 15, 9, 3, 8, 12, 13, 1, 10, 2, 7, 4, 5)
|
||
|
|
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
|
||
|
|
LOAD_MSG_AVX2(SI, 10, 8, 7, 1, 2, 4, 6, 5, 15, 9, 3, 13, 11, 14, 12, 0)
|
||
|
|
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
|
||
|
|
|
||
|
|
ROUND_AVX2(32(SP), 64(SP), 96(SP), 128(SP), Y10, Y4, Y5)
|
||
|
|
ROUND_AVX2(160(SP), 192(SP), 224(SP), 256(SP), Y10, Y4, Y5)
|
||
|
|
|
||
|
|
VPXOR Y0, Y8, Y8
|
||
|
|
VPXOR Y1, Y9, Y9
|
||
|
|
VPXOR Y2, Y8, Y8
|
||
|
|
VPXOR Y3, Y9, Y9
|
||
|
|
|
||
|
|
LEAQ 128(SI), SI
|
||
|
|
SUBQ $128, DI
|
||
|
|
JNE loop
|
||
|
|
|
||
|
|
MOVQ R8, 0(BX)
|
||
|
|
MOVQ R9, 8(BX)
|
||
|
|
|
||
|
|
VMOVDQU Y8, 0(AX)
|
||
|
|
VMOVDQU Y9, 32(AX)
|
||
|
|
|
||
|
|
MOVQ DX, SP
|
||
|
|
RET
|
||
|
|
|
||
|
|
// unfortunately the BYTE representation of VPUNPCKLQDQ and VPUNPCKHQDQ must be used
|
||
|
|
#define VPUNPCKLQDQ_X8_X8_X10 BYTE $0xC4; BYTE $0x41; BYTE $0x39; BYTE $0x6C; BYTE $0xD0
|
||
|
|
#define VPUNPCKHQDQ_X7_X10_X6 BYTE $0xC4; BYTE $0xC1; BYTE $0x41; BYTE $0x6D; BYTE $0xF2
|
||
|
|
#define VPUNPCKLQDQ_X7_X7_X10 BYTE $0xC5; BYTE $0x41; BYTE $0x6C; BYTE $0xD7
|
||
|
|
#define VPUNPCKHQDQ_X8_X10_X7 BYTE $0xC4; BYTE $0xC1; BYTE $0x39; BYTE $0x6D; BYTE $0xFA
|
||
|
|
#define VPUNPCKLQDQ_X3_X3_X10 BYTE $0xC5; BYTE $0x61; BYTE $0x6C; BYTE $0xD3
|
||
|
|
#define VPUNPCKHQDQ_X2_X10_X2 BYTE $0xC4; BYTE $0xC1; BYTE $0x69; BYTE $0x6D; BYTE $0xD2
|
||
|
|
#define VPUNPCKLQDQ_X9_X9_X10 BYTE $0xC4; BYTE $0x41; BYTE $0x31; BYTE $0x6C; BYTE $0xD1
|
||
|
|
#define VPUNPCKHQDQ_X3_X10_X3 BYTE $0xC4; BYTE $0xC1; BYTE $0x61; BYTE $0x6D; BYTE $0xDA
|
||
|
|
#define VPUNPCKLQDQ_X2_X2_X10 BYTE $0xC5; BYTE $0x69; BYTE $0x6C; BYTE $0xD2
|
||
|
|
#define VPUNPCKHQDQ_X3_X10_X2 BYTE $0xC4; BYTE $0xC1; BYTE $0x61; BYTE $0x6D; BYTE $0xD2
|
||
|
|
#define VPUNPCKHQDQ_X8_X10_X3 BYTE $0xC4; BYTE $0xC1; BYTE $0x39; BYTE $0x6D; BYTE $0xDA
|
||
|
|
#define VPUNPCKHQDQ_X6_X10_X6 BYTE $0xC4; BYTE $0xC1; BYTE $0x49; BYTE $0x6D; BYTE $0xF2
|
||
|
|
#define VPUNPCKHQDQ_X7_X10_X7 BYTE $0xC4; BYTE $0xC1; BYTE $0x41; BYTE $0x6D; BYTE $0xFA
|
||
|
|
|
||
|
|
// shuffle X2 and X6 using the temp registers X8, X9, X10
|
||
|
|
#define SHUFFLE_AVX() \
|
||
|
|
VMOVDQA X4, X9; \
|
||
|
|
VMOVDQA X5, X4; \
|
||
|
|
VMOVDQA X9, X5; \
|
||
|
|
VMOVDQA X6, X8; \
|
||
|
|
VPUNPCKLQDQ_X8_X8_X10; \
|
||
|
|
VPUNPCKHQDQ_X7_X10_X6; \
|
||
|
|
VPUNPCKLQDQ_X7_X7_X10; \
|
||
|
|
VPUNPCKHQDQ_X8_X10_X7; \
|
||
|
|
VPUNPCKLQDQ_X3_X3_X10; \
|
||
|
|
VMOVDQA X2, X9; \
|
||
|
|
VPUNPCKHQDQ_X2_X10_X2; \
|
||
|
|
VPUNPCKLQDQ_X9_X9_X10; \
|
||
|
|
VPUNPCKHQDQ_X3_X10_X3; \
|
||
|
|
|
||
|
|
// inverse shuffle X2 and X6 using the temp registers X8, X9, X10
|
||
|
|
#define SHUFFLE_AVX_INV() \
|
||
|
|
VMOVDQA X4, X9; \
|
||
|
|
VMOVDQA X5, X4; \
|
||
|
|
VMOVDQA X9, X5; \
|
||
|
|
VMOVDQA X2, X8; \
|
||
|
|
VPUNPCKLQDQ_X2_X2_X10; \
|
||
|
|
VPUNPCKHQDQ_X3_X10_X2; \
|
||
|
|
VPUNPCKLQDQ_X3_X3_X10; \
|
||
|
|
VPUNPCKHQDQ_X8_X10_X3; \
|
||
|
|
VPUNPCKLQDQ_X7_X7_X10; \
|
||
|
|
VMOVDQA X6, X9; \
|
||
|
|
VPUNPCKHQDQ_X6_X10_X6; \
|
||
|
|
VPUNPCKLQDQ_X9_X9_X10; \
|
||
|
|
VPUNPCKHQDQ_X7_X10_X7; \
|
||
|
|
|
||
|
|
#define HALF_ROUND_AVX(v0, v1, v2, v3, v4, v5, v6, v7, m0, m1, m2, m3, t0, c40, c48) \
|
||
|
|
VPADDQ m0, v0, v0; \
|
||
|
|
VPADDQ v2, v0, v0; \
|
||
|
|
VPADDQ m1, v1, v1; \
|
||
|
|
VPADDQ v3, v1, v1; \
|
||
|
|
VPXOR v0, v6, v6; \
|
||
|
|
VPXOR v1, v7, v7; \
|
||
|
|
VPSHUFD $-79, v6, v6; \
|
||
|
|
VPSHUFD $-79, v7, v7; \
|
||
|
|
VPADDQ v6, v4, v4; \
|
||
|
|
VPADDQ v7, v5, v5; \
|
||
|
|
VPXOR v4, v2, v2; \
|
||
|
|
VPXOR v5, v3, v3; \
|
||
|
|
VPSHUFB c40, v2, v2; \
|
||
|
|
VPSHUFB c40, v3, v3; \
|
||
|
|
VPADDQ m2, v0, v0; \
|
||
|
|
VPADDQ v2, v0, v0; \
|
||
|
|
VPADDQ m3, v1, v1; \
|
||
|
|
VPADDQ v3, v1, v1; \
|
||
|
|
VPXOR v0, v6, v6; \
|
||
|
|
VPXOR v1, v7, v7; \
|
||
|
|
VPSHUFB c48, v6, v6; \
|
||
|
|
VPSHUFB c48, v7, v7; \
|
||
|
|
VPADDQ v6, v4, v4; \
|
||
|
|
VPADDQ v7, v5, v5; \
|
||
|
|
VPXOR v4, v2, v2; \
|
||
|
|
VPXOR v5, v3, v3; \
|
||
|
|
VPADDQ v2, v2, t0; \
|
||
|
|
VPSRLQ $63, v2, v2; \
|
||
|
|
VPXOR t0, v2, v2; \
|
||
|
|
VPADDQ v3, v3, t0; \
|
||
|
|
VPSRLQ $63, v3, v3; \
|
||
|
|
VPXOR t0, v3, v3
|
||
|
|
|
||
|
|
// unfortunately the BYTE representation of VPINSRQ must be used
|
||
|
|
#define VPINSRQ_1_R10_X8_X8 BYTE $0xC4; BYTE $0x43; BYTE $0xB9; BYTE $0x22; BYTE $0xC2; BYTE $0x01
|
||
|
|
#define VPINSRQ_1_R11_X9_X9 BYTE $0xC4; BYTE $0x43; BYTE $0xB1; BYTE $0x22; BYTE $0xCB; BYTE $0x01
|
||
|
|
#define VPINSRQ_1_R12_X10_X10 BYTE $0xC4; BYTE $0x43; BYTE $0xA9; BYTE $0x22; BYTE $0xD4; BYTE $0x01
|
||
|
|
#define VPINSRQ_1_R13_X11_X11 BYTE $0xC4; BYTE $0x43; BYTE $0xA1; BYTE $0x22; BYTE $0xDD; BYTE $0x01
|
||
|
|
|
||
|
|
#define VPINSRQ_1_R9_X8_X8 BYTE $0xC4; BYTE $0x43; BYTE $0xB9; BYTE $0x22; BYTE $0xC1; BYTE $0x01
|
||
|
|
|
||
|
|
// load src into X8, X9, X10 and X11 using R10, R11, R12 and R13 for temp registers
|
||
|
|
#define LOAD_MSG_AVX(src, i0, i1, i2, i3, i4, i5, i6, i7) \
|
||
|
|
MOVQ i0*8(src), X8; \
|
||
|
|
MOVQ i1*8(src), R10; \
|
||
|
|
MOVQ i2*8(src), X9; \
|
||
|
|
MOVQ i3*8(src), R11; \
|
||
|
|
MOVQ i4*8(src), X10; \
|
||
|
|
MOVQ i5*8(src), R12; \
|
||
|
|
MOVQ i6*8(src), X11; \
|
||
|
|
MOVQ i7*8(src), R13; \
|
||
|
|
VPINSRQ_1_R10_X8_X8; \
|
||
|
|
VPINSRQ_1_R11_X9_X9; \
|
||
|
|
VPINSRQ_1_R12_X10_X10; \
|
||
|
|
VPINSRQ_1_R13_X11_X11
|
||
|
|
|
||
|
|
// func hashBlocksAVX(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte)
|
||
|
|
TEXT ·hashBlocksAVX(SB), 4, $288-48 // frame size = 272 + 16 byte alignment
|
||
|
|
MOVQ h+0(FP), AX
|
||
|
|
MOVQ c+8(FP), BX
|
||
|
|
MOVQ flag+16(FP), CX
|
||
|
|
MOVQ blocks_base+24(FP), SI
|
||
|
|
MOVQ blocks_len+32(FP), DI
|
||
|
|
|
||
|
|
MOVQ SP, BP
|
||
|
|
MOVQ SP, R9
|
||
|
|
ADDQ $15, R9
|
||
|
|
ANDQ $~15, R9
|
||
|
|
MOVQ R9, SP
|
||
|
|
|
||
|
|
MOVOU ·AVX_c40<>(SB), X13
|
||
|
|
MOVOU ·AVX_c48<>(SB), X14
|
||
|
|
|
||
|
|
VMOVDQU ·AVX_iv3<>(SB), X0
|
||
|
|
VMOVDQA X0, 0(SP)
|
||
|
|
XORQ CX, 0(SP) // 0(SP) = ·AVX_iv3 ^ (CX || 0)
|
||
|
|
|
||
|
|
VMOVDQU 0(AX), X12
|
||
|
|
VMOVDQU 16(AX), X15
|
||
|
|
VMOVDQU 32(AX), X2
|
||
|
|
VMOVDQU 48(AX), X3
|
||
|
|
|
||
|
|
MOVQ 0(BX), R8
|
||
|
|
MOVQ 8(BX), R9
|
||
|
|
|
||
|
|
loop:
|
||
|
|
ADDQ $128, R8
|
||
|
|
CMPQ R8, $128
|
||
|
|
JGE noinc
|
||
|
|
INCQ R9
|
||
|
|
|
||
|
|
noinc:
|
||
|
|
MOVQ R8, X8
|
||
|
|
VPINSRQ_1_R9_X8_X8
|
||
|
|
|
||
|
|
VMOVDQA X12, X0
|
||
|
|
VMOVDQA X15, X1
|
||
|
|
VMOVDQU ·AVX_iv0<>(SB), X4
|
||
|
|
VMOVDQU ·AVX_iv1<>(SB), X5
|
||
|
|
VMOVDQU ·AVX_iv2<>(SB), X6
|
||
|
|
|
||
|
|
VPXOR X8, X6, X6
|
||
|
|
VMOVDQA 0(SP), X7
|
||
|
|
|
||
|
|
LOAD_MSG_AVX(SI, 0, 2, 4, 6, 1, 3, 5, 7)
|
||
|
|
VMOVDQA X8, 16(SP)
|
||
|
|
VMOVDQA X9, 32(SP)
|
||
|
|
VMOVDQA X10, 48(SP)
|
||
|
|
VMOVDQA X11, 64(SP)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX()
|
||
|
|
LOAD_MSG_AVX(SI, 8, 10, 12, 14, 9, 11, 13, 15)
|
||
|
|
VMOVDQA X8, 80(SP)
|
||
|
|
VMOVDQA X9, 96(SP)
|
||
|
|
VMOVDQA X10, 112(SP)
|
||
|
|
VMOVDQA X11, 128(SP)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX_INV()
|
||
|
|
|
||
|
|
LOAD_MSG_AVX(SI, 14, 4, 9, 13, 10, 8, 15, 6)
|
||
|
|
VMOVDQA X8, 144(SP)
|
||
|
|
VMOVDQA X9, 160(SP)
|
||
|
|
VMOVDQA X10, 176(SP)
|
||
|
|
VMOVDQA X11, 192(SP)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX()
|
||
|
|
LOAD_MSG_AVX(SI, 1, 0, 11, 5, 12, 2, 7, 3)
|
||
|
|
VMOVDQA X8, 208(SP)
|
||
|
|
VMOVDQA X9, 224(SP)
|
||
|
|
VMOVDQA X10, 240(SP)
|
||
|
|
VMOVDQA X11, 256(SP)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX_INV()
|
||
|
|
|
||
|
|
LOAD_MSG_AVX(SI, 11, 12, 5, 15, 8, 0, 2, 13)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX()
|
||
|
|
LOAD_MSG_AVX(SI, 10, 3, 7, 9, 14, 6, 1, 4)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX_INV()
|
||
|
|
|
||
|
|
LOAD_MSG_AVX(SI, 7, 3, 13, 11, 9, 1, 12, 14)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX()
|
||
|
|
LOAD_MSG_AVX(SI, 2, 5, 4, 15, 6, 10, 0, 8)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX_INV()
|
||
|
|
|
||
|
|
LOAD_MSG_AVX(SI, 9, 5, 2, 10, 0, 7, 4, 15)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX()
|
||
|
|
LOAD_MSG_AVX(SI, 14, 11, 6, 3, 1, 12, 8, 13)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX_INV()
|
||
|
|
|
||
|
|
LOAD_MSG_AVX(SI, 2, 6, 0, 8, 12, 10, 11, 3)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX()
|
||
|
|
LOAD_MSG_AVX(SI, 4, 7, 15, 1, 13, 5, 14, 9)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX_INV()
|
||
|
|
|
||
|
|
LOAD_MSG_AVX(SI, 12, 1, 14, 4, 5, 15, 13, 10)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX()
|
||
|
|
LOAD_MSG_AVX(SI, 0, 6, 9, 8, 7, 3, 2, 11)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX_INV()
|
||
|
|
|
||
|
|
LOAD_MSG_AVX(SI, 13, 7, 12, 3, 11, 14, 1, 9)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX()
|
||
|
|
LOAD_MSG_AVX(SI, 5, 15, 8, 2, 0, 4, 6, 10)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX_INV()
|
||
|
|
|
||
|
|
LOAD_MSG_AVX(SI, 6, 14, 11, 0, 15, 9, 3, 8)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX()
|
||
|
|
LOAD_MSG_AVX(SI, 12, 13, 1, 10, 2, 7, 4, 5)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX_INV()
|
||
|
|
|
||
|
|
LOAD_MSG_AVX(SI, 10, 8, 7, 1, 2, 4, 6, 5)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX()
|
||
|
|
LOAD_MSG_AVX(SI, 15, 9, 3, 13, 11, 14, 12, 0)
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
|
||
|
|
SHUFFLE_AVX_INV()
|
||
|
|
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, 16(SP), 32(SP), 48(SP), 64(SP), X11, X13, X14)
|
||
|
|
SHUFFLE_AVX()
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, 80(SP), 96(SP), 112(SP), 128(SP), X11, X13, X14)
|
||
|
|
SHUFFLE_AVX_INV()
|
||
|
|
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, 144(SP), 160(SP), 176(SP), 192(SP), X11, X13, X14)
|
||
|
|
SHUFFLE_AVX()
|
||
|
|
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, 208(SP), 224(SP), 240(SP), 256(SP), X11, X13, X14)
|
||
|
|
SHUFFLE_AVX_INV()
|
||
|
|
|
||
|
|
VMOVDQU 32(AX), X10
|
||
|
|
VMOVDQU 48(AX), X11
|
||
|
|
VPXOR X0, X12, X12
|
||
|
|
VPXOR X1, X15, X15
|
||
|
|
VPXOR X2, X10, X10
|
||
|
|
VPXOR X3, X11, X11
|
||
|
|
VPXOR X4, X12, X12
|
||
|
|
VPXOR X5, X15, X15
|
||
|
|
VPXOR X6, X10, X2
|
||
|
|
VPXOR X7, X11, X3
|
||
|
|
VMOVDQU X2, 32(AX)
|
||
|
|
VMOVDQU X3, 48(AX)
|
||
|
|
|
||
|
|
LEAQ 128(SI), SI
|
||
|
|
SUBQ $128, DI
|
||
|
|
JNE loop
|
||
|
|
|
||
|
|
VMOVDQU X12, 0(AX)
|
||
|
|
VMOVDQU X15, 16(AX)
|
||
|
|
|
||
|
|
MOVQ R8, 0(BX)
|
||
|
|
MOVQ R9, 8(BX)
|
||
|
|
|
||
|
|
VZEROUPPER
|
||
|
|
|
||
|
|
MOVQ BP, SP
|
||
|
|
RET
|
||
|
|
|
||
|
|
// func supportsAVX2() bool
|
||
|
|
TEXT ·supportsAVX2(SB), 4, $0-1
|
||
|
|
MOVQ runtime·support_avx2(SB), AX
|
||
|
|
MOVB AX, ret+0(FP)
|
||
|
|
RET
|
||
|
|
|
||
|
|
// func supportsAVX() bool
|
||
|
|
TEXT ·supportsAVX(SB), 4, $0-1
|
||
|
|
MOVQ runtime·support_avx(SB), AX
|
||
|
|
MOVB AX, ret+0(FP)
|
||
|
|
RET
|