// Code generated by command: go run gcm_amd64_asm.go -out ../../gcm_amd64.s -pkg aes. DO NOT EDIT.

//go:build !purego

#include "textflag.h"

// func gcmAesFinish(productTable *[256]byte, tagMask *[16]byte, T *[16]byte, pLen uint64, dLen uint64)
// Requires: PCLMULQDQ, SSE2, SSE4.1, SSSE3
TEXT ·gcmAesFinish(SB), NOSPLIT, $0-40
	MOVQ      productTable+0(FP), DI
	MOVQ      tagMask+8(FP), SI
	MOVQ      T+16(FP), DX
	MOVQ      pLen+24(FP), AX
	MOVQ      dLen+32(FP), CX
	MOVOU     (DX), X8
	MOVOU     (SI), X13
	MOVOU     bswapMask<>+0(SB), X15
	MOVOU     gcmPoly<>+0(SB), X14
	SHLQ      $0x03, AX
	SHLQ      $0x03, CX
	MOVQ      AX, X0
	PINSRQ    $0x01, CX, X0
	PXOR      X8, X0
	MOVOU     224(DI), X8
	MOVOU     240(DI), X10
	MOVOU     X8, X9
	PCLMULQDQ $0x00, X0, X8
	PCLMULQDQ $0x11, X0, X9
	PSHUFD    $0x4e, X0, X11
	PXOR      X0, X11
	PCLMULQDQ $0x00, X11, X10
	PXOR      X8, X10
	PXOR      X9, X10
	MOVOU     X10, X11
	PSRLDQ    $0x08, X10
	PSLLDQ    $0x08, X11
	PXOR      X10, X9
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	PXOR      X9, X8
	PSHUFB    X15, X8
	PXOR      X13, X8
	MOVOU     X8, (DX)
	RET

DATA bswapMask<>+0(SB)/8, $0x08090a0b0c0d0e0f
DATA bswapMask<>+8(SB)/8, $0x0001020304050607
GLOBL bswapMask<>(SB), RODATA|NOPTR, $16

DATA gcmPoly<>+0(SB)/8, $0x0000000000000001
DATA gcmPoly<>+8(SB)/8, $0xc200000000000000
GLOBL gcmPoly<>(SB), RODATA|NOPTR, $16

// func gcmAesInit(productTable *[256]byte, ks []uint32)
// Requires: AES, PCLMULQDQ, SSE2, SSSE3
TEXT ·gcmAesInit(SB), NOSPLIT, $0-32
	MOVQ  productTable+0(FP), DI
	MOVQ  ks_base+8(FP), SI
	MOVQ  ks_len+16(FP), DX
	SHRQ  $0x02, DX
	DECQ  DX
	MOVOU bswapMask<>+0(SB), X15
	MOVOU gcmPoly<>+0(SB), X14

	// Encrypt block 0, with the AES key to generate the hash key H
	MOVOU  (SI), X0
	MOVOU  16(SI), X11
	AESENC X11, X0
	MOVOU  32(SI), X11
	AESENC X11, X0
	MOVOU  48(SI), X11
	AESENC X11, X0
	MOVOU  64(SI), X11
	AESENC X11, X0
	MOVOU  80(SI), X11
	AESENC X11, X0
	MOVOU  96(SI), X11
	AESENC X11, X0
	MOVOU  112(SI), X11
	AESENC X11, X0
	MOVOU  128(SI), X11
	AESENC X11, X0
	MOVOU  144(SI), X11
	AESENC X11, X0
	MOVOU  160(SI), X11
	CMPQ   DX, $0x0c
	JB     initEncLast
	AESENC X11, X0
	MOVOU  176(SI), X11
	AESENC X11, X0
	MOVOU  192(SI), X11
	JE     initEncLast
	AESENC X11, X0
	MOVOU  208(SI), X11
	AESENC X11, X0
	MOVOU  224(SI), X11

initEncLast:
	AESENCLAST X11, X0
	PSHUFB     X15, X0

	// H * 2
	PSHUFD $0xff, X0, X11
	MOVOU  X0, X12
	PSRAL  $0x1f, X11
	PAND   X14, X11
	PSRLL  $0x1f, X12
	PSLLDQ $0x04, X12
	PSLLL  $0x01, X0
	PXOR   X11, X0
	PXOR   X12, X0

	// Karatsuba pre-computations
	MOVOU  X0, 224(DI)
	PSHUFD $0x4e, X0, X1
	PXOR   X0, X1
	MOVOU  X1, 240(DI)
	MOVOU  X0, X2
	MOVOU  X1, X3

	// Now prepare powers of H and pre-computations for them
	MOVQ $0x00000007, AX

initLoop:
	MOVOU     X2, X11
	MOVOU     X2, X12
	MOVOU     X3, X13
	PCLMULQDQ $0x00, X0, X11
	PCLMULQDQ $0x11, X0, X12
	PCLMULQDQ $0x00, X1, X13
	PXOR      X11, X13
	PXOR      X12, X13
	MOVOU     X13, X4
	PSLLDQ    $0x08, X4
	PSRLDQ    $0x08, X13
	PXOR      X4, X11
	PXOR      X13, X12
	MOVOU     X14, X2
	PCLMULQDQ $0x01, X11, X2
	PSHUFD    $0x4e, X11, X11
	PXOR      X2, X11
	MOVOU     X14, X2
	PCLMULQDQ $0x01, X11, X2
	PSHUFD    $0x4e, X11, X11
	PXOR      X11, X2
	PXOR      X12, X2
	MOVOU     X2, 192(DI)
	PSHUFD    $0x4e, X2, X3
	PXOR      X2, X3
	MOVOU     X3, 208(DI)
	DECQ      AX
	LEAQ      -32(DI), DI
	JNE       initLoop
	RET

// func gcmAesData(productTable *[256]byte, data []byte, T *[16]byte)
// Requires: PCLMULQDQ, SSE2, SSE4.1, SSSE3
TEXT ·gcmAesData(SB), NOSPLIT, $0-40
	MOVQ  productTable+0(FP), DI
	MOVQ  data_base+8(FP), SI
	MOVQ  data_len+16(FP), DX
	MOVQ  T+32(FP), CX
	PXOR  X8, X8
	MOVOU bswapMask<>+0(SB), X15
	MOVOU gcmPoly<>+0(SB), X14
	TESTQ DX, DX
	JEQ   dataBail
	CMPQ  DX, $0x0d
	JE    dataTLS
	CMPQ  DX, $0x80
	JB    startSinglesLoop
	JMP   dataOctaLoop

dataTLS:
	MOVOU  224(DI), X12
	MOVOU  240(DI), X13
	PXOR   X0, X0
	MOVQ   (SI), X0
	PINSRD $0x02, 8(SI), X0
	PINSRB $0x0c, 12(SI), X0
	XORQ   DX, DX
	JMP    dataMul

dataOctaLoop:
	CMPQ      DX, $0x80
	JB        startSinglesLoop
	SUBQ      $0x80, DX
	MOVOU     (SI), X0
	MOVOU     16(SI), X1
	MOVOU     32(SI), X2
	MOVOU     48(SI), X3
	MOVOU     64(SI), X4
	MOVOU     80(SI), X5
	MOVOU     96(SI), X6
	MOVOU     112(SI), X7
	LEAQ      128(SI), SI
	PSHUFB    X15, X0
	PSHUFB    X15, X1
	PSHUFB    X15, X2
	PSHUFB    X15, X3
	PSHUFB    X15, X4
	PSHUFB    X15, X5
	PSHUFB    X15, X6
	PSHUFB    X15, X7
	PXOR      X8, X0
	MOVOU     (DI), X8
	MOVOU     16(DI), X10
	MOVOU     X8, X9
	PSHUFD    $0x4e, X0, X12
	PXOR      X0, X12
	PCLMULQDQ $0x00, X0, X8
	PCLMULQDQ $0x11, X0, X9
	PCLMULQDQ $0x00, X12, X10
	MOVOU     32(DI), X12
	MOVOU     X12, X13
	PCLMULQDQ $0x00, X1, X12
	PXOR      X12, X8
	PCLMULQDQ $0x11, X1, X13
	PXOR      X13, X9
	PSHUFD    $0x4e, X1, X12
	PXOR      X12, X1
	MOVOU     48(DI), X12
	PCLMULQDQ $0x00, X1, X12
	PXOR      X12, X10
	MOVOU     64(DI), X12
	MOVOU     X12, X13
	PCLMULQDQ $0x00, X2, X12
	PXOR      X12, X8
	PCLMULQDQ $0x11, X2, X13
	PXOR      X13, X9
	PSHUFD    $0x4e, X2, X12
	PXOR      X12, X2
	MOVOU     80(DI), X12
	PCLMULQDQ $0x00, X2, X12
	PXOR      X12, X10
	MOVOU     96(DI), X12
	MOVOU     X12, X13
	PCLMULQDQ $0x00, X3, X12
	PXOR      X12, X8
	PCLMULQDQ $0x11, X3, X13
	PXOR      X13, X9
	PSHUFD    $0x4e, X3, X12
	PXOR      X12, X3
	MOVOU     112(DI), X12
	PCLMULQDQ $0x00, X3, X12
	PXOR      X12, X10
	MOVOU     128(DI), X12
	MOVOU     X12, X13
	PCLMULQDQ $0x00, X4, X12
	PXOR      X12, X8
	PCLMULQDQ $0x11, X4, X13
	PXOR      X13, X9
	PSHUFD    $0x4e, X4, X12
	PXOR      X12, X4
	MOVOU     144(DI), X12
	PCLMULQDQ $0x00, X4, X12
	PXOR      X12, X10
	MOVOU     160(DI), X12
	MOVOU     X12, X13
	PCLMULQDQ $0x00, X5, X12
	PXOR      X12, X8
	PCLMULQDQ $0x11, X5, X13
	PXOR      X13, X9
	PSHUFD    $0x4e, X5, X12
	PXOR      X12, X5
	MOVOU     176(DI), X12
	PCLMULQDQ $0x00, X5, X12
	PXOR      X12, X10
	MOVOU     192(DI), X12
	MOVOU     X12, X13
	PCLMULQDQ $0x00, X6, X12
	PXOR      X12, X8
	PCLMULQDQ $0x11, X6, X13
	PXOR      X13, X9
	PSHUFD    $0x4e, X6, X12
	PXOR      X12, X6
	MOVOU     208(DI), X12
	PCLMULQDQ $0x00, X6, X12
	PXOR      X12, X10
	MOVOU     224(DI), X12
	MOVOU     X12, X13
	PCLMULQDQ $0x00, X7, X12
	PXOR      X12, X8
	PCLMULQDQ $0x11, X7, X13
	PXOR      X13, X9
	PSHUFD    $0x4e, X7, X12
	PXOR      X12, X7
	MOVOU     240(DI), X12
	PCLMULQDQ $0x00, X7, X12
	PXOR      X12, X10
	PXOR      X8, X10
	PXOR      X9, X10
	MOVOU     X10, X11
	PSRLDQ    $0x08, X10
	PSLLDQ    $0x08, X11
	PXOR      X10, X9
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	PXOR      X9, X8
	JMP       dataOctaLoop

startSinglesLoop:
	MOVOU 224(DI), X12
	MOVOU 240(DI), X13

dataSinglesLoop:
	CMPQ  DX, $0x10
	JB    dataEnd
	SUBQ  $0x10, DX
	MOVOU (SI), X0

dataMul:
	PSHUFB    X15, X0
	PXOR      X8, X0
	MOVOU     X12, X8
	MOVOU     X13, X10
	MOVOU     X12, X9
	PSHUFD    $0x4e, X0, X11
	PXOR      X0, X11
	PCLMULQDQ $0x00, X0, X8
	PCLMULQDQ $0x11, X0, X9
	PCLMULQDQ $0x00, X11, X10
	PXOR      X8, X10
	PXOR      X9, X10
	MOVOU     X10, X11
	PSRLDQ    $0x08, X10
	PSLLDQ    $0x08, X11
	PXOR      X10, X9
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	PXOR      X9, X8
	LEAQ      16(SI), SI
	JMP       dataSinglesLoop

dataEnd:
	TESTQ DX, DX
	JEQ   dataBail
	PXOR  X0, X0
	LEAQ  -1(SI)(DX*1), SI

dataLoadLoop:
	PSLLDQ $0x01, X0
	PINSRB $0x00, (SI), X0
	LEAQ   -1(SI), SI
	DECQ   DX
	JNE    dataLoadLoop
	JMP    dataMul

dataBail:
	MOVOU X8, (CX)
	RET

// func gcmAesEnc(productTable *[256]byte, dst []byte, src []byte, ctr *[16]byte, T *[16]byte, ks []uint32)
// Requires: AES, PCLMULQDQ, SSE2, SSE4.1, SSSE3
TEXT ·gcmAesEnc(SB), $256-96
	MOVQ   productTable+0(FP), DI
	MOVQ   dst_base+8(FP), DX
	MOVQ   src_base+32(FP), SI
	MOVQ   src_len+40(FP), R9
	MOVQ   ctr+56(FP), CX
	MOVQ   T+64(FP), R8
	MOVQ   ks_base+72(FP), AX
	MOVQ   ks_len+80(FP), R13
	SHRQ   $0x02, R13
	DECQ   R13
	MOVOU  bswapMask<>+0(SB), X15
	MOVOU  gcmPoly<>+0(SB), X14
	MOVOU  (R8), X8
	PXOR   X9, X9
	PXOR   X10, X10
	MOVOU  (CX), X0
	MOVL   12(CX), R10
	MOVOU  (AX), X11
	MOVL   12(AX), R12
	BSWAPL R10
	BSWAPL R12
	PXOR   X0, X11
	MOVOU  X11, 128(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 140(SP)
	CMPQ   R9, $0x80
	JB     gcmAesEncSingles
	SUBQ   $0x80, R9

	// We have at least 8 blocks to encrypt, prepare the rest of the counters
	MOVOU  X11, 144(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 156(SP)
	MOVOU  X11, 160(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 172(SP)
	MOVOU  X11, 176(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 188(SP)
	MOVOU  X11, 192(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 204(SP)
	MOVOU  X11, 208(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 220(SP)
	MOVOU  X11, 224(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 236(SP)
	MOVOU  X11, 240(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 252(SP)
	MOVOU  128(SP), X0
	MOVOU  144(SP), X1
	MOVOU  160(SP), X2
	MOVOU  176(SP), X3
	MOVOU  192(SP), X4
	MOVOU  208(SP), X5
	MOVOU  224(SP), X6
	MOVOU  240(SP), X7
	MOVOU  16(AX), X11
	AESENC X11, X0
	AESENC X11, X1
	AESENC X11, X2
	AESENC X11, X3
	AESENC X11, X4
	AESENC X11, X5
	AESENC X11, X6
	AESENC X11, X7
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 140(SP)
	MOVOU  32(AX), X11
	AESENC X11, X0
	AESENC X11, X1
	AESENC X11, X2
	AESENC X11, X3
	AESENC X11, X4
	AESENC X11, X5
	AESENC X11, X6
	AESENC X11, X7
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 156(SP)
	MOVOU  48(AX), X11
	AESENC X11, X0
	AESENC X11, X1
	AESENC X11, X2
	AESENC X11, X3
	AESENC X11, X4
	AESENC X11, X5
	AESENC X11, X6
	AESENC X11, X7
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 172(SP)
	MOVOU  64(AX), X11
	AESENC X11, X0
	AESENC X11, X1
	AESENC X11, X2
	AESENC X11, X3
	AESENC X11, X4
	AESENC X11, X5
	AESENC X11, X6
	AESENC X11, X7
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 188(SP)
	MOVOU  80(AX), X11
	AESENC X11, X0
	AESENC X11, X1
	AESENC X11, X2
	AESENC X11, X3
	AESENC X11, X4
	AESENC X11, X5
	AESENC X11, X6
	AESENC X11, X7
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 204(SP)
	MOVOU  96(AX), X11
	AESENC X11, X0
	AESENC X11, X1
	AESENC X11, X2
	AESENC X11, X3
	AESENC X11, X4
	AESENC X11, X5
	AESENC X11, X6
	AESENC X11, X7
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 220(SP)
	MOVOU  112(AX), X11
	AESENC X11, X0
	AESENC X11, X1
	AESENC X11, X2
	AESENC X11, X3
	AESENC X11, X4
	AESENC X11, X5
	AESENC X11, X6
	AESENC X11, X7
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 236(SP)
	MOVOU  128(AX), X11
	AESENC X11, X0
	AESENC X11, X1
	AESENC X11, X2
	AESENC X11, X3
	AESENC X11, X4
	AESENC X11, X5
	AESENC X11, X6
	AESENC X11, X7
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 252(SP)
	MOVOU  144(AX), X11
	AESENC X11, X0
	AESENC X11, X1
	AESENC X11, X2
	AESENC X11, X3
	AESENC X11, X4
	AESENC X11, X5
	AESENC X11, X6
	AESENC X11, X7
	MOVOU  160(AX), X11
	CMPQ   R13, $0x0c
	JB     encLast1
	AESENC X11, X0
	AESENC X11, X1
	AESENC X11, X2
	AESENC X11, X3
	AESENC X11, X4
	AESENC X11, X5
	AESENC X11, X6
	AESENC X11, X7
	MOVOU  176(AX), X11
	AESENC X11, X0
	AESENC X11, X1
	AESENC X11, X2
	AESENC X11, X3
	AESENC X11, X4
	AESENC X11, X5
	AESENC X11, X6
	AESENC X11, X7
	MOVOU  192(AX), X11
	JE     encLast1
	AESENC X11, X0
	AESENC X11, X1
	AESENC X11, X2
	AESENC X11, X3
	AESENC X11, X4
	AESENC X11, X5
	AESENC X11, X6
	AESENC X11, X7
	MOVOU  208(AX), X11
	AESENC X11, X0
	AESENC X11, X1
	AESENC X11, X2
	AESENC X11, X3
	AESENC X11, X4
	AESENC X11, X5
	AESENC X11, X6
	AESENC X11, X7
	MOVOU  224(AX), X11

encLast1:
	AESENCLAST X11, X0
	AESENCLAST X11, X1
	AESENCLAST X11, X2
	AESENCLAST X11, X3
	AESENCLAST X11, X4
	AESENCLAST X11, X5
	AESENCLAST X11, X6
	AESENCLAST X11, X7
	MOVOU      (SI), X11
	PXOR       X11, X0
	MOVOU      16(SI), X11
	PXOR       X11, X1
	MOVOU      32(SI), X11
	PXOR       X11, X2
	MOVOU      48(SI), X11
	PXOR       X11, X3
	MOVOU      64(SI), X11
	PXOR       X11, X4
	MOVOU      80(SI), X11
	PXOR       X11, X5
	MOVOU      96(SI), X11
	PXOR       X11, X6
	MOVOU      112(SI), X11
	PXOR       X11, X7
	MOVOU      X0, (DX)
	PSHUFB     X15, X0
	PXOR       X8, X0
	MOVOU      X1, 16(DX)
	PSHUFB     X15, X1
	MOVOU      X2, 32(DX)
	PSHUFB     X15, X2
	MOVOU      X3, 48(DX)
	PSHUFB     X15, X3
	MOVOU      X4, 64(DX)
	PSHUFB     X15, X4
	MOVOU      X5, 80(DX)
	PSHUFB     X15, X5
	MOVOU      X6, 96(DX)
	PSHUFB     X15, X6
	MOVOU      X7, 112(DX)
	PSHUFB     X15, X7
	MOVOU      X0, (SP)
	MOVOU      X1, 16(SP)
	MOVOU      X2, 32(SP)
	MOVOU      X3, 48(SP)
	MOVOU      X4, 64(SP)
	MOVOU      X5, 80(SP)
	MOVOU      X6, 96(SP)
	MOVOU      X7, 112(SP)
	LEAQ       128(SI), SI
	LEAQ       128(DX), DX

gcmAesEncOctetsLoop:
	CMPQ      R9, $0x80
	JB        gcmAesEncOctetsEnd
	SUBQ      $0x80, R9
	MOVOU     128(SP), X0
	MOVOU     144(SP), X1
	MOVOU     160(SP), X2
	MOVOU     176(SP), X3
	MOVOU     192(SP), X4
	MOVOU     208(SP), X5
	MOVOU     224(SP), X6
	MOVOU     240(SP), X7
	MOVOU     (SP), X11
	PSHUFD    $0x4e, X11, X12
	PXOR      X11, X12
	MOVOU     (DI), X8
	MOVOU     16(DI), X10
	MOVOU     X8, X9
	PCLMULQDQ $0x00, X12, X10
	PCLMULQDQ $0x00, X11, X8
	PCLMULQDQ $0x11, X11, X9
	MOVOU     16(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	MOVOU     32(DI), X12
	MOVOU     X12, X13
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     16(SP), X11
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PSHUFD    $0x4e, X11, X12
	PCLMULQDQ $0x11, X11, X13
	PXOR      X12, X11
	PXOR      X13, X9
	MOVOU     48(DI), X13
	PCLMULQDQ $0x00, X13, X11
	PXOR      X11, X10
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 140(SP)
	MOVOU     32(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	MOVOU     64(DI), X12
	MOVOU     X12, X13
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     32(SP), X11
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PSHUFD    $0x4e, X11, X12
	PCLMULQDQ $0x11, X11, X13
	PXOR      X12, X11
	PXOR      X13, X9
	MOVOU     80(DI), X13
	PCLMULQDQ $0x00, X13, X11
	PXOR      X11, X10
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 156(SP)
	MOVOU     48(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	MOVOU     96(DI), X12
	MOVOU     X12, X13
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     48(SP), X11
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PSHUFD    $0x4e, X11, X12
	PCLMULQDQ $0x11, X11, X13
	PXOR      X12, X11
	PXOR      X13, X9
	MOVOU     112(DI), X13
	PCLMULQDQ $0x00, X13, X11
	PXOR      X11, X10
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 172(SP)
	MOVOU     64(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	MOVOU     128(DI), X12
	MOVOU     X12, X13
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     64(SP), X11
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PSHUFD    $0x4e, X11, X12
	PCLMULQDQ $0x11, X11, X13
	PXOR      X12, X11
	PXOR      X13, X9
	MOVOU     144(DI), X13
	PCLMULQDQ $0x00, X13, X11
	PXOR      X11, X10
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 188(SP)
	MOVOU     80(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	MOVOU     160(DI), X12
	MOVOU     X12, X13
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     80(SP), X11
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PSHUFD    $0x4e, X11, X12
	PCLMULQDQ $0x11, X11, X13
	PXOR      X12, X11
	PXOR      X13, X9
	MOVOU     176(DI), X13
	PCLMULQDQ $0x00, X13, X11
	PXOR      X11, X10
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 204(SP)
	MOVOU     96(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	MOVOU     192(DI), X12
	MOVOU     X12, X13
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     96(SP), X11
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PSHUFD    $0x4e, X11, X12
	PCLMULQDQ $0x11, X11, X13
	PXOR      X12, X11
	PXOR      X13, X9
	MOVOU     208(DI), X13
	PCLMULQDQ $0x00, X13, X11
	PXOR      X11, X10
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 220(SP)
	MOVOU     112(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	MOVOU     224(DI), X12
	MOVOU     X12, X13
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     112(SP), X11
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PSHUFD    $0x4e, X11, X12
	PCLMULQDQ $0x11, X11, X13
	PXOR      X12, X11
	PXOR      X13, X9
	MOVOU     240(DI), X13
	PCLMULQDQ $0x00, X13, X11
	PXOR      X11, X10
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 236(SP)
	MOVOU     128(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 252(SP)
	PXOR      X8, X10
	PXOR      X9, X10
	MOVOU     X10, X11
	PSRLDQ    $0x08, X10
	PSLLDQ    $0x08, X11
	PXOR      X10, X9
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	MOVOU     144(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	PXOR      X9, X8
	MOVOU     160(AX), X11
	CMPQ      R13, $0x0c
	JB        encLast2
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     176(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     192(AX), X11
	JE        encLast2
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     208(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     224(AX), X11

encLast2:
	AESENCLAST X11, X0
	AESENCLAST X11, X1
	AESENCLAST X11, X2
	AESENCLAST X11, X3
	AESENCLAST X11, X4
	AESENCLAST X11, X5
	AESENCLAST X11, X6
	AESENCLAST X11, X7
	MOVOU      (SI), X11
	PXOR       X11, X0
	MOVOU      16(SI), X11
	PXOR       X11, X1
	MOVOU      32(SI), X11
	PXOR       X11, X2
	MOVOU      48(SI), X11
	PXOR       X11, X3
	MOVOU      64(SI), X11
	PXOR       X11, X4
	MOVOU      80(SI), X11
	PXOR       X11, X5
	MOVOU      96(SI), X11
	PXOR       X11, X6
	MOVOU      112(SI), X11
	PXOR       X11, X7
	MOVOU      X0, (DX)
	PSHUFB     X15, X0
	PXOR       X8, X0
	MOVOU      X1, 16(DX)
	PSHUFB     X15, X1
	MOVOU      X2, 32(DX)
	PSHUFB     X15, X2
	MOVOU      X3, 48(DX)
	PSHUFB     X15, X3
	MOVOU      X4, 64(DX)
	PSHUFB     X15, X4
	MOVOU      X5, 80(DX)
	PSHUFB     X15, X5
	MOVOU      X6, 96(DX)
	PSHUFB     X15, X6
	MOVOU      X7, 112(DX)
	PSHUFB     X15, X7
	MOVOU      X0, (SP)
	MOVOU      X1, 16(SP)
	MOVOU      X2, 32(SP)
	MOVOU      X3, 48(SP)
	MOVOU      X4, 64(SP)
	MOVOU      X5, 80(SP)
	MOVOU      X6, 96(SP)
	MOVOU      X7, 112(SP)
	LEAQ       128(SI), SI
	LEAQ       128(DX), DX
	JMP        gcmAesEncOctetsLoop

gcmAesEncOctetsEnd:
	MOVOU     (SP), X11
	MOVOU     (DI), X8
	MOVOU     16(DI), X10
	MOVOU     X8, X9
	PSHUFD    $0x4e, X11, X12
	PXOR      X11, X12
	PCLMULQDQ $0x00, X11, X8
	PCLMULQDQ $0x11, X11, X9
	PCLMULQDQ $0x00, X12, X10
	MOVOU     16(SP), X11
	MOVOU     32(DI), X12
	MOVOU     X12, X13
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PCLMULQDQ $0x11, X11, X13
	PXOR      X13, X9
	PSHUFD    $0x4e, X11, X12
	PXOR      X12, X11
	MOVOU     48(DI), X12
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X10
	MOVOU     32(SP), X11
	MOVOU     64(DI), X12
	MOVOU     X12, X13
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PCLMULQDQ $0x11, X11, X13
	PXOR      X13, X9
	PSHUFD    $0x4e, X11, X12
	PXOR      X12, X11
	MOVOU     80(DI), X12
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X10
	MOVOU     48(SP), X11
	MOVOU     96(DI), X12
	MOVOU     X12, X13
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PCLMULQDQ $0x11, X11, X13
	PXOR      X13, X9
	PSHUFD    $0x4e, X11, X12
	PXOR      X12, X11
	MOVOU     112(DI), X12
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X10
	MOVOU     64(SP), X11
	MOVOU     128(DI), X12
	MOVOU     X12, X13
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PCLMULQDQ $0x11, X11, X13
	PXOR      X13, X9
	PSHUFD    $0x4e, X11, X12
	PXOR      X12, X11
	MOVOU     144(DI), X12
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X10
	MOVOU     80(SP), X11
	MOVOU     160(DI), X12
	MOVOU     X12, X13
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PCLMULQDQ $0x11, X11, X13
	PXOR      X13, X9
	PSHUFD    $0x4e, X11, X12
	PXOR      X12, X11
	MOVOU     176(DI), X12
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X10
	MOVOU     96(SP), X11
	MOVOU     192(DI), X12
	MOVOU     X12, X13
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PCLMULQDQ $0x11, X11, X13
	PXOR      X13, X9
	PSHUFD    $0x4e, X11, X12
	PXOR      X12, X11
	MOVOU     208(DI), X12
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X10
	MOVOU     112(SP), X11
	MOVOU     224(DI), X12
	MOVOU     X12, X13
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PCLMULQDQ $0x11, X11, X13
	PXOR      X13, X9
	PSHUFD    $0x4e, X11, X12
	PXOR      X12, X11
	MOVOU     240(DI), X12
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X10
	PXOR      X8, X10
	PXOR      X9, X10
	MOVOU     X10, X11
	PSRLDQ    $0x08, X10
	PSLLDQ    $0x08, X11
	PXOR      X10, X9
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	PXOR      X9, X8
	TESTQ     R9, R9
	JE        gcmAesEncDone
	SUBQ      $0x07, R10

gcmAesEncSingles:
	MOVOU 16(AX), X1
	MOVOU 32(AX), X2
	MOVOU 48(AX), X3
	MOVOU 64(AX), X4
	MOVOU 80(AX), X5
	MOVOU 96(AX), X6
	MOVOU 112(AX), X7
	MOVOU 224(DI), X13

gcmAesEncSinglesLoop:
	CMPQ   R9, $0x10
	JB     gcmAesEncTail
	SUBQ   $0x10, R9
	MOVOU  128(SP), X0
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 140(SP)
	AESENC X1, X0
	AESENC X2, X0
	AESENC X3, X0
	AESENC X4, X0
	AESENC X5, X0
	AESENC X6, X0
	AESENC X7, X0
	MOVOU  128(AX), X11
	AESENC X11, X0
	MOVOU  144(AX), X11
	AESENC X11, X0
	MOVOU  160(AX), X11
	CMPQ   R13, $0x0c
	JB     encLast3
	AESENC X11, X0
	MOVOU  176(AX), X11
	AESENC X11, X0
	MOVOU  192(AX), X11
	JE     encLast3
	AESENC X11, X0
	MOVOU  208(AX), X11
	AESENC X11, X0
	MOVOU  224(AX), X11

encLast3:
	AESENCLAST X11, X0
	MOVOU      (SI), X11
	PXOR       X11, X0
	MOVOU      X0, (DX)
	PSHUFB     X15, X0
	PXOR       X8, X0
	MOVOU      X13, X8
	MOVOU      X13, X9
	MOVOU      240(DI), X10
	PSHUFD     $0x4e, X0, X11
	PXOR       X0, X11
	PCLMULQDQ  $0x00, X0, X8
	PCLMULQDQ  $0x11, X0, X9
	PCLMULQDQ  $0x00, X11, X10
	PXOR       X8, X10
	PXOR       X9, X10
	MOVOU      X10, X11
	PSRLDQ     $0x08, X10
	PSLLDQ     $0x08, X11
	PXOR       X10, X9
	PXOR       X11, X8
	MOVOU      X14, X11
	PCLMULQDQ  $0x01, X8, X11
	PSHUFD     $0x4e, X8, X8
	PXOR       X11, X8
	MOVOU      X14, X11
	PCLMULQDQ  $0x01, X8, X11
	PSHUFD     $0x4e, X8, X8
	PXOR       X11, X8
	PXOR       X9, X8
	LEAQ       16(SI), SI
	LEAQ       16(DX), DX
	JMP        gcmAesEncSinglesLoop

gcmAesEncTail:
	TESTQ  R9, R9
	JE     gcmAesEncDone
	MOVOU  128(SP), X0
	AESENC X1, X0
	AESENC X2, X0
	AESENC X3, X0
	AESENC X4, X0
	AESENC X5, X0
	AESENC X6, X0
	AESENC X7, X0
	MOVOU  128(AX), X11
	AESENC X11, X0
	MOVOU  144(AX), X11
	AESENC X11, X0
	MOVOU  160(AX), X11
	CMPQ   R13, $0x0c
	JB     encLast4
	AESENC X11, X0
	MOVOU  176(AX), X11
	AESENC X11, X0
	MOVOU  192(AX), X11
	JE     encLast4
	AESENC X11, X0
	MOVOU  208(AX), X11
	AESENC X11, X0
	MOVOU  224(AX), X11

encLast4:
	AESENCLAST X11, X0
	MOVOU      X0, X11
	LEAQ       -1(SI)(R9*1), SI
	MOVQ       R9, R11
	SHLQ       $0x04, R11
	LEAQ       andMask<>+0(SB), R10
	MOVOU      -16(R10)(R11*1), X12
	PXOR       X0, X0

ptxLoadLoop:
	PSLLDQ    $0x01, X0
	PINSRB    $0x00, (SI), X0
	LEAQ      -1(SI), SI
	DECQ      R9
	JNE       ptxLoadLoop
	PXOR      X11, X0
	PAND      X12, X0
	MOVOU     X0, (DX)
	PSHUFB    X15, X0
	PXOR      X8, X0
	MOVOU     X13, X8
	MOVOU     X13, X9
	MOVOU     240(DI), X10
	PSHUFD    $0x4e, X0, X11
	PXOR      X0, X11
	PCLMULQDQ $0x00, X0, X8
	PCLMULQDQ $0x11, X0, X9
	PCLMULQDQ $0x00, X11, X10
	PXOR      X8, X10
	PXOR      X9, X10
	MOVOU     X10, X11
	PSRLDQ    $0x08, X10
	PSLLDQ    $0x08, X11
	PXOR      X10, X9
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	PXOR      X9, X8

gcmAesEncDone:
	MOVOU X8, (R8)
	RET

DATA andMask<>+0(SB)/8, $0x00000000000000ff
DATA andMask<>+8(SB)/8, $0x0000000000000000
DATA andMask<>+16(SB)/8, $0x000000000000ffff
DATA andMask<>+24(SB)/8, $0x0000000000000000
DATA andMask<>+32(SB)/8, $0x0000000000ffffff
DATA andMask<>+40(SB)/8, $0x0000000000000000
DATA andMask<>+48(SB)/8, $0x00000000ffffffff
DATA andMask<>+56(SB)/8, $0x0000000000000000
DATA andMask<>+64(SB)/8, $0x000000ffffffffff
DATA andMask<>+72(SB)/8, $0x0000000000000000
DATA andMask<>+80(SB)/8, $0x0000ffffffffffff
DATA andMask<>+88(SB)/8, $0x0000000000000000
DATA andMask<>+96(SB)/8, $0x00ffffffffffffff
DATA andMask<>+104(SB)/8, $0x0000000000000000
DATA andMask<>+112(SB)/8, $0xffffffffffffffff
DATA andMask<>+120(SB)/8, $0x0000000000000000
DATA andMask<>+128(SB)/8, $0xffffffffffffffff
DATA andMask<>+136(SB)/8, $0x00000000000000ff
DATA andMask<>+144(SB)/8, $0xffffffffffffffff
DATA andMask<>+152(SB)/8, $0x000000000000ffff
DATA andMask<>+160(SB)/8, $0xffffffffffffffff
DATA andMask<>+168(SB)/8, $0x0000000000ffffff
DATA andMask<>+176(SB)/8, $0xffffffffffffffff
DATA andMask<>+184(SB)/8, $0x00000000ffffffff
DATA andMask<>+192(SB)/8, $0xffffffffffffffff
DATA andMask<>+200(SB)/8, $0x000000ffffffffff
DATA andMask<>+208(SB)/8, $0xffffffffffffffff
DATA andMask<>+216(SB)/8, $0x0000ffffffffffff
DATA andMask<>+224(SB)/8, $0xffffffffffffffff
DATA andMask<>+232(SB)/8, $0x00ffffffffffffff
GLOBL andMask<>(SB), RODATA|NOPTR, $240

// func gcmAesDec(productTable *[256]byte, dst []byte, src []byte, ctr *[16]byte, T *[16]byte, ks []uint32)
// Requires: AES, PCLMULQDQ, SSE2, SSE4.1, SSSE3
TEXT ·gcmAesDec(SB), $128-96
	MOVQ   productTable+0(FP), DI
	MOVQ   dst_base+8(FP), SI
	MOVQ   src_base+32(FP), DX
	MOVQ   src_len+40(FP), R9
	MOVQ   ctr+56(FP), CX
	MOVQ   T+64(FP), R8
	MOVQ   ks_base+72(FP), AX
	MOVQ   ks_len+80(FP), R13
	SHRQ   $0x02, R13
	DECQ   R13
	MOVOU  bswapMask<>+0(SB), X15
	MOVOU  gcmPoly<>+0(SB), X14
	MOVOU  (R8), X8
	PXOR   X9, X9
	PXOR   X10, X10
	MOVOU  (CX), X0
	MOVL   12(CX), R10
	MOVOU  (AX), X11
	MOVL   12(AX), R12
	BSWAPL R10
	BSWAPL R12
	PXOR   X0, X11
	MOVOU  X11, (SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 12(SP)
	CMPQ   R9, $0x80
	JB     gcmAesDecSingles
	MOVOU  X11, 16(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 28(SP)
	MOVOU  X11, 32(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 44(SP)
	MOVOU  X11, 48(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 60(SP)
	MOVOU  X11, 64(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 76(SP)
	MOVOU  X11, 80(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 92(SP)
	MOVOU  X11, 96(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 108(SP)
	MOVOU  X11, 112(SP)
	ADDL   $0x01, R10
	MOVL   R10, R11
	XORL   R12, R11
	BSWAPL R11
	MOVL   R11, 124(SP)

gcmAesDecOctetsLoop:
	CMPQ      R9, $0x80
	JB        gcmAesDecEndOctets
	SUBQ      $0x80, R9
	MOVOU     (SP), X0
	MOVOU     16(SP), X1
	MOVOU     32(SP), X2
	MOVOU     48(SP), X3
	MOVOU     64(SP), X4
	MOVOU     80(SP), X5
	MOVOU     96(SP), X6
	MOVOU     112(SP), X7
	MOVOU     (DX), X11
	PSHUFB    X15, X11
	PXOR      X8, X11
	PSHUFD    $0x4e, X11, X12
	PXOR      X11, X12
	MOVOU     (DI), X8
	MOVOU     16(DI), X10
	MOVOU     X8, X9
	PCLMULQDQ $0x00, X12, X10
	PCLMULQDQ $0x00, X11, X8
	PCLMULQDQ $0x11, X11, X9
	MOVOU     16(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	MOVOU     32(DI), X12
	MOVOU     X12, X13
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     16(DX), X11
	PSHUFB    X15, X11
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PSHUFD    $0x4e, X11, X12
	PCLMULQDQ $0x11, X11, X13
	PXOR      X12, X11
	PXOR      X13, X9
	MOVOU     48(DI), X13
	PCLMULQDQ $0x00, X13, X11
	PXOR      X11, X10
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 12(SP)
	MOVOU     32(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	MOVOU     64(DI), X12
	MOVOU     X12, X13
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     32(DX), X11
	PSHUFB    X15, X11
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PSHUFD    $0x4e, X11, X12
	PCLMULQDQ $0x11, X11, X13
	PXOR      X12, X11
	PXOR      X13, X9
	MOVOU     80(DI), X13
	PCLMULQDQ $0x00, X13, X11
	PXOR      X11, X10
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 28(SP)
	MOVOU     48(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	MOVOU     96(DI), X12
	MOVOU     X12, X13
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     48(DX), X11
	PSHUFB    X15, X11
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PSHUFD    $0x4e, X11, X12
	PCLMULQDQ $0x11, X11, X13
	PXOR      X12, X11
	PXOR      X13, X9
	MOVOU     112(DI), X13
	PCLMULQDQ $0x00, X13, X11
	PXOR      X11, X10
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 44(SP)
	MOVOU     64(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	MOVOU     128(DI), X12
	MOVOU     X12, X13
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     64(DX), X11
	PSHUFB    X15, X11
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PSHUFD    $0x4e, X11, X12
	PCLMULQDQ $0x11, X11, X13
	PXOR      X12, X11
	PXOR      X13, X9
	MOVOU     144(DI), X13
	PCLMULQDQ $0x00, X13, X11
	PXOR      X11, X10
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 60(SP)
	MOVOU     80(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	MOVOU     160(DI), X12
	MOVOU     X12, X13
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     80(DX), X11
	PSHUFB    X15, X11
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PSHUFD    $0x4e, X11, X12
	PCLMULQDQ $0x11, X11, X13
	PXOR      X12, X11
	PXOR      X13, X9
	MOVOU     176(DI), X13
	PCLMULQDQ $0x00, X13, X11
	PXOR      X11, X10
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 76(SP)
	MOVOU     96(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	MOVOU     192(DI), X12
	MOVOU     X12, X13
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     96(DX), X11
	PSHUFB    X15, X11
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PSHUFD    $0x4e, X11, X12
	PCLMULQDQ $0x11, X11, X13
	PXOR      X12, X11
	PXOR      X13, X9
	MOVOU     208(DI), X13
	PCLMULQDQ $0x00, X13, X11
	PXOR      X11, X10
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 92(SP)
	MOVOU     112(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	MOVOU     224(DI), X12
	MOVOU     X12, X13
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     112(DX), X11
	PSHUFB    X15, X11
	PCLMULQDQ $0x00, X11, X12
	PXOR      X12, X8
	PSHUFD    $0x4e, X11, X12
	PCLMULQDQ $0x11, X11, X13
	PXOR      X12, X11
	PXOR      X13, X9
	MOVOU     240(DI), X13
	PCLMULQDQ $0x00, X13, X11
	PXOR      X11, X10
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 108(SP)
	MOVOU     128(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 124(SP)
	PXOR      X8, X10
	PXOR      X9, X10
	MOVOU     X10, X11
	PSRLDQ    $0x08, X10
	PSLLDQ    $0x08, X11
	PXOR      X10, X9
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	MOVOU     144(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	PXOR      X9, X8
	MOVOU     160(AX), X11
	CMPQ      R13, $0x0c
	JB        decLast1
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     176(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     192(AX), X11
	JE        decLast1
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     208(AX), X11
	AESENC    X11, X0
	AESENC    X11, X1
	AESENC    X11, X2
	AESENC    X11, X3
	AESENC    X11, X4
	AESENC    X11, X5
	AESENC    X11, X6
	AESENC    X11, X7
	MOVOU     224(AX), X11

decLast1:
	AESENCLAST X11, X0
	AESENCLAST X11, X1
	AESENCLAST X11, X2
	AESENCLAST X11, X3
	AESENCLAST X11, X4
	AESENCLAST X11, X5
	AESENCLAST X11, X6
	AESENCLAST X11, X7
	MOVOU      (DX), X11
	PXOR       X11, X0
	MOVOU      16(DX), X11
	PXOR       X11, X1
	MOVOU      32(DX), X11
	PXOR       X11, X2
	MOVOU      48(DX), X11
	PXOR       X11, X3
	MOVOU      64(DX), X11
	PXOR       X11, X4
	MOVOU      80(DX), X11
	PXOR       X11, X5
	MOVOU      96(DX), X11
	PXOR       X11, X6
	MOVOU      112(DX), X11
	PXOR       X11, X7
	MOVOU      X0, (SI)
	MOVOU      X1, 16(SI)
	MOVOU      X2, 32(SI)
	MOVOU      X3, 48(SI)
	MOVOU      X4, 64(SI)
	MOVOU      X5, 80(SI)
	MOVOU      X6, 96(SI)
	MOVOU      X7, 112(SI)
	LEAQ       128(SI), SI
	LEAQ       128(DX), DX
	JMP        gcmAesDecOctetsLoop

gcmAesDecEndOctets:
	SUBQ $0x07, R10

gcmAesDecSingles:
	MOVOU 16(AX), X1
	MOVOU 32(AX), X2
	MOVOU 48(AX), X3
	MOVOU 64(AX), X4
	MOVOU 80(AX), X5
	MOVOU 96(AX), X6
	MOVOU 112(AX), X7
	MOVOU 224(DI), X13

gcmAesDecSinglesLoop:
	CMPQ      R9, $0x10
	JB        gcmAesDecTail
	SUBQ      $0x10, R9
	MOVOU     (DX), X0
	MOVOU     X0, X12
	PSHUFB    X15, X0
	PXOR      X8, X0
	MOVOU     X13, X8
	MOVOU     X13, X9
	MOVOU     240(DI), X10
	PCLMULQDQ $0x00, X0, X8
	PCLMULQDQ $0x11, X0, X9
	PSHUFD    $0x4e, X0, X11
	PXOR      X0, X11
	PCLMULQDQ $0x00, X11, X10
	PXOR      X8, X10
	PXOR      X9, X10
	MOVOU     X10, X11
	PSRLDQ    $0x08, X10
	PSLLDQ    $0x08, X11
	PXOR      X10, X9
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	PXOR      X9, X8
	MOVOU     (SP), X0
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 12(SP)
	AESENC    X1, X0
	AESENC    X2, X0
	AESENC    X3, X0
	AESENC    X4, X0
	AESENC    X5, X0
	AESENC    X6, X0
	AESENC    X7, X0
	MOVOU     128(AX), X11
	AESENC    X11, X0
	MOVOU     144(AX), X11
	AESENC    X11, X0
	MOVOU     160(AX), X11
	CMPQ      R13, $0x0c
	JB        decLast2
	AESENC    X11, X0
	MOVOU     176(AX), X11
	AESENC    X11, X0
	MOVOU     192(AX), X11
	JE        decLast2
	AESENC    X11, X0
	MOVOU     208(AX), X11
	AESENC    X11, X0
	MOVOU     224(AX), X11

decLast2:
	AESENCLAST X11, X0
	PXOR       X12, X0
	MOVOU      X0, (SI)
	LEAQ       16(SI), SI
	LEAQ       16(DX), DX
	JMP        gcmAesDecSinglesLoop

gcmAesDecTail:
	TESTQ     R9, R9
	JE        gcmAesDecDone
	MOVQ      R9, R11
	SHLQ      $0x04, R11
	LEAQ      andMask<>+0(SB), R10
	MOVOU     -16(R10)(R11*1), X12
	MOVOU     (DX), X0
	PAND      X12, X0
	MOVOU     X0, X12
	PSHUFB    X15, X0
	PXOR      X8, X0
	MOVOU     224(DI), X8
	MOVOU     240(DI), X10
	MOVOU     X8, X9
	PCLMULQDQ $0x00, X0, X8
	PCLMULQDQ $0x11, X0, X9
	PSHUFD    $0x4e, X0, X11
	PXOR      X0, X11
	PCLMULQDQ $0x00, X11, X10
	PXOR      X8, X10
	PXOR      X9, X10
	MOVOU     X10, X11
	PSRLDQ    $0x08, X10
	PSLLDQ    $0x08, X11
	PXOR      X10, X9
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	MOVOU     X14, X11
	PCLMULQDQ $0x01, X8, X11
	PSHUFD    $0x4e, X8, X8
	PXOR      X11, X8
	PXOR      X9, X8
	MOVOU     (SP), X0
	ADDL      $0x01, R10
	MOVL      R10, R11
	XORL      R12, R11
	BSWAPL    R11
	MOVL      R11, 12(SP)
	AESENC    X1, X0
	AESENC    X2, X0
	AESENC    X3, X0
	AESENC    X4, X0
	AESENC    X5, X0
	AESENC    X6, X0
	AESENC    X7, X0
	MOVOU     128(AX), X11
	AESENC    X11, X0
	MOVOU     144(AX), X11
	AESENC    X11, X0
	MOVOU     160(AX), X11
	CMPQ      R13, $0x0c
	JB        decLast3
	AESENC    X11, X0
	MOVOU     176(AX), X11
	AESENC    X11, X0
	MOVOU     192(AX), X11
	JE        decLast3
	AESENC    X11, X0
	MOVOU     208(AX), X11
	AESENC    X11, X0
	MOVOU     224(AX), X11

decLast3:
	AESENCLAST X11, X0
	PXOR       X12, X0

ptxStoreLoop:
	PEXTRB $0x00, X0, (SI)
	PSRLDQ $0x01, X0
	LEAQ   1(SI), SI
	DECQ   R9
	JNE    ptxStoreLoop

gcmAesDecDone:
	MOVOU X8, (R8)
	RET
