// Code generated by command: go run ctr_amd64_asm.go -out ../../ctr_amd64.s. DO NOT EDIT.

//go:build !purego

#include "textflag.h"

// func ctrBlocks1Asm(nr int, xk *[60]uint32, dst *[16]byte, src *[16]byte, ivlo uint64, ivhi uint64)
// Requires: AES, SSE, SSE2, SSE4.1, SSSE3
TEXT ·ctrBlocks1Asm(SB), $0-48
	MOVQ   nr+0(FP), AX
	MOVQ   xk+8(FP), CX
	MOVQ   dst+16(FP), DX
	MOVQ   src+24(FP), BX
	MOVQ   ivlo+32(FP), SI
	MOVQ   ivhi+40(FP), DI
	MOVOU  bswapMask<>+0(SB), X0
	MOVQ   SI, X1
	PINSRQ $0x01, DI, X1
	PSHUFB X0, X1
	MOVUPS (CX), X0
	PXOR   X0, X1
	ADDQ   $0x10, CX
	SUBQ   $0x0c, AX
	JE     enc192
	JB     enc128
	MOVUPS (CX), X0
	AESENC X0, X1
	MOVUPS 16(CX), X0
	AESENC X0, X1
	ADDQ   $0x20, CX

enc192:
	MOVUPS (CX), X0
	AESENC X0, X1
	MOVUPS 16(CX), X0
	AESENC X0, X1
	ADDQ   $0x20, CX

enc128:
	MOVUPS     (CX), X0
	AESENC     X0, X1
	MOVUPS     16(CX), X0
	AESENC     X0, X1
	MOVUPS     32(CX), X0
	AESENC     X0, X1
	MOVUPS     48(CX), X0
	AESENC     X0, X1
	MOVUPS     64(CX), X0
	AESENC     X0, X1
	MOVUPS     80(CX), X0
	AESENC     X0, X1
	MOVUPS     96(CX), X0
	AESENC     X0, X1
	MOVUPS     112(CX), X0
	AESENC     X0, X1
	MOVUPS     128(CX), X0
	AESENC     X0, X1
	MOVUPS     144(CX), X0
	AESENCLAST X0, X1
	MOVUPS     (BX), X0
	PXOR       X1, X0
	MOVUPS     X0, (DX)
	RET

DATA bswapMask<>+0(SB)/8, $0x08090a0b0c0d0e0f
DATA bswapMask<>+8(SB)/8, $0x0001020304050607
GLOBL bswapMask<>(SB), RODATA|NOPTR, $16

// func ctrBlocks2Asm(nr int, xk *[60]uint32, dst *[32]byte, src *[32]byte, ivlo uint64, ivhi uint64)
// Requires: AES, SSE, SSE2, SSE4.1, SSSE3
TEXT ·ctrBlocks2Asm(SB), $0-48
	MOVQ   nr+0(FP), AX
	MOVQ   xk+8(FP), CX
	MOVQ   dst+16(FP), DX
	MOVQ   src+24(FP), BX
	MOVQ   ivlo+32(FP), SI
	MOVQ   ivhi+40(FP), DI
	MOVOU  bswapMask<>+0(SB), X0
	MOVQ   SI, X1
	PINSRQ $0x01, DI, X1
	PSHUFB X0, X1
	ADDQ   $0x01, SI
	ADCQ   $0x00, DI
	MOVQ   SI, X2
	PINSRQ $0x01, DI, X2
	PSHUFB X0, X2
	MOVUPS (CX), X0
	PXOR   X0, X1
	PXOR   X0, X2
	ADDQ   $0x10, CX
	SUBQ   $0x0c, AX
	JE     enc192
	JB     enc128
	MOVUPS (CX), X0
	AESENC X0, X1
	AESENC X0, X2
	MOVUPS 16(CX), X0
	AESENC X0, X1
	AESENC X0, X2
	ADDQ   $0x20, CX

enc192:
	MOVUPS (CX), X0
	AESENC X0, X1
	AESENC X0, X2
	MOVUPS 16(CX), X0
	AESENC X0, X1
	AESENC X0, X2
	ADDQ   $0x20, CX

enc128:
	MOVUPS     (CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	MOVUPS     16(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	MOVUPS     32(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	MOVUPS     48(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	MOVUPS     64(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	MOVUPS     80(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	MOVUPS     96(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	MOVUPS     112(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	MOVUPS     128(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	MOVUPS     144(CX), X0
	AESENCLAST X0, X1
	AESENCLAST X0, X2
	MOVUPS     (BX), X0
	PXOR       X1, X0
	MOVUPS     X0, (DX)
	MOVUPS     16(BX), X0
	PXOR       X2, X0
	MOVUPS     X0, 16(DX)
	RET

// func ctrBlocks4Asm(nr int, xk *[60]uint32, dst *[64]byte, src *[64]byte, ivlo uint64, ivhi uint64)
// Requires: AES, SSE, SSE2, SSE4.1, SSSE3
TEXT ·ctrBlocks4Asm(SB), $0-48
	MOVQ   nr+0(FP), AX
	MOVQ   xk+8(FP), CX
	MOVQ   dst+16(FP), DX
	MOVQ   src+24(FP), BX
	MOVQ   ivlo+32(FP), SI
	MOVQ   ivhi+40(FP), DI
	MOVOU  bswapMask<>+0(SB), X0
	MOVQ   SI, X1
	PINSRQ $0x01, DI, X1
	PSHUFB X0, X1
	ADDQ   $0x01, SI
	ADCQ   $0x00, DI
	MOVQ   SI, X2
	PINSRQ $0x01, DI, X2
	PSHUFB X0, X2
	ADDQ   $0x01, SI
	ADCQ   $0x00, DI
	MOVQ   SI, X3
	PINSRQ $0x01, DI, X3
	PSHUFB X0, X3
	ADDQ   $0x01, SI
	ADCQ   $0x00, DI
	MOVQ   SI, X4
	PINSRQ $0x01, DI, X4
	PSHUFB X0, X4
	MOVUPS (CX), X0
	PXOR   X0, X1
	PXOR   X0, X2
	PXOR   X0, X3
	PXOR   X0, X4
	ADDQ   $0x10, CX
	SUBQ   $0x0c, AX
	JE     enc192
	JB     enc128
	MOVUPS (CX), X0
	AESENC X0, X1
	AESENC X0, X2
	AESENC X0, X3
	AESENC X0, X4
	MOVUPS 16(CX), X0
	AESENC X0, X1
	AESENC X0, X2
	AESENC X0, X3
	AESENC X0, X4
	ADDQ   $0x20, CX

enc192:
	MOVUPS (CX), X0
	AESENC X0, X1
	AESENC X0, X2
	AESENC X0, X3
	AESENC X0, X4
	MOVUPS 16(CX), X0
	AESENC X0, X1
	AESENC X0, X2
	AESENC X0, X3
	AESENC X0, X4
	ADDQ   $0x20, CX

enc128:
	MOVUPS     (CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	MOVUPS     16(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	MOVUPS     32(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	MOVUPS     48(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	MOVUPS     64(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	MOVUPS     80(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	MOVUPS     96(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	MOVUPS     112(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	MOVUPS     128(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	MOVUPS     144(CX), X0
	AESENCLAST X0, X1
	AESENCLAST X0, X2
	AESENCLAST X0, X3
	AESENCLAST X0, X4
	MOVUPS     (BX), X0
	PXOR       X1, X0
	MOVUPS     X0, (DX)
	MOVUPS     16(BX), X0
	PXOR       X2, X0
	MOVUPS     X0, 16(DX)
	MOVUPS     32(BX), X0
	PXOR       X3, X0
	MOVUPS     X0, 32(DX)
	MOVUPS     48(BX), X0
	PXOR       X4, X0
	MOVUPS     X0, 48(DX)
	RET

// func ctrBlocks8Asm(nr int, xk *[60]uint32, dst *[128]byte, src *[128]byte, ivlo uint64, ivhi uint64)
// Requires: AES, SSE, SSE2, SSE4.1, SSSE3
TEXT ·ctrBlocks8Asm(SB), $0-48
	MOVQ   nr+0(FP), AX
	MOVQ   xk+8(FP), CX
	MOVQ   dst+16(FP), DX
	MOVQ   src+24(FP), BX
	MOVQ   ivlo+32(FP), SI
	MOVQ   ivhi+40(FP), DI
	MOVOU  bswapMask<>+0(SB), X0
	MOVQ   SI, X1
	PINSRQ $0x01, DI, X1
	MOVAPS X1, X8
	PSHUFB X0, X1
	MOVQ   SI, R8
	ADDQ   $0x07, R8
	JC     ctr8_slow
	XORQ   R8, R8
	INCQ   R8
	PXOR   X9, X9
	PINSRQ $0x00, R8, X9
	PADDQ  X9, X8
	MOVAPS X8, X2
	PADDQ  X9, X8
	MOVAPS X8, X3
	PADDQ  X9, X8
	MOVAPS X8, X4
	PADDQ  X9, X8
	MOVAPS X8, X5
	PADDQ  X9, X8
	MOVAPS X8, X6
	PADDQ  X9, X8
	MOVAPS X8, X7
	PADDQ  X9, X8
	MOVAPS X8, X8
	JMP    ctr8_done

ctr8_slow:
	ADDQ   $0x01, SI
	ADCQ   $0x00, DI
	MOVQ   SI, X2
	PINSRQ $0x01, DI, X2
	ADDQ   $0x01, SI
	ADCQ   $0x00, DI
	MOVQ   SI, X3
	PINSRQ $0x01, DI, X3
	ADDQ   $0x01, SI
	ADCQ   $0x00, DI
	MOVQ   SI, X4
	PINSRQ $0x01, DI, X4
	ADDQ   $0x01, SI
	ADCQ   $0x00, DI
	MOVQ   SI, X5
	PINSRQ $0x01, DI, X5
	ADDQ   $0x01, SI
	ADCQ   $0x00, DI
	MOVQ   SI, X6
	PINSRQ $0x01, DI, X6
	ADDQ   $0x01, SI
	ADCQ   $0x00, DI
	MOVQ   SI, X7
	PINSRQ $0x01, DI, X7
	ADDQ   $0x01, SI
	ADCQ   $0x00, DI
	MOVQ   SI, X8
	PINSRQ $0x01, DI, X8

ctr8_done:
	PSHUFB X0, X2
	PSHUFB X0, X3
	PSHUFB X0, X4
	PSHUFB X0, X5
	PSHUFB X0, X6
	PSHUFB X0, X7
	PSHUFB X0, X8
	MOVUPS (CX), X0
	PXOR   X0, X1
	PXOR   X0, X2
	PXOR   X0, X3
	PXOR   X0, X4
	PXOR   X0, X5
	PXOR   X0, X6
	PXOR   X0, X7
	PXOR   X0, X8
	ADDQ   $0x10, CX
	SUBQ   $0x0c, AX
	JE     enc192
	JB     enc128
	MOVUPS (CX), X0
	AESENC X0, X1
	AESENC X0, X2
	AESENC X0, X3
	AESENC X0, X4
	AESENC X0, X5
	AESENC X0, X6
	AESENC X0, X7
	AESENC X0, X8
	MOVUPS 16(CX), X0
	AESENC X0, X1
	AESENC X0, X2
	AESENC X0, X3
	AESENC X0, X4
	AESENC X0, X5
	AESENC X0, X6
	AESENC X0, X7
	AESENC X0, X8
	ADDQ   $0x20, CX

enc192:
	MOVUPS (CX), X0
	AESENC X0, X1
	AESENC X0, X2
	AESENC X0, X3
	AESENC X0, X4
	AESENC X0, X5
	AESENC X0, X6
	AESENC X0, X7
	AESENC X0, X8
	MOVUPS 16(CX), X0
	AESENC X0, X1
	AESENC X0, X2
	AESENC X0, X3
	AESENC X0, X4
	AESENC X0, X5
	AESENC X0, X6
	AESENC X0, X7
	AESENC X0, X8
	ADDQ   $0x20, CX

enc128:
	MOVUPS     (CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	AESENC     X0, X5
	AESENC     X0, X6
	AESENC     X0, X7
	AESENC     X0, X8
	MOVUPS     16(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	AESENC     X0, X5
	AESENC     X0, X6
	AESENC     X0, X7
	AESENC     X0, X8
	MOVUPS     32(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	AESENC     X0, X5
	AESENC     X0, X6
	AESENC     X0, X7
	AESENC     X0, X8
	MOVUPS     48(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	AESENC     X0, X5
	AESENC     X0, X6
	AESENC     X0, X7
	AESENC     X0, X8
	MOVUPS     64(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	AESENC     X0, X5
	AESENC     X0, X6
	AESENC     X0, X7
	AESENC     X0, X8
	MOVUPS     80(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	AESENC     X0, X5
	AESENC     X0, X6
	AESENC     X0, X7
	AESENC     X0, X8
	MOVUPS     96(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	AESENC     X0, X5
	AESENC     X0, X6
	AESENC     X0, X7
	AESENC     X0, X8
	MOVUPS     112(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	AESENC     X0, X5
	AESENC     X0, X6
	AESENC     X0, X7
	AESENC     X0, X8
	MOVUPS     128(CX), X0
	AESENC     X0, X1
	AESENC     X0, X2
	AESENC     X0, X3
	AESENC     X0, X4
	AESENC     X0, X5
	AESENC     X0, X6
	AESENC     X0, X7
	AESENC     X0, X8
	MOVUPS     144(CX), X0
	AESENCLAST X0, X1
	AESENCLAST X0, X2
	AESENCLAST X0, X3
	AESENCLAST X0, X4
	AESENCLAST X0, X5
	AESENCLAST X0, X6
	AESENCLAST X0, X7
	AESENCLAST X0, X8
	MOVUPS     (BX), X0
	PXOR       X1, X0
	MOVUPS     X0, (DX)
	MOVUPS     16(BX), X0
	PXOR       X2, X0
	MOVUPS     X0, 16(DX)
	MOVUPS     32(BX), X0
	PXOR       X3, X0
	MOVUPS     X0, 32(DX)
	MOVUPS     48(BX), X0
	PXOR       X4, X0
	MOVUPS     X0, 48(DX)
	MOVUPS     64(BX), X0
	PXOR       X5, X0
	MOVUPS     X0, 64(DX)
	MOVUPS     80(BX), X0
	PXOR       X6, X0
	MOVUPS     X0, 80(DX)
	MOVUPS     96(BX), X0
	PXOR       X7, X0
	MOVUPS     X0, 96(DX)
	MOVUPS     112(BX), X0
	PXOR       X8, X0
	MOVUPS     X0, 112(DX)
	RET
