867 lines
21 KiB
ArmAsm
867 lines
21 KiB
ArmAsm
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
||
|
|
/*
|
||
|
|
* linux/arch/arm64/crypto/aes-modes.S - chaining mode wrappers for AES
|
||
|
|
*
|
||
|
|
* Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
|
||
|
|
*/
|
||
|
|
|
||
|
|
/* included by aes-ce.S and aes-neon.S */
|
||
|
|
|
||
|
|
.text
|
||
|
|
.align 4
|
||
|
|
|
||
|
|
#ifndef MAX_STRIDE
|
||
|
|
#define MAX_STRIDE 4
|
||
|
|
#endif
|
||
|
|
|
||
|
|
#if MAX_STRIDE == 4
|
||
|
|
#define ST4(x...) x
|
||
|
|
#define ST5(x...)
|
||
|
|
#else
|
||
|
|
#define ST4(x...)
|
||
|
|
#define ST5(x...) x
|
||
|
|
#endif
|
||
|
|
|
||
|
|
SYM_FUNC_START_LOCAL(aes_encrypt_block4x)
|
||
|
|
encrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7
|
||
|
|
ret
|
||
|
|
SYM_FUNC_END(aes_encrypt_block4x)
|
||
|
|
|
||
|
|
SYM_FUNC_START_LOCAL(aes_decrypt_block4x)
|
||
|
|
decrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7
|
||
|
|
ret
|
||
|
|
SYM_FUNC_END(aes_decrypt_block4x)
|
||
|
|
|
||
|
|
#if MAX_STRIDE == 5
|
||
|
|
SYM_FUNC_START_LOCAL(aes_encrypt_block5x)
|
||
|
|
encrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7
|
||
|
|
ret
|
||
|
|
SYM_FUNC_END(aes_encrypt_block5x)
|
||
|
|
|
||
|
|
SYM_FUNC_START_LOCAL(aes_decrypt_block5x)
|
||
|
|
decrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7
|
||
|
|
ret
|
||
|
|
SYM_FUNC_END(aes_decrypt_block5x)
|
||
|
|
#endif
|
||
|
|
|
||
|
|
/*
|
||
|
|
* aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
|
||
|
|
* int blocks)
|
||
|
|
* aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
|
||
|
|
* int blocks)
|
||
|
|
*/
|
||
|
|
|
||
|
|
AES_FUNC_START(aes_ecb_encrypt)
|
||
|
|
frame_push 0
|
||
|
|
|
||
|
|
enc_prepare w3, x2, x5
|
||
|
|
|
||
|
|
.LecbencloopNx:
|
||
|
|
subs w4, w4, #MAX_STRIDE
|
||
|
|
bmi .Lecbenc1x
|
||
|
|
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
|
||
|
|
ST4( bl aes_encrypt_block4x )
|
||
|
|
ST5( ld1 {v4.16b}, [x1], #16 )
|
||
|
|
ST5( bl aes_encrypt_block5x )
|
||
|
|
st1 {v0.16b-v3.16b}, [x0], #64
|
||
|
|
ST5( st1 {v4.16b}, [x0], #16 )
|
||
|
|
b .LecbencloopNx
|
||
|
|
.Lecbenc1x:
|
||
|
|
adds w4, w4, #MAX_STRIDE
|
||
|
|
beq .Lecbencout
|
||
|
|
.Lecbencloop:
|
||
|
|
ld1 {v0.16b}, [x1], #16 /* get next pt block */
|
||
|
|
encrypt_block v0, w3, x2, x5, w6
|
||
|
|
st1 {v0.16b}, [x0], #16
|
||
|
|
subs w4, w4, #1
|
||
|
|
bne .Lecbencloop
|
||
|
|
.Lecbencout:
|
||
|
|
frame_pop
|
||
|
|
ret
|
||
|
|
AES_FUNC_END(aes_ecb_encrypt)
|
||
|
|
|
||
|
|
|
||
|
|
AES_FUNC_START(aes_ecb_decrypt)
|
||
|
|
frame_push 0
|
||
|
|
|
||
|
|
dec_prepare w3, x2, x5
|
||
|
|
|
||
|
|
.LecbdecloopNx:
|
||
|
|
subs w4, w4, #MAX_STRIDE
|
||
|
|
bmi .Lecbdec1x
|
||
|
|
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
|
||
|
|
ST4( bl aes_decrypt_block4x )
|
||
|
|
ST5( ld1 {v4.16b}, [x1], #16 )
|
||
|
|
ST5( bl aes_decrypt_block5x )
|
||
|
|
st1 {v0.16b-v3.16b}, [x0], #64
|
||
|
|
ST5( st1 {v4.16b}, [x0], #16 )
|
||
|
|
b .LecbdecloopNx
|
||
|
|
.Lecbdec1x:
|
||
|
|
adds w4, w4, #MAX_STRIDE
|
||
|
|
beq .Lecbdecout
|
||
|
|
.Lecbdecloop:
|
||
|
|
ld1 {v0.16b}, [x1], #16 /* get next ct block */
|
||
|
|
decrypt_block v0, w3, x2, x5, w6
|
||
|
|
st1 {v0.16b}, [x0], #16
|
||
|
|
subs w4, w4, #1
|
||
|
|
bne .Lecbdecloop
|
||
|
|
.Lecbdecout:
|
||
|
|
frame_pop
|
||
|
|
ret
|
||
|
|
AES_FUNC_END(aes_ecb_decrypt)
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
|
||
|
|
* int blocks, u8 iv[])
|
||
|
|
* aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
|
||
|
|
* int blocks, u8 iv[])
|
||
|
|
* aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],
|
||
|
|
* int rounds, int blocks, u8 iv[],
|
||
|
|
* u32 const rk2[]);
|
||
|
|
* aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],
|
||
|
|
* int rounds, int blocks, u8 iv[],
|
||
|
|
* u32 const rk2[]);
|
||
|
|
*/
|
||
|
|
|
||
|
|
AES_FUNC_START(aes_essiv_cbc_encrypt)
|
||
|
|
ld1 {v4.16b}, [x5] /* get iv */
|
||
|
|
|
||
|
|
mov w8, #14 /* AES-256: 14 rounds */
|
||
|
|
enc_prepare w8, x6, x7
|
||
|
|
encrypt_block v4, w8, x6, x7, w9
|
||
|
|
enc_switch_key w3, x2, x6
|
||
|
|
b .Lcbcencloop4x
|
||
|
|
|
||
|
|
AES_FUNC_START(aes_cbc_encrypt)
|
||
|
|
ld1 {v4.16b}, [x5] /* get iv */
|
||
|
|
enc_prepare w3, x2, x6
|
||
|
|
|
||
|
|
.Lcbcencloop4x:
|
||
|
|
subs w4, w4, #4
|
||
|
|
bmi .Lcbcenc1x
|
||
|
|
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
|
||
|
|
eor v0.16b, v0.16b, v4.16b /* ..and xor with iv */
|
||
|
|
encrypt_block v0, w3, x2, x6, w7
|
||
|
|
eor v1.16b, v1.16b, v0.16b
|
||
|
|
encrypt_block v1, w3, x2, x6, w7
|
||
|
|
eor v2.16b, v2.16b, v1.16b
|
||
|
|
encrypt_block v2, w3, x2, x6, w7
|
||
|
|
eor v3.16b, v3.16b, v2.16b
|
||
|
|
encrypt_block v3, w3, x2, x6, w7
|
||
|
|
st1 {v0.16b-v3.16b}, [x0], #64
|
||
|
|
mov v4.16b, v3.16b
|
||
|
|
b .Lcbcencloop4x
|
||
|
|
.Lcbcenc1x:
|
||
|
|
adds w4, w4, #4
|
||
|
|
beq .Lcbcencout
|
||
|
|
.Lcbcencloop:
|
||
|
|
ld1 {v0.16b}, [x1], #16 /* get next pt block */
|
||
|
|
eor v4.16b, v4.16b, v0.16b /* ..and xor with iv */
|
||
|
|
encrypt_block v4, w3, x2, x6, w7
|
||
|
|
st1 {v4.16b}, [x0], #16
|
||
|
|
subs w4, w4, #1
|
||
|
|
bne .Lcbcencloop
|
||
|
|
.Lcbcencout:
|
||
|
|
st1 {v4.16b}, [x5] /* return iv */
|
||
|
|
ret
|
||
|
|
AES_FUNC_END(aes_cbc_encrypt)
|
||
|
|
AES_FUNC_END(aes_essiv_cbc_encrypt)
|
||
|
|
|
||
|
|
AES_FUNC_START(aes_essiv_cbc_decrypt)
|
||
|
|
ld1 {cbciv.16b}, [x5] /* get iv */
|
||
|
|
|
||
|
|
mov w8, #14 /* AES-256: 14 rounds */
|
||
|
|
enc_prepare w8, x6, x7
|
||
|
|
encrypt_block cbciv, w8, x6, x7, w9
|
||
|
|
b .Lessivcbcdecstart
|
||
|
|
|
||
|
|
AES_FUNC_START(aes_cbc_decrypt)
|
||
|
|
ld1 {cbciv.16b}, [x5] /* get iv */
|
||
|
|
.Lessivcbcdecstart:
|
||
|
|
frame_push 0
|
||
|
|
dec_prepare w3, x2, x6
|
||
|
|
|
||
|
|
.LcbcdecloopNx:
|
||
|
|
subs w4, w4, #MAX_STRIDE
|
||
|
|
bmi .Lcbcdec1x
|
||
|
|
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
|
||
|
|
#if MAX_STRIDE == 5
|
||
|
|
ld1 {v4.16b}, [x1], #16 /* get 1 ct block */
|
||
|
|
mov v5.16b, v0.16b
|
||
|
|
mov v6.16b, v1.16b
|
||
|
|
mov v7.16b, v2.16b
|
||
|
|
bl aes_decrypt_block5x
|
||
|
|
sub x1, x1, #32
|
||
|
|
eor v0.16b, v0.16b, cbciv.16b
|
||
|
|
eor v1.16b, v1.16b, v5.16b
|
||
|
|
ld1 {v5.16b}, [x1], #16 /* reload 1 ct block */
|
||
|
|
ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */
|
||
|
|
eor v2.16b, v2.16b, v6.16b
|
||
|
|
eor v3.16b, v3.16b, v7.16b
|
||
|
|
eor v4.16b, v4.16b, v5.16b
|
||
|
|
#else
|
||
|
|
mov v4.16b, v0.16b
|
||
|
|
mov v5.16b, v1.16b
|
||
|
|
mov v6.16b, v2.16b
|
||
|
|
bl aes_decrypt_block4x
|
||
|
|
sub x1, x1, #16
|
||
|
|
eor v0.16b, v0.16b, cbciv.16b
|
||
|
|
eor v1.16b, v1.16b, v4.16b
|
||
|
|
ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */
|
||
|
|
eor v2.16b, v2.16b, v5.16b
|
||
|
|
eor v3.16b, v3.16b, v6.16b
|
||
|
|
#endif
|
||
|
|
st1 {v0.16b-v3.16b}, [x0], #64
|
||
|
|
ST5( st1 {v4.16b}, [x0], #16 )
|
||
|
|
b .LcbcdecloopNx
|
||
|
|
.Lcbcdec1x:
|
||
|
|
adds w4, w4, #MAX_STRIDE
|
||
|
|
beq .Lcbcdecout
|
||
|
|
.Lcbcdecloop:
|
||
|
|
ld1 {v1.16b}, [x1], #16 /* get next ct block */
|
||
|
|
mov v0.16b, v1.16b /* ...and copy to v0 */
|
||
|
|
decrypt_block v0, w3, x2, x6, w7
|
||
|
|
eor v0.16b, v0.16b, cbciv.16b /* xor with iv => pt */
|
||
|
|
mov cbciv.16b, v1.16b /* ct is next iv */
|
||
|
|
st1 {v0.16b}, [x0], #16
|
||
|
|
subs w4, w4, #1
|
||
|
|
bne .Lcbcdecloop
|
||
|
|
.Lcbcdecout:
|
||
|
|
st1 {cbciv.16b}, [x5] /* return iv */
|
||
|
|
frame_pop
|
||
|
|
ret
|
||
|
|
AES_FUNC_END(aes_cbc_decrypt)
|
||
|
|
AES_FUNC_END(aes_essiv_cbc_decrypt)
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
|
||
|
|
* int rounds, int bytes, u8 const iv[])
|
||
|
|
* aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
|
||
|
|
* int rounds, int bytes, u8 const iv[])
|
||
|
|
*/
|
||
|
|
|
||
|
|
AES_FUNC_START(aes_cbc_cts_encrypt)
|
||
|
|
adr_l x8, .Lcts_permute_table
|
||
|
|
sub x4, x4, #16
|
||
|
|
add x9, x8, #32
|
||
|
|
add x8, x8, x4
|
||
|
|
sub x9, x9, x4
|
||
|
|
ld1 {v3.16b}, [x8]
|
||
|
|
ld1 {v4.16b}, [x9]
|
||
|
|
|
||
|
|
ld1 {v0.16b}, [x1], x4 /* overlapping loads */
|
||
|
|
ld1 {v1.16b}, [x1]
|
||
|
|
|
||
|
|
ld1 {v5.16b}, [x5] /* get iv */
|
||
|
|
enc_prepare w3, x2, x6
|
||
|
|
|
||
|
|
eor v0.16b, v0.16b, v5.16b /* xor with iv */
|
||
|
|
tbl v1.16b, {v1.16b}, v4.16b
|
||
|
|
encrypt_block v0, w3, x2, x6, w7
|
||
|
|
|
||
|
|
eor v1.16b, v1.16b, v0.16b
|
||
|
|
tbl v0.16b, {v0.16b}, v3.16b
|
||
|
|
encrypt_block v1, w3, x2, x6, w7
|
||
|
|
|
||
|
|
add x4, x0, x4
|
||
|
|
st1 {v0.16b}, [x4] /* overlapping stores */
|
||
|
|
st1 {v1.16b}, [x0]
|
||
|
|
ret
|
||
|
|
AES_FUNC_END(aes_cbc_cts_encrypt)
|
||
|
|
|
||
|
|
AES_FUNC_START(aes_cbc_cts_decrypt)
|
||
|
|
adr_l x8, .Lcts_permute_table
|
||
|
|
sub x4, x4, #16
|
||
|
|
add x9, x8, #32
|
||
|
|
add x8, x8, x4
|
||
|
|
sub x9, x9, x4
|
||
|
|
ld1 {v3.16b}, [x8]
|
||
|
|
ld1 {v4.16b}, [x9]
|
||
|
|
|
||
|
|
ld1 {v0.16b}, [x1], x4 /* overlapping loads */
|
||
|
|
ld1 {v1.16b}, [x1]
|
||
|
|
|
||
|
|
ld1 {v5.16b}, [x5] /* get iv */
|
||
|
|
dec_prepare w3, x2, x6
|
||
|
|
|
||
|
|
decrypt_block v0, w3, x2, x6, w7
|
||
|
|
tbl v2.16b, {v0.16b}, v3.16b
|
||
|
|
eor v2.16b, v2.16b, v1.16b
|
||
|
|
|
||
|
|
tbx v0.16b, {v1.16b}, v4.16b
|
||
|
|
decrypt_block v0, w3, x2, x6, w7
|
||
|
|
eor v0.16b, v0.16b, v5.16b /* xor with iv */
|
||
|
|
|
||
|
|
add x4, x0, x4
|
||
|
|
st1 {v2.16b}, [x4] /* overlapping stores */
|
||
|
|
st1 {v0.16b}, [x0]
|
||
|
|
ret
|
||
|
|
AES_FUNC_END(aes_cbc_cts_decrypt)
|
||
|
|
|
||
|
|
.section ".rodata", "a"
|
||
|
|
.align 6
|
||
|
|
.Lcts_permute_table:
|
||
|
|
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
|
||
|
|
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
|
||
|
|
.byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
|
||
|
|
.byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf
|
||
|
|
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
|
||
|
|
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
|
||
|
|
.previous
|
||
|
|
|
||
|
|
/*
|
||
|
|
* This macro generates the code for CTR and XCTR mode.
|
||
|
|
*/
|
||
|
|
.macro ctr_encrypt xctr
|
||
|
|
// Arguments
|
||
|
|
OUT .req x0
|
||
|
|
IN .req x1
|
||
|
|
KEY .req x2
|
||
|
|
ROUNDS_W .req w3
|
||
|
|
BYTES_W .req w4
|
||
|
|
IV .req x5
|
||
|
|
BYTE_CTR_W .req w6 // XCTR only
|
||
|
|
// Intermediate values
|
||
|
|
CTR_W .req w11 // XCTR only
|
||
|
|
CTR .req x11 // XCTR only
|
||
|
|
IV_PART .req x12
|
||
|
|
BLOCKS .req x13
|
||
|
|
BLOCKS_W .req w13
|
||
|
|
|
||
|
|
frame_push 0
|
||
|
|
|
||
|
|
enc_prepare ROUNDS_W, KEY, IV_PART
|
||
|
|
ld1 {vctr.16b}, [IV]
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Keep 64 bits of the IV in a register. For CTR mode this lets us
|
||
|
|
* easily increment the IV. For XCTR mode this lets us efficiently XOR
|
||
|
|
* the 64-bit counter with the IV.
|
||
|
|
*/
|
||
|
|
.if \xctr
|
||
|
|
umov IV_PART, vctr.d[0]
|
||
|
|
lsr CTR_W, BYTE_CTR_W, #4
|
||
|
|
.else
|
||
|
|
umov IV_PART, vctr.d[1]
|
||
|
|
rev IV_PART, IV_PART
|
||
|
|
.endif
|
||
|
|
|
||
|
|
.LctrloopNx\xctr:
|
||
|
|
add BLOCKS_W, BYTES_W, #15
|
||
|
|
sub BYTES_W, BYTES_W, #MAX_STRIDE << 4
|
||
|
|
lsr BLOCKS_W, BLOCKS_W, #4
|
||
|
|
mov w8, #MAX_STRIDE
|
||
|
|
cmp BLOCKS_W, w8
|
||
|
|
csel BLOCKS_W, BLOCKS_W, w8, lt
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Set up the counter values in v0-v{MAX_STRIDE-1}.
|
||
|
|
*
|
||
|
|
* If we are encrypting less than MAX_STRIDE blocks, the tail block
|
||
|
|
* handling code expects the last keystream block to be in
|
||
|
|
* v{MAX_STRIDE-1}. For example: if encrypting two blocks with
|
||
|
|
* MAX_STRIDE=5, then v3 and v4 should have the next two counter blocks.
|
||
|
|
*/
|
||
|
|
.if \xctr
|
||
|
|
add CTR, CTR, BLOCKS
|
||
|
|
.else
|
||
|
|
adds IV_PART, IV_PART, BLOCKS
|
||
|
|
.endif
|
||
|
|
mov v0.16b, vctr.16b
|
||
|
|
mov v1.16b, vctr.16b
|
||
|
|
mov v2.16b, vctr.16b
|
||
|
|
mov v3.16b, vctr.16b
|
||
|
|
ST5( mov v4.16b, vctr.16b )
|
||
|
|
.if \xctr
|
||
|
|
sub x6, CTR, #MAX_STRIDE - 1
|
||
|
|
sub x7, CTR, #MAX_STRIDE - 2
|
||
|
|
sub x8, CTR, #MAX_STRIDE - 3
|
||
|
|
sub x9, CTR, #MAX_STRIDE - 4
|
||
|
|
ST5( sub x10, CTR, #MAX_STRIDE - 5 )
|
||
|
|
eor x6, x6, IV_PART
|
||
|
|
eor x7, x7, IV_PART
|
||
|
|
eor x8, x8, IV_PART
|
||
|
|
eor x9, x9, IV_PART
|
||
|
|
ST5( eor x10, x10, IV_PART )
|
||
|
|
mov v0.d[0], x6
|
||
|
|
mov v1.d[0], x7
|
||
|
|
mov v2.d[0], x8
|
||
|
|
mov v3.d[0], x9
|
||
|
|
ST5( mov v4.d[0], x10 )
|
||
|
|
.else
|
||
|
|
bcs 0f
|
||
|
|
.subsection 1
|
||
|
|
/*
|
||
|
|
* This subsection handles carries.
|
||
|
|
*
|
||
|
|
* Conditional branching here is allowed with respect to time
|
||
|
|
* invariance since the branches are dependent on the IV instead
|
||
|
|
* of the plaintext or key. This code is rarely executed in
|
||
|
|
* practice anyway.
|
||
|
|
*/
|
||
|
|
|
||
|
|
/* Apply carry to outgoing counter. */
|
||
|
|
0: umov x8, vctr.d[0]
|
||
|
|
rev x8, x8
|
||
|
|
add x8, x8, #1
|
||
|
|
rev x8, x8
|
||
|
|
ins vctr.d[0], x8
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Apply carry to counter blocks if needed.
|
||
|
|
*
|
||
|
|
* Since the carry flag was set, we know 0 <= IV_PART <
|
||
|
|
* MAX_STRIDE. Using the value of IV_PART we can determine how
|
||
|
|
* many counter blocks need to be updated.
|
||
|
|
*/
|
||
|
|
cbz IV_PART, 2f
|
||
|
|
adr x16, 1f
|
||
|
|
sub x16, x16, IV_PART, lsl #3
|
||
|
|
br x16
|
||
|
|
bti c
|
||
|
|
mov v0.d[0], vctr.d[0]
|
||
|
|
bti c
|
||
|
|
mov v1.d[0], vctr.d[0]
|
||
|
|
bti c
|
||
|
|
mov v2.d[0], vctr.d[0]
|
||
|
|
bti c
|
||
|
|
mov v3.d[0], vctr.d[0]
|
||
|
|
ST5( bti c )
|
||
|
|
ST5( mov v4.d[0], vctr.d[0] )
|
||
|
|
1: b 2f
|
||
|
|
.previous
|
||
|
|
|
||
|
|
2: rev x7, IV_PART
|
||
|
|
ins vctr.d[1], x7
|
||
|
|
sub x7, IV_PART, #MAX_STRIDE - 1
|
||
|
|
sub x8, IV_PART, #MAX_STRIDE - 2
|
||
|
|
sub x9, IV_PART, #MAX_STRIDE - 3
|
||
|
|
rev x7, x7
|
||
|
|
rev x8, x8
|
||
|
|
mov v1.d[1], x7
|
||
|
|
rev x9, x9
|
||
|
|
ST5( sub x10, IV_PART, #MAX_STRIDE - 4 )
|
||
|
|
mov v2.d[1], x8
|
||
|
|
ST5( rev x10, x10 )
|
||
|
|
mov v3.d[1], x9
|
||
|
|
ST5( mov v4.d[1], x10 )
|
||
|
|
.endif
|
||
|
|
|
||
|
|
/*
|
||
|
|
* If there are at least MAX_STRIDE blocks left, XOR the data with
|
||
|
|
* keystream and store. Otherwise jump to tail handling.
|
||
|
|
*/
|
||
|
|
tbnz BYTES_W, #31, .Lctrtail\xctr
|
||
|
|
ld1 {v5.16b-v7.16b}, [IN], #48
|
||
|
|
ST4( bl aes_encrypt_block4x )
|
||
|
|
ST5( bl aes_encrypt_block5x )
|
||
|
|
eor v0.16b, v5.16b, v0.16b
|
||
|
|
ST4( ld1 {v5.16b}, [IN], #16 )
|
||
|
|
eor v1.16b, v6.16b, v1.16b
|
||
|
|
ST5( ld1 {v5.16b-v6.16b}, [IN], #32 )
|
||
|
|
eor v2.16b, v7.16b, v2.16b
|
||
|
|
eor v3.16b, v5.16b, v3.16b
|
||
|
|
ST5( eor v4.16b, v6.16b, v4.16b )
|
||
|
|
st1 {v0.16b-v3.16b}, [OUT], #64
|
||
|
|
ST5( st1 {v4.16b}, [OUT], #16 )
|
||
|
|
cbz BYTES_W, .Lctrout\xctr
|
||
|
|
b .LctrloopNx\xctr
|
||
|
|
|
||
|
|
.Lctrout\xctr:
|
||
|
|
.if !\xctr
|
||
|
|
st1 {vctr.16b}, [IV] /* return next CTR value */
|
||
|
|
.endif
|
||
|
|
frame_pop
|
||
|
|
ret
|
||
|
|
|
||
|
|
.Lctrtail\xctr:
|
||
|
|
/*
|
||
|
|
* Handle up to MAX_STRIDE * 16 - 1 bytes of plaintext
|
||
|
|
*
|
||
|
|
* This code expects the last keystream block to be in v{MAX_STRIDE-1}.
|
||
|
|
* For example: if encrypting two blocks with MAX_STRIDE=5, then v3 and
|
||
|
|
* v4 should have the next two counter blocks.
|
||
|
|
*
|
||
|
|
* This allows us to store the ciphertext by writing to overlapping
|
||
|
|
* regions of memory. Any invalid ciphertext blocks get overwritten by
|
||
|
|
* correctly computed blocks. This approach greatly simplifies the
|
||
|
|
* logic for storing the ciphertext.
|
||
|
|
*/
|
||
|
|
mov x16, #16
|
||
|
|
ands w7, BYTES_W, #0xf
|
||
|
|
csel x13, x7, x16, ne
|
||
|
|
|
||
|
|
ST5( cmp BYTES_W, #64 - (MAX_STRIDE << 4))
|
||
|
|
ST5( csel x14, x16, xzr, gt )
|
||
|
|
cmp BYTES_W, #48 - (MAX_STRIDE << 4)
|
||
|
|
csel x15, x16, xzr, gt
|
||
|
|
cmp BYTES_W, #32 - (MAX_STRIDE << 4)
|
||
|
|
csel x16, x16, xzr, gt
|
||
|
|
cmp BYTES_W, #16 - (MAX_STRIDE << 4)
|
||
|
|
|
||
|
|
adr_l x9, .Lcts_permute_table
|
||
|
|
add x9, x9, x13
|
||
|
|
ble .Lctrtail1x\xctr
|
||
|
|
|
||
|
|
ST5( ld1 {v5.16b}, [IN], x14 )
|
||
|
|
ld1 {v6.16b}, [IN], x15
|
||
|
|
ld1 {v7.16b}, [IN], x16
|
||
|
|
|
||
|
|
ST4( bl aes_encrypt_block4x )
|
||
|
|
ST5( bl aes_encrypt_block5x )
|
||
|
|
|
||
|
|
ld1 {v8.16b}, [IN], x13
|
||
|
|
ld1 {v9.16b}, [IN]
|
||
|
|
ld1 {v10.16b}, [x9]
|
||
|
|
|
||
|
|
ST4( eor v6.16b, v6.16b, v0.16b )
|
||
|
|
ST4( eor v7.16b, v7.16b, v1.16b )
|
||
|
|
ST4( tbl v3.16b, {v3.16b}, v10.16b )
|
||
|
|
ST4( eor v8.16b, v8.16b, v2.16b )
|
||
|
|
ST4( eor v9.16b, v9.16b, v3.16b )
|
||
|
|
|
||
|
|
ST5( eor v5.16b, v5.16b, v0.16b )
|
||
|
|
ST5( eor v6.16b, v6.16b, v1.16b )
|
||
|
|
ST5( tbl v4.16b, {v4.16b}, v10.16b )
|
||
|
|
ST5( eor v7.16b, v7.16b, v2.16b )
|
||
|
|
ST5( eor v8.16b, v8.16b, v3.16b )
|
||
|
|
ST5( eor v9.16b, v9.16b, v4.16b )
|
||
|
|
|
||
|
|
ST5( st1 {v5.16b}, [OUT], x14 )
|
||
|
|
st1 {v6.16b}, [OUT], x15
|
||
|
|
st1 {v7.16b}, [OUT], x16
|
||
|
|
add x13, x13, OUT
|
||
|
|
st1 {v9.16b}, [x13] // overlapping stores
|
||
|
|
st1 {v8.16b}, [OUT]
|
||
|
|
b .Lctrout\xctr
|
||
|
|
|
||
|
|
.Lctrtail1x\xctr:
|
||
|
|
/*
|
||
|
|
* Handle <= 16 bytes of plaintext
|
||
|
|
*
|
||
|
|
* This code always reads and writes 16 bytes. To avoid out of bounds
|
||
|
|
* accesses, XCTR and CTR modes must use a temporary buffer when
|
||
|
|
* encrypting/decrypting less than 16 bytes.
|
||
|
|
*
|
||
|
|
* This code is unusual in that it loads the input and stores the output
|
||
|
|
* relative to the end of the buffers rather than relative to the start.
|
||
|
|
* This causes unusual behaviour when encrypting/decrypting less than 16
|
||
|
|
* bytes; the end of the data is expected to be at the end of the
|
||
|
|
* temporary buffer rather than the start of the data being at the start
|
||
|
|
* of the temporary buffer.
|
||
|
|
*/
|
||
|
|
sub x8, x7, #16
|
||
|
|
csel x7, x7, x8, eq
|
||
|
|
add IN, IN, x7
|
||
|
|
add OUT, OUT, x7
|
||
|
|
ld1 {v5.16b}, [IN]
|
||
|
|
ld1 {v6.16b}, [OUT]
|
||
|
|
ST5( mov v3.16b, v4.16b )
|
||
|
|
encrypt_block v3, ROUNDS_W, KEY, x8, w7
|
||
|
|
ld1 {v10.16b-v11.16b}, [x9]
|
||
|
|
tbl v3.16b, {v3.16b}, v10.16b
|
||
|
|
sshr v11.16b, v11.16b, #7
|
||
|
|
eor v5.16b, v5.16b, v3.16b
|
||
|
|
bif v5.16b, v6.16b, v11.16b
|
||
|
|
st1 {v5.16b}, [OUT]
|
||
|
|
b .Lctrout\xctr
|
||
|
|
|
||
|
|
// Arguments
|
||
|
|
.unreq OUT
|
||
|
|
.unreq IN
|
||
|
|
.unreq KEY
|
||
|
|
.unreq ROUNDS_W
|
||
|
|
.unreq BYTES_W
|
||
|
|
.unreq IV
|
||
|
|
.unreq BYTE_CTR_W // XCTR only
|
||
|
|
// Intermediate values
|
||
|
|
.unreq CTR_W // XCTR only
|
||
|
|
.unreq CTR // XCTR only
|
||
|
|
.unreq IV_PART
|
||
|
|
.unreq BLOCKS
|
||
|
|
.unreq BLOCKS_W
|
||
|
|
.endm
|
||
|
|
|
||
|
|
/*
|
||
|
|
* aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
|
||
|
|
* int bytes, u8 ctr[])
|
||
|
|
*
|
||
|
|
* The input and output buffers must always be at least 16 bytes even if
|
||
|
|
* encrypting/decrypting less than 16 bytes. Otherwise out of bounds
|
||
|
|
* accesses will occur. The data to be encrypted/decrypted is expected
|
||
|
|
* to be at the end of this 16-byte temporary buffer rather than the
|
||
|
|
* start.
|
||
|
|
*/
|
||
|
|
|
||
|
|
AES_FUNC_START(aes_ctr_encrypt)
|
||
|
|
ctr_encrypt 0
|
||
|
|
AES_FUNC_END(aes_ctr_encrypt)
|
||
|
|
|
||
|
|
/*
|
||
|
|
* aes_xctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
|
||
|
|
* int bytes, u8 const iv[], int byte_ctr)
|
||
|
|
*
|
||
|
|
* The input and output buffers must always be at least 16 bytes even if
|
||
|
|
* encrypting/decrypting less than 16 bytes. Otherwise out of bounds
|
||
|
|
* accesses will occur. The data to be encrypted/decrypted is expected
|
||
|
|
* to be at the end of this 16-byte temporary buffer rather than the
|
||
|
|
* start.
|
||
|
|
*/
|
||
|
|
|
||
|
|
AES_FUNC_START(aes_xctr_encrypt)
|
||
|
|
ctr_encrypt 1
|
||
|
|
AES_FUNC_END(aes_xctr_encrypt)
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
|
||
|
|
* int bytes, u8 const rk2[], u8 iv[], int first)
|
||
|
|
* aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
|
||
|
|
* int bytes, u8 const rk2[], u8 iv[], int first)
|
||
|
|
*/
|
||
|
|
|
||
|
|
.macro next_tweak, out, in, tmp
|
||
|
|
sshr \tmp\().2d, \in\().2d, #63
|
||
|
|
and \tmp\().16b, \tmp\().16b, xtsmask.16b
|
||
|
|
add \out\().2d, \in\().2d, \in\().2d
|
||
|
|
ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8
|
||
|
|
eor \out\().16b, \out\().16b, \tmp\().16b
|
||
|
|
.endm
|
||
|
|
|
||
|
|
.macro xts_load_mask, tmp
|
||
|
|
movi xtsmask.2s, #0x1
|
||
|
|
movi \tmp\().2s, #0x87
|
||
|
|
uzp1 xtsmask.4s, xtsmask.4s, \tmp\().4s
|
||
|
|
.endm
|
||
|
|
|
||
|
|
AES_FUNC_START(aes_xts_encrypt)
|
||
|
|
frame_push 0
|
||
|
|
|
||
|
|
ld1 {v4.16b}, [x6]
|
||
|
|
xts_load_mask v8
|
||
|
|
cbz w7, .Lxtsencnotfirst
|
||
|
|
|
||
|
|
enc_prepare w3, x5, x8
|
||
|
|
xts_cts_skip_tw w7, .LxtsencNx
|
||
|
|
encrypt_block v4, w3, x5, x8, w7 /* first tweak */
|
||
|
|
enc_switch_key w3, x2, x8
|
||
|
|
b .LxtsencNx
|
||
|
|
|
||
|
|
.Lxtsencnotfirst:
|
||
|
|
enc_prepare w3, x2, x8
|
||
|
|
.LxtsencloopNx:
|
||
|
|
next_tweak v4, v4, v8
|
||
|
|
.LxtsencNx:
|
||
|
|
subs w4, w4, #64
|
||
|
|
bmi .Lxtsenc1x
|
||
|
|
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
|
||
|
|
next_tweak v5, v4, v8
|
||
|
|
eor v0.16b, v0.16b, v4.16b
|
||
|
|
next_tweak v6, v5, v8
|
||
|
|
eor v1.16b, v1.16b, v5.16b
|
||
|
|
eor v2.16b, v2.16b, v6.16b
|
||
|
|
next_tweak v7, v6, v8
|
||
|
|
eor v3.16b, v3.16b, v7.16b
|
||
|
|
bl aes_encrypt_block4x
|
||
|
|
eor v3.16b, v3.16b, v7.16b
|
||
|
|
eor v0.16b, v0.16b, v4.16b
|
||
|
|
eor v1.16b, v1.16b, v5.16b
|
||
|
|
eor v2.16b, v2.16b, v6.16b
|
||
|
|
st1 {v0.16b-v3.16b}, [x0], #64
|
||
|
|
mov v4.16b, v7.16b
|
||
|
|
cbz w4, .Lxtsencret
|
||
|
|
xts_reload_mask v8
|
||
|
|
b .LxtsencloopNx
|
||
|
|
.Lxtsenc1x:
|
||
|
|
adds w4, w4, #64
|
||
|
|
beq .Lxtsencout
|
||
|
|
subs w4, w4, #16
|
||
|
|
bmi .LxtsencctsNx
|
||
|
|
.Lxtsencloop:
|
||
|
|
ld1 {v0.16b}, [x1], #16
|
||
|
|
.Lxtsencctsout:
|
||
|
|
eor v0.16b, v0.16b, v4.16b
|
||
|
|
encrypt_block v0, w3, x2, x8, w7
|
||
|
|
eor v0.16b, v0.16b, v4.16b
|
||
|
|
cbz w4, .Lxtsencout
|
||
|
|
subs w4, w4, #16
|
||
|
|
next_tweak v4, v4, v8
|
||
|
|
bmi .Lxtsenccts
|
||
|
|
st1 {v0.16b}, [x0], #16
|
||
|
|
b .Lxtsencloop
|
||
|
|
.Lxtsencout:
|
||
|
|
st1 {v0.16b}, [x0]
|
||
|
|
.Lxtsencret:
|
||
|
|
st1 {v4.16b}, [x6]
|
||
|
|
frame_pop
|
||
|
|
ret
|
||
|
|
|
||
|
|
.LxtsencctsNx:
|
||
|
|
mov v0.16b, v3.16b
|
||
|
|
sub x0, x0, #16
|
||
|
|
.Lxtsenccts:
|
||
|
|
adr_l x8, .Lcts_permute_table
|
||
|
|
|
||
|
|
add x1, x1, w4, sxtw /* rewind input pointer */
|
||
|
|
add w4, w4, #16 /* # bytes in final block */
|
||
|
|
add x9, x8, #32
|
||
|
|
add x8, x8, x4
|
||
|
|
sub x9, x9, x4
|
||
|
|
add x4, x0, x4 /* output address of final block */
|
||
|
|
|
||
|
|
ld1 {v1.16b}, [x1] /* load final block */
|
||
|
|
ld1 {v2.16b}, [x8]
|
||
|
|
ld1 {v3.16b}, [x9]
|
||
|
|
|
||
|
|
tbl v2.16b, {v0.16b}, v2.16b
|
||
|
|
tbx v0.16b, {v1.16b}, v3.16b
|
||
|
|
st1 {v2.16b}, [x4] /* overlapping stores */
|
||
|
|
mov w4, wzr
|
||
|
|
b .Lxtsencctsout
|
||
|
|
AES_FUNC_END(aes_xts_encrypt)
|
||
|
|
|
||
|
|
AES_FUNC_START(aes_xts_decrypt)
|
||
|
|
frame_push 0
|
||
|
|
|
||
|
|
/* subtract 16 bytes if we are doing CTS */
|
||
|
|
sub w8, w4, #0x10
|
||
|
|
tst w4, #0xf
|
||
|
|
csel w4, w4, w8, eq
|
||
|
|
|
||
|
|
ld1 {v4.16b}, [x6]
|
||
|
|
xts_load_mask v8
|
||
|
|
xts_cts_skip_tw w7, .Lxtsdecskiptw
|
||
|
|
cbz w7, .Lxtsdecnotfirst
|
||
|
|
|
||
|
|
enc_prepare w3, x5, x8
|
||
|
|
encrypt_block v4, w3, x5, x8, w7 /* first tweak */
|
||
|
|
.Lxtsdecskiptw:
|
||
|
|
dec_prepare w3, x2, x8
|
||
|
|
b .LxtsdecNx
|
||
|
|
|
||
|
|
.Lxtsdecnotfirst:
|
||
|
|
dec_prepare w3, x2, x8
|
||
|
|
.LxtsdecloopNx:
|
||
|
|
next_tweak v4, v4, v8
|
||
|
|
.LxtsdecNx:
|
||
|
|
subs w4, w4, #64
|
||
|
|
bmi .Lxtsdec1x
|
||
|
|
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
|
||
|
|
next_tweak v5, v4, v8
|
||
|
|
eor v0.16b, v0.16b, v4.16b
|
||
|
|
next_tweak v6, v5, v8
|
||
|
|
eor v1.16b, v1.16b, v5.16b
|
||
|
|
eor v2.16b, v2.16b, v6.16b
|
||
|
|
next_tweak v7, v6, v8
|
||
|
|
eor v3.16b, v3.16b, v7.16b
|
||
|
|
bl aes_decrypt_block4x
|
||
|
|
eor v3.16b, v3.16b, v7.16b
|
||
|
|
eor v0.16b, v0.16b, v4.16b
|
||
|
|
eor v1.16b, v1.16b, v5.16b
|
||
|
|
eor v2.16b, v2.16b, v6.16b
|
||
|
|
st1 {v0.16b-v3.16b}, [x0], #64
|
||
|
|
mov v4.16b, v7.16b
|
||
|
|
cbz w4, .Lxtsdecout
|
||
|
|
xts_reload_mask v8
|
||
|
|
b .LxtsdecloopNx
|
||
|
|
.Lxtsdec1x:
|
||
|
|
adds w4, w4, #64
|
||
|
|
beq .Lxtsdecout
|
||
|
|
subs w4, w4, #16
|
||
|
|
.Lxtsdecloop:
|
||
|
|
ld1 {v0.16b}, [x1], #16
|
||
|
|
bmi .Lxtsdeccts
|
||
|
|
.Lxtsdecctsout:
|
||
|
|
eor v0.16b, v0.16b, v4.16b
|
||
|
|
decrypt_block v0, w3, x2, x8, w7
|
||
|
|
eor v0.16b, v0.16b, v4.16b
|
||
|
|
st1 {v0.16b}, [x0], #16
|
||
|
|
cbz w4, .Lxtsdecout
|
||
|
|
subs w4, w4, #16
|
||
|
|
next_tweak v4, v4, v8
|
||
|
|
b .Lxtsdecloop
|
||
|
|
.Lxtsdecout:
|
||
|
|
st1 {v4.16b}, [x6]
|
||
|
|
frame_pop
|
||
|
|
ret
|
||
|
|
|
||
|
|
.Lxtsdeccts:
|
||
|
|
adr_l x8, .Lcts_permute_table
|
||
|
|
|
||
|
|
add x1, x1, w4, sxtw /* rewind input pointer */
|
||
|
|
add w4, w4, #16 /* # bytes in final block */
|
||
|
|
add x9, x8, #32
|
||
|
|
add x8, x8, x4
|
||
|
|
sub x9, x9, x4
|
||
|
|
add x4, x0, x4 /* output address of final block */
|
||
|
|
|
||
|
|
next_tweak v5, v4, v8
|
||
|
|
|
||
|
|
ld1 {v1.16b}, [x1] /* load final block */
|
||
|
|
ld1 {v2.16b}, [x8]
|
||
|
|
ld1 {v3.16b}, [x9]
|
||
|
|
|
||
|
|
eor v0.16b, v0.16b, v5.16b
|
||
|
|
decrypt_block v0, w3, x2, x8, w7
|
||
|
|
eor v0.16b, v0.16b, v5.16b
|
||
|
|
|
||
|
|
tbl v2.16b, {v0.16b}, v2.16b
|
||
|
|
tbx v0.16b, {v1.16b}, v3.16b
|
||
|
|
|
||
|
|
st1 {v2.16b}, [x4] /* overlapping stores */
|
||
|
|
mov w4, wzr
|
||
|
|
b .Lxtsdecctsout
|
||
|
|
AES_FUNC_END(aes_xts_decrypt)
|
||
|
|
|
||
|
|
/*
|
||
|
|
* aes_mac_update(u8 const in[], u32 const rk[], int rounds,
|
||
|
|
* int blocks, u8 dg[], int enc_before, int enc_after)
|
||
|
|
*/
|
||
|
|
AES_FUNC_START(aes_mac_update)
|
||
|
|
ld1 {v0.16b}, [x4] /* get dg */
|
||
|
|
enc_prepare w2, x1, x7
|
||
|
|
cbz w5, .Lmacloop4x
|
||
|
|
|
||
|
|
encrypt_block v0, w2, x1, x7, w8
|
||
|
|
|
||
|
|
.Lmacloop4x:
|
||
|
|
subs w3, w3, #4
|
||
|
|
bmi .Lmac1x
|
||
|
|
ld1 {v1.16b-v4.16b}, [x0], #64 /* get next pt block */
|
||
|
|
eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */
|
||
|
|
encrypt_block v0, w2, x1, x7, w8
|
||
|
|
eor v0.16b, v0.16b, v2.16b
|
||
|
|
encrypt_block v0, w2, x1, x7, w8
|
||
|
|
eor v0.16b, v0.16b, v3.16b
|
||
|
|
encrypt_block v0, w2, x1, x7, w8
|
||
|
|
eor v0.16b, v0.16b, v4.16b
|
||
|
|
cmp w3, wzr
|
||
|
|
csinv x5, x6, xzr, eq
|
||
|
|
cbz w5, .Lmacout
|
||
|
|
encrypt_block v0, w2, x1, x7, w8
|
||
|
|
st1 {v0.16b}, [x4] /* return dg */
|
||
|
|
cond_yield .Lmacout, x7, x8
|
||
|
|
b .Lmacloop4x
|
||
|
|
.Lmac1x:
|
||
|
|
add w3, w3, #4
|
||
|
|
.Lmacloop:
|
||
|
|
cbz w3, .Lmacout
|
||
|
|
ld1 {v1.16b}, [x0], #16 /* get next pt block */
|
||
|
|
eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */
|
||
|
|
|
||
|
|
subs w3, w3, #1
|
||
|
|
csinv x5, x6, xzr, eq
|
||
|
|
cbz w5, .Lmacout
|
||
|
|
|
||
|
|
.Lmacenc:
|
||
|
|
encrypt_block v0, w2, x1, x7, w8
|
||
|
|
b .Lmacloop
|
||
|
|
|
||
|
|
.Lmacout:
|
||
|
|
st1 {v0.16b}, [x4] /* return dg */
|
||
|
|
mov w0, w3
|
||
|
|
ret
|
||
|
|
AES_FUNC_END(aes_mac_update)
|