362 lines
10 KiB
ArmAsm
362 lines
10 KiB
ArmAsm
|
/* SPDX-License-Identifier: GPL-2.0 */
|
||
|
|
/*
|
||
|
|
* Implementation of POLYVAL using ARMv8 Crypto Extensions.
|
||
|
|
*
|
||
|
|
* Copyright 2021 Google LLC
|
||
|
|
*/
|
||
|
|
/*
|
||
|
|
* This is an efficient implementation of POLYVAL using ARMv8 Crypto Extensions
|
||
|
|
* It works on 8 blocks at a time, by precomputing the first 8 keys powers h^8,
|
||
|
|
* ..., h^1 in the POLYVAL finite field. This precomputation allows us to split
|
||
|
|
* finite field multiplication into two steps.
|
||
|
|
*
|
||
|
|
* In the first step, we consider h^i, m_i as normal polynomials of degree less
|
||
|
|
* than 128. We then compute p(x) = h^8m_0 + ... + h^1m_7 where multiplication
|
||
|
|
* is simply polynomial multiplication.
|
||
|
|
*
|
||
|
|
* In the second step, we compute the reduction of p(x) modulo the finite field
|
||
|
|
* modulus g(x) = x^128 + x^127 + x^126 + x^121 + 1.
|
||
|
|
*
|
||
|
|
* This two step process is equivalent to computing h^8m_0 + ... + h^1m_7 where
|
||
|
|
* multiplication is finite field multiplication. The advantage is that the
|
||
|
|
* two-step process only requires 1 finite field reduction for every 8
|
||
|
|
* polynomial multiplications. Further parallelism is gained by interleaving the
|
||
|
|
* multiplications and polynomial reductions.
|
||
|
|
*/
|
||
|
|
|
||
|
|
#include <linux/linkage.h>
|
||
|
|
#define STRIDE_BLOCKS 8
|
||
|
|
|
||
|
|
KEY_POWERS .req x0
|
||
|
|
MSG .req x1
|
||
|
|
BLOCKS_LEFT .req x2
|
||
|
|
ACCUMULATOR .req x3
|
||
|
|
KEY_START .req x10
|
||
|
|
EXTRA_BYTES .req x11
|
||
|
|
TMP .req x13
|
||
|
|
|
||
|
|
M0 .req v0
|
||
|
|
M1 .req v1
|
||
|
|
M2 .req v2
|
||
|
|
M3 .req v3
|
||
|
|
M4 .req v4
|
||
|
|
M5 .req v5
|
||
|
|
M6 .req v6
|
||
|
|
M7 .req v7
|
||
|
|
KEY8 .req v8
|
||
|
|
KEY7 .req v9
|
||
|
|
KEY6 .req v10
|
||
|
|
KEY5 .req v11
|
||
|
|
KEY4 .req v12
|
||
|
|
KEY3 .req v13
|
||
|
|
KEY2 .req v14
|
||
|
|
KEY1 .req v15
|
||
|
|
PL .req v16
|
||
|
|
PH .req v17
|
||
|
|
TMP_V .req v18
|
||
|
|
LO .req v20
|
||
|
|
MI .req v21
|
||
|
|
HI .req v22
|
||
|
|
SUM .req v23
|
||
|
|
GSTAR .req v24
|
||
|
|
|
||
|
|
.text
|
||
|
|
|
||
|
|
.arch armv8-a+crypto
|
||
|
|
.align 4
|
||
|
|
|
||
|
|
.Lgstar:
|
||
|
|
.quad 0xc200000000000000, 0xc200000000000000
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Computes the product of two 128-bit polynomials in X and Y and XORs the
|
||
|
|
* components of the 256-bit product into LO, MI, HI.
|
||
|
|
*
|
||
|
|
* Given:
|
||
|
|
* X = [X_1 : X_0]
|
||
|
|
* Y = [Y_1 : Y_0]
|
||
|
|
*
|
||
|
|
* We compute:
|
||
|
|
* LO += X_0 * Y_0
|
||
|
|
* MI += (X_0 + X_1) * (Y_0 + Y_1)
|
||
|
|
* HI += X_1 * Y_1
|
||
|
|
*
|
||
|
|
* Later, the 256-bit result can be extracted as:
|
||
|
|
* [HI_1 : HI_0 + HI_1 + MI_1 + LO_1 : LO_1 + HI_0 + MI_0 + LO_0 : LO_0]
|
||
|
|
* This step is done when computing the polynomial reduction for efficiency
|
||
|
|
* reasons.
|
||
|
|
*
|
||
|
|
* Karatsuba multiplication is used instead of Schoolbook multiplication because
|
||
|
|
* it was found to be slightly faster on ARM64 CPUs.
|
||
|
|
*
|
||
|
|
*/
|
||
|
|
.macro karatsuba1 X Y
|
||
|
|
X .req \X
|
||
|
|
Y .req \Y
|
||
|
|
ext v25.16b, X.16b, X.16b, #8
|
||
|
|
ext v26.16b, Y.16b, Y.16b, #8
|
||
|
|
eor v25.16b, v25.16b, X.16b
|
||
|
|
eor v26.16b, v26.16b, Y.16b
|
||
|
|
pmull2 v28.1q, X.2d, Y.2d
|
||
|
|
pmull v29.1q, X.1d, Y.1d
|
||
|
|
pmull v27.1q, v25.1d, v26.1d
|
||
|
|
eor HI.16b, HI.16b, v28.16b
|
||
|
|
eor LO.16b, LO.16b, v29.16b
|
||
|
|
eor MI.16b, MI.16b, v27.16b
|
||
|
|
.unreq X
|
||
|
|
.unreq Y
|
||
|
|
.endm
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Same as karatsuba1, except overwrites HI, LO, MI rather than XORing into
|
||
|
|
* them.
|
||
|
|
*/
|
||
|
|
.macro karatsuba1_store X Y
|
||
|
|
X .req \X
|
||
|
|
Y .req \Y
|
||
|
|
ext v25.16b, X.16b, X.16b, #8
|
||
|
|
ext v26.16b, Y.16b, Y.16b, #8
|
||
|
|
eor v25.16b, v25.16b, X.16b
|
||
|
|
eor v26.16b, v26.16b, Y.16b
|
||
|
|
pmull2 HI.1q, X.2d, Y.2d
|
||
|
|
pmull LO.1q, X.1d, Y.1d
|
||
|
|
pmull MI.1q, v25.1d, v26.1d
|
||
|
|
.unreq X
|
||
|
|
.unreq Y
|
||
|
|
.endm
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Computes the 256-bit polynomial represented by LO, HI, MI. Stores
|
||
|
|
* the result in PL, PH.
|
||
|
|
* [PH : PL] =
|
||
|
|
* [HI_1 : HI_1 + HI_0 + MI_1 + LO_1 : HI_0 + MI_0 + LO_1 + LO_0 : LO_0]
|
||
|
|
*/
|
||
|
|
.macro karatsuba2
|
||
|
|
// v4 = [HI_1 + MI_1 : HI_0 + MI_0]
|
||
|
|
eor v4.16b, HI.16b, MI.16b
|
||
|
|
// v4 = [HI_1 + MI_1 + LO_1 : HI_0 + MI_0 + LO_0]
|
||
|
|
eor v4.16b, v4.16b, LO.16b
|
||
|
|
// v5 = [HI_0 : LO_1]
|
||
|
|
ext v5.16b, LO.16b, HI.16b, #8
|
||
|
|
// v4 = [HI_1 + HI_0 + MI_1 + LO_1 : HI_0 + MI_0 + LO_1 + LO_0]
|
||
|
|
eor v4.16b, v4.16b, v5.16b
|
||
|
|
// HI = [HI_0 : HI_1]
|
||
|
|
ext HI.16b, HI.16b, HI.16b, #8
|
||
|
|
// LO = [LO_0 : LO_1]
|
||
|
|
ext LO.16b, LO.16b, LO.16b, #8
|
||
|
|
// PH = [HI_1 : HI_1 + HI_0 + MI_1 + LO_1]
|
||
|
|
ext PH.16b, v4.16b, HI.16b, #8
|
||
|
|
// PL = [HI_0 + MI_0 + LO_1 + LO_0 : LO_0]
|
||
|
|
ext PL.16b, LO.16b, v4.16b, #8
|
||
|
|
.endm
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Computes the 128-bit reduction of PH : PL. Stores the result in dest.
|
||
|
|
*
|
||
|
|
* This macro computes p(x) mod g(x) where p(x) is in montgomery form and g(x) =
|
||
|
|
* x^128 + x^127 + x^126 + x^121 + 1.
|
||
|
|
*
|
||
|
|
* We have a 256-bit polynomial PH : PL = P_3 : P_2 : P_1 : P_0 that is the
|
||
|
|
* product of two 128-bit polynomials in Montgomery form. We need to reduce it
|
||
|
|
* mod g(x). Also, since polynomials in Montgomery form have an "extra" factor
|
||
|
|
* of x^128, this product has two extra factors of x^128. To get it back into
|
||
|
|
* Montgomery form, we need to remove one of these factors by dividing by x^128.
|
||
|
|
*
|
||
|
|
* To accomplish both of these goals, we add multiples of g(x) that cancel out
|
||
|
|
* the low 128 bits P_1 : P_0, leaving just the high 128 bits. Since the low
|
||
|
|
* bits are zero, the polynomial division by x^128 can be done by right
|
||
|
|
* shifting.
|
||
|
|
*
|
||
|
|
* Since the only nonzero term in the low 64 bits of g(x) is the constant term,
|
||
|
|
* the multiple of g(x) needed to cancel out P_0 is P_0 * g(x). The CPU can
|
||
|
|
* only do 64x64 bit multiplications, so split P_0 * g(x) into x^128 * P_0 +
|
||
|
|
* x^64 * g*(x) * P_0 + P_0, where g*(x) is bits 64-127 of g(x). Adding this to
|
||
|
|
* the original polynomial gives P_3 : P_2 + P_0 + T_1 : P_1 + T_0 : 0, where T
|
||
|
|
* = T_1 : T_0 = g*(x) * P_0. Thus, bits 0-63 got "folded" into bits 64-191.
|
||
|
|
*
|
||
|
|
* Repeating this same process on the next 64 bits "folds" bits 64-127 into bits
|
||
|
|
* 128-255, giving the answer in bits 128-255. This time, we need to cancel P_1
|
||
|
|
* + T_0 in bits 64-127. The multiple of g(x) required is (P_1 + T_0) * g(x) *
|
||
|
|
* x^64. Adding this to our previous computation gives P_3 + P_1 + T_0 + V_1 :
|
||
|
|
* P_2 + P_0 + T_1 + V_0 : 0 : 0, where V = V_1 : V_0 = g*(x) * (P_1 + T_0).
|
||
|
|
*
|
||
|
|
* So our final computation is:
|
||
|
|
* T = T_1 : T_0 = g*(x) * P_0
|
||
|
|
* V = V_1 : V_0 = g*(x) * (P_1 + T_0)
|
||
|
|
* p(x) / x^{128} mod g(x) = P_3 + P_1 + T_0 + V_1 : P_2 + P_0 + T_1 + V_0
|
||
|
|
*
|
||
|
|
* The implementation below saves a XOR instruction by computing P_1 + T_0 : P_0
|
||
|
|
* + T_1 and XORing into dest, rather than separately XORing P_1 : P_0 and T_0 :
|
||
|
|
* T_1 into dest. This allows us to reuse P_1 + T_0 when computing V.
|
||
|
|
*/
|
||
|
|
.macro montgomery_reduction dest
|
||
|
|
DEST .req \dest
|
||
|
|
// TMP_V = T_1 : T_0 = P_0 * g*(x)
|
||
|
|
pmull TMP_V.1q, PL.1d, GSTAR.1d
|
||
|
|
// TMP_V = T_0 : T_1
|
||
|
|
ext TMP_V.16b, TMP_V.16b, TMP_V.16b, #8
|
||
|
|
// TMP_V = P_1 + T_0 : P_0 + T_1
|
||
|
|
eor TMP_V.16b, PL.16b, TMP_V.16b
|
||
|
|
// PH = P_3 + P_1 + T_0 : P_2 + P_0 + T_1
|
||
|
|
eor PH.16b, PH.16b, TMP_V.16b
|
||
|
|
// TMP_V = V_1 : V_0 = (P_1 + T_0) * g*(x)
|
||
|
|
pmull2 TMP_V.1q, TMP_V.2d, GSTAR.2d
|
||
|
|
eor DEST.16b, PH.16b, TMP_V.16b
|
||
|
|
.unreq DEST
|
||
|
|
.endm
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Compute Polyval on 8 blocks.
|
||
|
|
*
|
||
|
|
* If reduce is set, also computes the montgomery reduction of the
|
||
|
|
* previous full_stride call and XORs with the first message block.
|
||
|
|
* (m_0 + REDUCE(PL, PH))h^8 + ... + m_7h^1.
|
||
|
|
* I.e., the first multiplication uses m_0 + REDUCE(PL, PH) instead of m_0.
|
||
|
|
*
|
||
|
|
* Sets PL, PH.
|
||
|
|
*/
|
||
|
|
.macro full_stride reduce
|
||
|
|
eor LO.16b, LO.16b, LO.16b
|
||
|
|
eor MI.16b, MI.16b, MI.16b
|
||
|
|
eor HI.16b, HI.16b, HI.16b
|
||
|
|
|
||
|
|
ld1 {M0.16b, M1.16b, M2.16b, M3.16b}, [MSG], #64
|
||
|
|
ld1 {M4.16b, M5.16b, M6.16b, M7.16b}, [MSG], #64
|
||
|
|
|
||
|
|
karatsuba1 M7 KEY1
|
||
|
|
.if \reduce
|
||
|
|
pmull TMP_V.1q, PL.1d, GSTAR.1d
|
||
|
|
.endif
|
||
|
|
|
||
|
|
karatsuba1 M6 KEY2
|
||
|
|
.if \reduce
|
||
|
|
ext TMP_V.16b, TMP_V.16b, TMP_V.16b, #8
|
||
|
|
.endif
|
||
|
|
|
||
|
|
karatsuba1 M5 KEY3
|
||
|
|
.if \reduce
|
||
|
|
eor TMP_V.16b, PL.16b, TMP_V.16b
|
||
|
|
.endif
|
||
|
|
|
||
|
|
karatsuba1 M4 KEY4
|
||
|
|
.if \reduce
|
||
|
|
eor PH.16b, PH.16b, TMP_V.16b
|
||
|
|
.endif
|
||
|
|
|
||
|
|
karatsuba1 M3 KEY5
|
||
|
|
.if \reduce
|
||
|
|
pmull2 TMP_V.1q, TMP_V.2d, GSTAR.2d
|
||
|
|
.endif
|
||
|
|
|
||
|
|
karatsuba1 M2 KEY6
|
||
|
|
.if \reduce
|
||
|
|
eor SUM.16b, PH.16b, TMP_V.16b
|
||
|
|
.endif
|
||
|
|
|
||
|
|
karatsuba1 M1 KEY7
|
||
|
|
eor M0.16b, M0.16b, SUM.16b
|
||
|
|
|
||
|
|
karatsuba1 M0 KEY8
|
||
|
|
karatsuba2
|
||
|
|
.endm
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Handle any extra blocks after full_stride loop.
|
||
|
|
*/
|
||
|
|
.macro partial_stride
|
||
|
|
add KEY_POWERS, KEY_START, #(STRIDE_BLOCKS << 4)
|
||
|
|
sub KEY_POWERS, KEY_POWERS, BLOCKS_LEFT, lsl #4
|
||
|
|
ld1 {KEY1.16b}, [KEY_POWERS], #16
|
||
|
|
|
||
|
|
ld1 {TMP_V.16b}, [MSG], #16
|
||
|
|
eor SUM.16b, SUM.16b, TMP_V.16b
|
||
|
|
karatsuba1_store KEY1 SUM
|
||
|
|
sub BLOCKS_LEFT, BLOCKS_LEFT, #1
|
||
|
|
|
||
|
|
tst BLOCKS_LEFT, #4
|
||
|
|
beq .Lpartial4BlocksDone
|
||
|
|
ld1 {M0.16b, M1.16b, M2.16b, M3.16b}, [MSG], #64
|
||
|
|
ld1 {KEY8.16b, KEY7.16b, KEY6.16b, KEY5.16b}, [KEY_POWERS], #64
|
||
|
|
karatsuba1 M0 KEY8
|
||
|
|
karatsuba1 M1 KEY7
|
||
|
|
karatsuba1 M2 KEY6
|
||
|
|
karatsuba1 M3 KEY5
|
||
|
|
.Lpartial4BlocksDone:
|
||
|
|
tst BLOCKS_LEFT, #2
|
||
|
|
beq .Lpartial2BlocksDone
|
||
|
|
ld1 {M0.16b, M1.16b}, [MSG], #32
|
||
|
|
ld1 {KEY8.16b, KEY7.16b}, [KEY_POWERS], #32
|
||
|
|
karatsuba1 M0 KEY8
|
||
|
|
karatsuba1 M1 KEY7
|
||
|
|
.Lpartial2BlocksDone:
|
||
|
|
tst BLOCKS_LEFT, #1
|
||
|
|
beq .LpartialDone
|
||
|
|
ld1 {M0.16b}, [MSG], #16
|
||
|
|
ld1 {KEY8.16b}, [KEY_POWERS], #16
|
||
|
|
karatsuba1 M0 KEY8
|
||
|
|
.LpartialDone:
|
||
|
|
karatsuba2
|
||
|
|
montgomery_reduction SUM
|
||
|
|
.endm
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Perform montgomery multiplication in GF(2^128) and store result in op1.
|
||
|
|
*
|
||
|
|
* Computes op1*op2*x^{-128} mod x^128 + x^127 + x^126 + x^121 + 1
|
||
|
|
* If op1, op2 are in montgomery form, this computes the montgomery
|
||
|
|
* form of op1*op2.
|
||
|
|
*
|
||
|
|
* void pmull_polyval_mul(u8 *op1, const u8 *op2);
|
||
|
|
*/
|
||
|
|
SYM_FUNC_START(pmull_polyval_mul)
|
||
|
|
adr TMP, .Lgstar
|
||
|
|
ld1 {GSTAR.2d}, [TMP]
|
||
|
|
ld1 {v0.16b}, [x0]
|
||
|
|
ld1 {v1.16b}, [x1]
|
||
|
|
karatsuba1_store v0 v1
|
||
|
|
karatsuba2
|
||
|
|
montgomery_reduction SUM
|
||
|
|
st1 {SUM.16b}, [x0]
|
||
|
|
ret
|
||
|
|
SYM_FUNC_END(pmull_polyval_mul)
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Perform polynomial evaluation as specified by POLYVAL. This computes:
|
||
|
|
* h^n * accumulator + h^n * m_0 + ... + h^1 * m_{n-1}
|
||
|
|
* where n=nblocks, h is the hash key, and m_i are the message blocks.
|
||
|
|
*
|
||
|
|
* x0 - pointer to precomputed key powers h^8 ... h^1
|
||
|
|
* x1 - pointer to message blocks
|
||
|
|
* x2 - number of blocks to hash
|
||
|
|
* x3 - pointer to accumulator
|
||
|
|
*
|
||
|
|
* void pmull_polyval_update(const struct polyval_ctx *ctx, const u8 *in,
|
||
|
|
* size_t nblocks, u8 *accumulator);
|
||
|
|
*/
|
||
|
|
SYM_FUNC_START(pmull_polyval_update)
|
||
|
|
adr TMP, .Lgstar
|
||
|
|
mov KEY_START, KEY_POWERS
|
||
|
|
ld1 {GSTAR.2d}, [TMP]
|
||
|
|
ld1 {SUM.16b}, [ACCUMULATOR]
|
||
|
|
subs BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
|
||
|
|
blt .LstrideLoopExit
|
||
|
|
ld1 {KEY8.16b, KEY7.16b, KEY6.16b, KEY5.16b}, [KEY_POWERS], #64
|
||
|
|
ld1 {KEY4.16b, KEY3.16b, KEY2.16b, KEY1.16b}, [KEY_POWERS], #64
|
||
|
|
full_stride 0
|
||
|
|
subs BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
|
||
|
|
blt .LstrideLoopExitReduce
|
||
|
|
.LstrideLoop:
|
||
|
|
full_stride 1
|
||
|
|
subs BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
|
||
|
|
bge .LstrideLoop
|
||
|
|
.LstrideLoopExitReduce:
|
||
|
|
montgomery_reduction SUM
|
||
|
|
.LstrideLoopExit:
|
||
|
|
adds BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
|
||
|
|
beq .LskipPartial
|
||
|
|
partial_stride
|
||
|
|
.LskipPartial:
|
||
|
|
st1 {SUM.16b}, [ACCUMULATOR]
|
||
|
|
ret
|
||
|
|
SYM_FUNC_END(pmull_polyval_update)
|