319 lines
11 KiB
ArmAsm
319 lines
11 KiB
ArmAsm
|
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
||
|
|
/*
|
||
|
|
* Fast SHA-256 implementation for SPE instruction set (PPC)
|
||
|
|
*
|
||
|
|
* This code makes use of the SPE SIMD instruction set as defined in
|
||
|
|
* http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf
|
||
|
|
* Implementation is based on optimization guide notes from
|
||
|
|
* http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf
|
||
|
|
*
|
||
|
|
* Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
|
||
|
|
*/
|
||
|
|
|
||
|
|
#include <asm/ppc_asm.h>
|
||
|
|
#include <asm/asm-offsets.h>
|
||
|
|
|
||
|
|
#define rHP r3 /* pointer to hash values in memory */
|
||
|
|
#define rKP r24 /* pointer to round constants */
|
||
|
|
#define rWP r4 /* pointer to input data */
|
||
|
|
|
||
|
|
#define rH0 r5 /* 8 32 bit hash values in 8 registers */
|
||
|
|
#define rH1 r6
|
||
|
|
#define rH2 r7
|
||
|
|
#define rH3 r8
|
||
|
|
#define rH4 r9
|
||
|
|
#define rH5 r10
|
||
|
|
#define rH6 r11
|
||
|
|
#define rH7 r12
|
||
|
|
|
||
|
|
#define rW0 r14 /* 64 bit registers. 16 words in 8 registers */
|
||
|
|
#define rW1 r15
|
||
|
|
#define rW2 r16
|
||
|
|
#define rW3 r17
|
||
|
|
#define rW4 r18
|
||
|
|
#define rW5 r19
|
||
|
|
#define rW6 r20
|
||
|
|
#define rW7 r21
|
||
|
|
|
||
|
|
#define rT0 r22 /* 64 bit temporaries */
|
||
|
|
#define rT1 r23
|
||
|
|
#define rT2 r0 /* 32 bit temporaries */
|
||
|
|
#define rT3 r25
|
||
|
|
|
||
|
|
#define CMP_KN_LOOP
|
||
|
|
#define CMP_KC_LOOP \
|
||
|
|
cmpwi rT1,0;
|
||
|
|
|
||
|
|
#define INITIALIZE \
|
||
|
|
stwu r1,-128(r1); /* create stack frame */ \
|
||
|
|
evstdw r14,8(r1); /* We must save non volatile */ \
|
||
|
|
evstdw r15,16(r1); /* registers. Take the chance */ \
|
||
|
|
evstdw r16,24(r1); /* and save the SPE part too */ \
|
||
|
|
evstdw r17,32(r1); \
|
||
|
|
evstdw r18,40(r1); \
|
||
|
|
evstdw r19,48(r1); \
|
||
|
|
evstdw r20,56(r1); \
|
||
|
|
evstdw r21,64(r1); \
|
||
|
|
evstdw r22,72(r1); \
|
||
|
|
evstdw r23,80(r1); \
|
||
|
|
stw r24,88(r1); /* save normal registers */ \
|
||
|
|
stw r25,92(r1);
|
||
|
|
|
||
|
|
|
||
|
|
#define FINALIZE \
|
||
|
|
evldw r14,8(r1); /* restore SPE registers */ \
|
||
|
|
evldw r15,16(r1); \
|
||
|
|
evldw r16,24(r1); \
|
||
|
|
evldw r17,32(r1); \
|
||
|
|
evldw r18,40(r1); \
|
||
|
|
evldw r19,48(r1); \
|
||
|
|
evldw r20,56(r1); \
|
||
|
|
evldw r21,64(r1); \
|
||
|
|
evldw r22,72(r1); \
|
||
|
|
evldw r23,80(r1); \
|
||
|
|
lwz r24,88(r1); /* restore normal registers */ \
|
||
|
|
lwz r25,92(r1); \
|
||
|
|
xor r0,r0,r0; \
|
||
|
|
stw r0,8(r1); /* Delete sensitive data */ \
|
||
|
|
stw r0,16(r1); /* that we might have pushed */ \
|
||
|
|
stw r0,24(r1); /* from other context that runs */ \
|
||
|
|
stw r0,32(r1); /* the same code. Assume that */ \
|
||
|
|
stw r0,40(r1); /* the lower part of the GPRs */ \
|
||
|
|
stw r0,48(r1); /* was already overwritten on */ \
|
||
|
|
stw r0,56(r1); /* the way down to here */ \
|
||
|
|
stw r0,64(r1); \
|
||
|
|
stw r0,72(r1); \
|
||
|
|
stw r0,80(r1); \
|
||
|
|
addi r1,r1,128; /* cleanup stack frame */
|
||
|
|
|
||
|
|
#ifdef __BIG_ENDIAN__
|
||
|
|
#define LOAD_DATA(reg, off) \
|
||
|
|
lwz reg,off(rWP); /* load data */
|
||
|
|
#define NEXT_BLOCK \
|
||
|
|
addi rWP,rWP,64; /* increment per block */
|
||
|
|
#else
|
||
|
|
#define LOAD_DATA(reg, off) \
|
||
|
|
lwbrx reg,0,rWP; /* load data */ \
|
||
|
|
addi rWP,rWP,4; /* increment per word */
|
||
|
|
#define NEXT_BLOCK /* nothing to do */
|
||
|
|
#endif
|
||
|
|
|
||
|
|
#define R_LOAD_W(a, b, c, d, e, f, g, h, w, off) \
|
||
|
|
LOAD_DATA(w, off) /* 1: W */ \
|
||
|
|
rotrwi rT0,e,6; /* 1: S1 = e rotr 6 */ \
|
||
|
|
rotrwi rT1,e,11; /* 1: S1' = e rotr 11 */ \
|
||
|
|
rotrwi rT2,e,25; /* 1: S1" = e rotr 25 */ \
|
||
|
|
xor rT0,rT0,rT1; /* 1: S1 = S1 xor S1' */ \
|
||
|
|
and rT3,e,f; /* 1: ch = e and f */ \
|
||
|
|
xor rT0,rT0,rT2; /* 1: S1 = S1 xor S1" */ \
|
||
|
|
andc rT1,g,e; /* 1: ch' = ~e and g */ \
|
||
|
|
lwz rT2,off(rKP); /* 1: K */ \
|
||
|
|
xor rT3,rT3,rT1; /* 1: ch = ch xor ch' */ \
|
||
|
|
add h,h,rT0; /* 1: temp1 = h + S1 */ \
|
||
|
|
add rT3,rT3,w; /* 1: temp1' = ch + w */ \
|
||
|
|
rotrwi rT0,a,2; /* 1: S0 = a rotr 2 */ \
|
||
|
|
add h,h,rT3; /* 1: temp1 = temp1 + temp1' */ \
|
||
|
|
rotrwi rT1,a,13; /* 1: S0' = a rotr 13 */ \
|
||
|
|
add h,h,rT2; /* 1: temp1 = temp1 + K */ \
|
||
|
|
rotrwi rT3,a,22; /* 1: S0" = a rotr 22 */ \
|
||
|
|
xor rT0,rT0,rT1; /* 1: S0 = S0 xor S0' */ \
|
||
|
|
add d,d,h; /* 1: d = d + temp1 */ \
|
||
|
|
xor rT3,rT0,rT3; /* 1: S0 = S0 xor S0" */ \
|
||
|
|
evmergelo w,w,w; /* shift W */ \
|
||
|
|
or rT2,a,b; /* 1: maj = a or b */ \
|
||
|
|
and rT1,a,b; /* 1: maj' = a and b */ \
|
||
|
|
and rT2,rT2,c; /* 1: maj = maj and c */ \
|
||
|
|
LOAD_DATA(w, off+4) /* 2: W */ \
|
||
|
|
or rT2,rT1,rT2; /* 1: maj = maj or maj' */ \
|
||
|
|
rotrwi rT0,d,6; /* 2: S1 = e rotr 6 */ \
|
||
|
|
add rT3,rT3,rT2; /* 1: temp2 = S0 + maj */ \
|
||
|
|
rotrwi rT1,d,11; /* 2: S1' = e rotr 11 */ \
|
||
|
|
add h,h,rT3; /* 1: h = temp1 + temp2 */ \
|
||
|
|
rotrwi rT2,d,25; /* 2: S1" = e rotr 25 */ \
|
||
|
|
xor rT0,rT0,rT1; /* 2: S1 = S1 xor S1' */ \
|
||
|
|
and rT3,d,e; /* 2: ch = e and f */ \
|
||
|
|
xor rT0,rT0,rT2; /* 2: S1 = S1 xor S1" */ \
|
||
|
|
andc rT1,f,d; /* 2: ch' = ~e and g */ \
|
||
|
|
lwz rT2,off+4(rKP); /* 2: K */ \
|
||
|
|
xor rT3,rT3,rT1; /* 2: ch = ch xor ch' */ \
|
||
|
|
add g,g,rT0; /* 2: temp1 = h + S1 */ \
|
||
|
|
add rT3,rT3,w; /* 2: temp1' = ch + w */ \
|
||
|
|
rotrwi rT0,h,2; /* 2: S0 = a rotr 2 */ \
|
||
|
|
add g,g,rT3; /* 2: temp1 = temp1 + temp1' */ \
|
||
|
|
rotrwi rT1,h,13; /* 2: S0' = a rotr 13 */ \
|
||
|
|
add g,g,rT2; /* 2: temp1 = temp1 + K */ \
|
||
|
|
rotrwi rT3,h,22; /* 2: S0" = a rotr 22 */ \
|
||
|
|
xor rT0,rT0,rT1; /* 2: S0 = S0 xor S0' */ \
|
||
|
|
or rT2,h,a; /* 2: maj = a or b */ \
|
||
|
|
xor rT3,rT0,rT3; /* 2: S0 = S0 xor S0" */ \
|
||
|
|
and rT1,h,a; /* 2: maj' = a and b */ \
|
||
|
|
and rT2,rT2,b; /* 2: maj = maj and c */ \
|
||
|
|
add c,c,g; /* 2: d = d + temp1 */ \
|
||
|
|
or rT2,rT1,rT2; /* 2: maj = maj or maj' */ \
|
||
|
|
add rT3,rT3,rT2; /* 2: temp2 = S0 + maj */ \
|
||
|
|
add g,g,rT3 /* 2: h = temp1 + temp2 */
|
||
|
|
|
||
|
|
#define R_CALC_W(a, b, c, d, e, f, g, h, w0, w1, w4, w5, w7, k, off) \
|
||
|
|
rotrwi rT2,e,6; /* 1: S1 = e rotr 6 */ \
|
||
|
|
evmergelohi rT0,w0,w1; /* w[-15] */ \
|
||
|
|
rotrwi rT3,e,11; /* 1: S1' = e rotr 11 */ \
|
||
|
|
evsrwiu rT1,rT0,3; /* s0 = w[-15] >> 3 */ \
|
||
|
|
xor rT2,rT2,rT3; /* 1: S1 = S1 xor S1' */ \
|
||
|
|
evrlwi rT0,rT0,25; /* s0' = w[-15] rotr 7 */ \
|
||
|
|
rotrwi rT3,e,25; /* 1: S1' = e rotr 25 */ \
|
||
|
|
evxor rT1,rT1,rT0; /* s0 = s0 xor s0' */ \
|
||
|
|
xor rT2,rT2,rT3; /* 1: S1 = S1 xor S1' */ \
|
||
|
|
evrlwi rT0,rT0,21; /* s0' = w[-15] rotr 18 */ \
|
||
|
|
add h,h,rT2; /* 1: temp1 = h + S1 */ \
|
||
|
|
evxor rT0,rT0,rT1; /* s0 = s0 xor s0' */ \
|
||
|
|
and rT2,e,f; /* 1: ch = e and f */ \
|
||
|
|
evaddw w0,w0,rT0; /* w = w[-16] + s0 */ \
|
||
|
|
andc rT3,g,e; /* 1: ch' = ~e and g */ \
|
||
|
|
evsrwiu rT0,w7,10; /* s1 = w[-2] >> 10 */ \
|
||
|
|
xor rT2,rT2,rT3; /* 1: ch = ch xor ch' */ \
|
||
|
|
evrlwi rT1,w7,15; /* s1' = w[-2] rotr 17 */ \
|
||
|
|
add h,h,rT2; /* 1: temp1 = temp1 + ch */ \
|
||
|
|
evxor rT0,rT0,rT1; /* s1 = s1 xor s1' */ \
|
||
|
|
rotrwi rT2,a,2; /* 1: S0 = a rotr 2 */ \
|
||
|
|
evrlwi rT1,w7,13; /* s1' = w[-2] rotr 19 */ \
|
||
|
|
rotrwi rT3,a,13; /* 1: S0' = a rotr 13 */ \
|
||
|
|
evxor rT0,rT0,rT1; /* s1 = s1 xor s1' */ \
|
||
|
|
xor rT2,rT2,rT3; /* 1: S0 = S0 xor S0' */ \
|
||
|
|
evldw rT1,off(rKP); /* k */ \
|
||
|
|
rotrwi rT3,a,22; /* 1: S0' = a rotr 22 */ \
|
||
|
|
evaddw w0,w0,rT0; /* w = w + s1 */ \
|
||
|
|
xor rT2,rT2,rT3; /* 1: S0 = S0 xor S0' */ \
|
||
|
|
evmergelohi rT0,w4,w5; /* w[-7] */ \
|
||
|
|
and rT3,a,b; /* 1: maj = a and b */ \
|
||
|
|
evaddw w0,w0,rT0; /* w = w + w[-7] */ \
|
||
|
|
CMP_K##k##_LOOP \
|
||
|
|
add rT2,rT2,rT3; /* 1: temp2 = S0 + maj */ \
|
||
|
|
evaddw rT1,rT1,w0; /* wk = w + k */ \
|
||
|
|
xor rT3,a,b; /* 1: maj = a xor b */ \
|
||
|
|
evmergehi rT0,rT1,rT1; /* wk1/wk2 */ \
|
||
|
|
and rT3,rT3,c; /* 1: maj = maj and c */ \
|
||
|
|
add h,h,rT0; /* 1: temp1 = temp1 + wk */ \
|
||
|
|
add rT2,rT2,rT3; /* 1: temp2 = temp2 + maj */ \
|
||
|
|
add g,g,rT1; /* 2: temp1 = temp1 + wk */ \
|
||
|
|
add d,d,h; /* 1: d = d + temp1 */ \
|
||
|
|
rotrwi rT0,d,6; /* 2: S1 = e rotr 6 */ \
|
||
|
|
add h,h,rT2; /* 1: h = temp1 + temp2 */ \
|
||
|
|
rotrwi rT1,d,11; /* 2: S1' = e rotr 11 */ \
|
||
|
|
rotrwi rT2,d,25; /* 2: S" = e rotr 25 */ \
|
||
|
|
xor rT0,rT0,rT1; /* 2: S1 = S1 xor S1' */ \
|
||
|
|
and rT3,d,e; /* 2: ch = e and f */ \
|
||
|
|
xor rT0,rT0,rT2; /* 2: S1 = S1 xor S1" */ \
|
||
|
|
andc rT1,f,d; /* 2: ch' = ~e and g */ \
|
||
|
|
add g,g,rT0; /* 2: temp1 = h + S1 */ \
|
||
|
|
xor rT3,rT3,rT1; /* 2: ch = ch xor ch' */ \
|
||
|
|
rotrwi rT0,h,2; /* 2: S0 = a rotr 2 */ \
|
||
|
|
add g,g,rT3; /* 2: temp1 = temp1 + ch */ \
|
||
|
|
rotrwi rT1,h,13; /* 2: S0' = a rotr 13 */ \
|
||
|
|
rotrwi rT3,h,22; /* 2: S0" = a rotr 22 */ \
|
||
|
|
xor rT0,rT0,rT1; /* 2: S0 = S0 xor S0' */ \
|
||
|
|
or rT2,h,a; /* 2: maj = a or b */ \
|
||
|
|
and rT1,h,a; /* 2: maj' = a and b */ \
|
||
|
|
and rT2,rT2,b; /* 2: maj = maj and c */ \
|
||
|
|
xor rT3,rT0,rT3; /* 2: S0 = S0 xor S0" */ \
|
||
|
|
or rT2,rT1,rT2; /* 2: maj = maj or maj' */ \
|
||
|
|
add c,c,g; /* 2: d = d + temp1 */ \
|
||
|
|
add rT3,rT3,rT2; /* 2: temp2 = S0 + maj */ \
|
||
|
|
add g,g,rT3 /* 2: h = temp1 + temp2 */
|
||
|
|
|
||
|
|
_GLOBAL(ppc_spe_sha256_transform)
|
||
|
|
INITIALIZE
|
||
|
|
|
||
|
|
mtctr r5
|
||
|
|
lwz rH0,0(rHP)
|
||
|
|
lwz rH1,4(rHP)
|
||
|
|
lwz rH2,8(rHP)
|
||
|
|
lwz rH3,12(rHP)
|
||
|
|
lwz rH4,16(rHP)
|
||
|
|
lwz rH5,20(rHP)
|
||
|
|
lwz rH6,24(rHP)
|
||
|
|
lwz rH7,28(rHP)
|
||
|
|
|
||
|
|
ppc_spe_sha256_main:
|
||
|
|
lis rKP,PPC_SPE_SHA256_K@ha
|
||
|
|
addi rKP,rKP,PPC_SPE_SHA256_K@l
|
||
|
|
|
||
|
|
R_LOAD_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, rW0, 0)
|
||
|
|
R_LOAD_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, rW1, 8)
|
||
|
|
R_LOAD_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, rW2, 16)
|
||
|
|
R_LOAD_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, rW3, 24)
|
||
|
|
R_LOAD_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, rW4, 32)
|
||
|
|
R_LOAD_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, rW5, 40)
|
||
|
|
R_LOAD_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, rW6, 48)
|
||
|
|
R_LOAD_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, rW7, 56)
|
||
|
|
ppc_spe_sha256_16_rounds:
|
||
|
|
addi rKP,rKP,64
|
||
|
|
R_CALC_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7,
|
||
|
|
rW0, rW1, rW4, rW5, rW7, N, 0)
|
||
|
|
R_CALC_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5,
|
||
|
|
rW1, rW2, rW5, rW6, rW0, N, 8)
|
||
|
|
R_CALC_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3,
|
||
|
|
rW2, rW3, rW6, rW7, rW1, N, 16)
|
||
|
|
R_CALC_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1,
|
||
|
|
rW3, rW4, rW7, rW0, rW2, N, 24)
|
||
|
|
R_CALC_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7,
|
||
|
|
rW4, rW5, rW0, rW1, rW3, N, 32)
|
||
|
|
R_CALC_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5,
|
||
|
|
rW5, rW6, rW1, rW2, rW4, N, 40)
|
||
|
|
R_CALC_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3,
|
||
|
|
rW6, rW7, rW2, rW3, rW5, N, 48)
|
||
|
|
R_CALC_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1,
|
||
|
|
rW7, rW0, rW3, rW4, rW6, C, 56)
|
||
|
|
bt gt,ppc_spe_sha256_16_rounds
|
||
|
|
|
||
|
|
lwz rW0,0(rHP)
|
||
|
|
NEXT_BLOCK
|
||
|
|
lwz rW1,4(rHP)
|
||
|
|
lwz rW2,8(rHP)
|
||
|
|
lwz rW3,12(rHP)
|
||
|
|
lwz rW4,16(rHP)
|
||
|
|
lwz rW5,20(rHP)
|
||
|
|
lwz rW6,24(rHP)
|
||
|
|
lwz rW7,28(rHP)
|
||
|
|
|
||
|
|
add rH0,rH0,rW0
|
||
|
|
stw rH0,0(rHP)
|
||
|
|
add rH1,rH1,rW1
|
||
|
|
stw rH1,4(rHP)
|
||
|
|
add rH2,rH2,rW2
|
||
|
|
stw rH2,8(rHP)
|
||
|
|
add rH3,rH3,rW3
|
||
|
|
stw rH3,12(rHP)
|
||
|
|
add rH4,rH4,rW4
|
||
|
|
stw rH4,16(rHP)
|
||
|
|
add rH5,rH5,rW5
|
||
|
|
stw rH5,20(rHP)
|
||
|
|
add rH6,rH6,rW6
|
||
|
|
stw rH6,24(rHP)
|
||
|
|
add rH7,rH7,rW7
|
||
|
|
stw rH7,28(rHP)
|
||
|
|
|
||
|
|
bdnz ppc_spe_sha256_main
|
||
|
|
|
||
|
|
FINALIZE
|
||
|
|
blr
|
||
|
|
|
||
|
|
.data
|
||
|
|
.align 5
|
||
|
|
PPC_SPE_SHA256_K:
|
||
|
|
.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
|
||
|
|
.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
|
||
|
|
.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
|
||
|
|
.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
|
||
|
|
.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
|
||
|
|
.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
|
||
|
|
.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
|
||
|
|
.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
|
||
|
|
.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
|
||
|
|
.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
|
||
|
|
.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
|
||
|
|
.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
|
||
|
|
.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
|
||
|
|
.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
|
||
|
|
.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
|
||
|
|
.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
|