1908 lines
47 KiB
Perl
1908 lines
47 KiB
Perl
|
#! /usr/bin/env perl
|
||
|
|
# Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved.
|
||
|
|
#
|
||
|
|
# Licensed under the Apache License 2.0 (the "License"). You may not use
|
||
|
|
# this file except in compliance with the License. You can obtain a copy
|
||
|
|
# in the file LICENSE in the source distribution or at
|
||
|
|
# https://www.openssl.org/source/license.html
|
||
|
|
|
||
|
|
|
||
|
|
# ====================================================================
|
||
|
|
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
|
||
|
|
# project. The module is, however, dual licensed under OpenSSL and
|
||
|
|
# CRYPTOGAMS licenses depending on where you obtain it. For further
|
||
|
|
# details see http://www.openssl.org/~appro/cryptogams/.
|
||
|
|
# ====================================================================
|
||
|
|
|
||
|
|
# March 2015
|
||
|
|
#
|
||
|
|
# "Teaser" Montgomery multiplication module for ARMv8. Needs more
|
||
|
|
# work. While it does improve RSA sign performance by 20-30% (less for
|
||
|
|
# longer keys) on most processors, for some reason RSA2048 is not
|
||
|
|
# faster and RSA4096 goes 15-20% slower on Cortex-A57. Multiplication
|
||
|
|
# instruction issue rate is limited on processor in question, meaning
|
||
|
|
# that dedicated squaring procedure is a must. Well, actually all
|
||
|
|
# contemporary AArch64 processors seem to have limited multiplication
|
||
|
|
# issue rate, i.e. they can't issue multiplication every cycle, which
|
||
|
|
# explains moderate improvement coefficients in comparison to
|
||
|
|
# compiler-generated code. Recall that compiler is instructed to use
|
||
|
|
# umulh and therefore uses same amount of multiplication instructions
|
||
|
|
# to do the job. Assembly's edge is to minimize number of "collateral"
|
||
|
|
# instructions and of course instruction scheduling.
|
||
|
|
#
|
||
|
|
# April 2015
|
||
|
|
#
|
||
|
|
# Squaring procedure that handles lengths divisible by 8 improves
|
||
|
|
# RSA/DSA performance by 25-40-60% depending on processor and key
|
||
|
|
# length. Overall improvement coefficients are always positive in
|
||
|
|
# comparison to compiler-generated code. On Cortex-A57 improvement
|
||
|
|
# is still modest on longest key lengths, while others exhibit e.g.
|
||
|
|
# 50-70% improvement for RSA4096 sign. RSA2048 sign is ~25% faster
|
||
|
|
# on Cortex-A57 and ~60-100% faster on others.
|
||
|
|
|
||
|
|
# $output is the last argument if it looks like a file (it has an extension)
|
||
|
|
# $flavour is the first argument if it doesn't look like a file
|
||
|
|
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||
|
|
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||
|
|
|
||
|
|
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
||
|
|
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
|
||
|
|
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
|
||
|
|
die "can't locate arm-xlate.pl";
|
||
|
|
|
||
|
|
open OUT,"| \"$^X\" $xlate $flavour \"$output\""
|
||
|
|
or die "can't call $xlate: $1";
|
||
|
|
*STDOUT=*OUT;
|
||
|
|
|
||
|
|
($lo0,$hi0,$aj,$m0,$alo,$ahi,
|
||
|
|
$lo1,$hi1,$nj,$m1,$nlo,$nhi,
|
||
|
|
$ovf, $i,$j,$tp,$tj) = map("x$_",6..17,19..24);
|
||
|
|
|
||
|
|
# int bn_mul_mont(
|
||
|
|
$rp="x0"; # BN_ULONG *rp,
|
||
|
|
$ap="x1"; # const BN_ULONG *ap,
|
||
|
|
$bp="x2"; # const BN_ULONG *bp,
|
||
|
|
$np="x3"; # const BN_ULONG *np,
|
||
|
|
$n0="x4"; # const BN_ULONG *n0,
|
||
|
|
$num="x5"; # int num);
|
||
|
|
|
||
|
|
$code.=<<___;
|
||
|
|
#include "arm_arch.h"
|
||
|
|
#ifndef __KERNEL__
|
||
|
|
.extern OPENSSL_armv8_rsa_neonized
|
||
|
|
.hidden OPENSSL_armv8_rsa_neonized
|
||
|
|
#endif
|
||
|
|
.text
|
||
|
|
|
||
|
|
.globl bn_mul_mont
|
||
|
|
.type bn_mul_mont,%function
|
||
|
|
.align 5
|
||
|
|
bn_mul_mont:
|
||
|
|
AARCH64_SIGN_LINK_REGISTER
|
||
|
|
.Lbn_mul_mont:
|
||
|
|
tst $num,#3
|
||
|
|
b.ne .Lmul_mont
|
||
|
|
cmp $num,#32
|
||
|
|
b.le .Lscalar_impl
|
||
|
|
#ifndef __KERNEL__
|
||
|
|
adrp x17,OPENSSL_armv8_rsa_neonized
|
||
|
|
ldr w17,[x17,#:lo12:OPENSSL_armv8_rsa_neonized]
|
||
|
|
cbnz w17, bn_mul8x_mont_neon
|
||
|
|
#endif
|
||
|
|
|
||
|
|
.Lscalar_impl:
|
||
|
|
tst $num,#7
|
||
|
|
b.eq __bn_sqr8x_mont
|
||
|
|
tst $num,#3
|
||
|
|
b.eq __bn_mul4x_mont
|
||
|
|
|
||
|
|
.Lmul_mont:
|
||
|
|
stp x29,x30,[sp,#-64]!
|
||
|
|
add x29,sp,#0
|
||
|
|
stp x19,x20,[sp,#16]
|
||
|
|
stp x21,x22,[sp,#32]
|
||
|
|
stp x23,x24,[sp,#48]
|
||
|
|
|
||
|
|
ldr $m0,[$bp],#8 // bp[0]
|
||
|
|
sub $tp,sp,$num,lsl#3
|
||
|
|
ldp $hi0,$aj,[$ap],#16 // ap[0..1]
|
||
|
|
lsl $num,$num,#3
|
||
|
|
ldr $n0,[$n0] // *n0
|
||
|
|
and $tp,$tp,#-16 // ABI says so
|
||
|
|
ldp $hi1,$nj,[$np],#16 // np[0..1]
|
||
|
|
|
||
|
|
mul $lo0,$hi0,$m0 // ap[0]*bp[0]
|
||
|
|
sub $j,$num,#16 // j=num-2
|
||
|
|
umulh $hi0,$hi0,$m0
|
||
|
|
mul $alo,$aj,$m0 // ap[1]*bp[0]
|
||
|
|
umulh $ahi,$aj,$m0
|
||
|
|
|
||
|
|
mul $m1,$lo0,$n0 // "tp[0]"*n0
|
||
|
|
mov sp,$tp // alloca
|
||
|
|
|
||
|
|
// (*) mul $lo1,$hi1,$m1 // np[0]*m1
|
||
|
|
umulh $hi1,$hi1,$m1
|
||
|
|
mul $nlo,$nj,$m1 // np[1]*m1
|
||
|
|
// (*) adds $lo1,$lo1,$lo0 // discarded
|
||
|
|
// (*) As for removal of first multiplication and addition
|
||
|
|
// instructions. The outcome of first addition is
|
||
|
|
// guaranteed to be zero, which leaves two computationally
|
||
|
|
// significant outcomes: it either carries or not. Then
|
||
|
|
// question is when does it carry? Is there alternative
|
||
|
|
// way to deduce it? If you follow operations, you can
|
||
|
|
// observe that condition for carry is quite simple:
|
||
|
|
// $lo0 being non-zero. So that carry can be calculated
|
||
|
|
// by adding -1 to $lo0. That's what next instruction does.
|
||
|
|
subs xzr,$lo0,#1 // (*)
|
||
|
|
umulh $nhi,$nj,$m1
|
||
|
|
adc $hi1,$hi1,xzr
|
||
|
|
cbz $j,.L1st_skip
|
||
|
|
|
||
|
|
.L1st:
|
||
|
|
ldr $aj,[$ap],#8
|
||
|
|
adds $lo0,$alo,$hi0
|
||
|
|
sub $j,$j,#8 // j--
|
||
|
|
adc $hi0,$ahi,xzr
|
||
|
|
|
||
|
|
ldr $nj,[$np],#8
|
||
|
|
adds $lo1,$nlo,$hi1
|
||
|
|
mul $alo,$aj,$m0 // ap[j]*bp[0]
|
||
|
|
adc $hi1,$nhi,xzr
|
||
|
|
umulh $ahi,$aj,$m0
|
||
|
|
|
||
|
|
adds $lo1,$lo1,$lo0
|
||
|
|
mul $nlo,$nj,$m1 // np[j]*m1
|
||
|
|
adc $hi1,$hi1,xzr
|
||
|
|
umulh $nhi,$nj,$m1
|
||
|
|
str $lo1,[$tp],#8 // tp[j-1]
|
||
|
|
cbnz $j,.L1st
|
||
|
|
|
||
|
|
.L1st_skip:
|
||
|
|
adds $lo0,$alo,$hi0
|
||
|
|
sub $ap,$ap,$num // rewind $ap
|
||
|
|
adc $hi0,$ahi,xzr
|
||
|
|
|
||
|
|
adds $lo1,$nlo,$hi1
|
||
|
|
sub $np,$np,$num // rewind $np
|
||
|
|
adc $hi1,$nhi,xzr
|
||
|
|
|
||
|
|
adds $lo1,$lo1,$lo0
|
||
|
|
sub $i,$num,#8 // i=num-1
|
||
|
|
adcs $hi1,$hi1,$hi0
|
||
|
|
|
||
|
|
adc $ovf,xzr,xzr // upmost overflow bit
|
||
|
|
stp $lo1,$hi1,[$tp]
|
||
|
|
|
||
|
|
.Louter:
|
||
|
|
ldr $m0,[$bp],#8 // bp[i]
|
||
|
|
ldp $hi0,$aj,[$ap],#16
|
||
|
|
ldr $tj,[sp] // tp[0]
|
||
|
|
add $tp,sp,#8
|
||
|
|
|
||
|
|
mul $lo0,$hi0,$m0 // ap[0]*bp[i]
|
||
|
|
sub $j,$num,#16 // j=num-2
|
||
|
|
umulh $hi0,$hi0,$m0
|
||
|
|
ldp $hi1,$nj,[$np],#16
|
||
|
|
mul $alo,$aj,$m0 // ap[1]*bp[i]
|
||
|
|
adds $lo0,$lo0,$tj
|
||
|
|
umulh $ahi,$aj,$m0
|
||
|
|
adc $hi0,$hi0,xzr
|
||
|
|
|
||
|
|
mul $m1,$lo0,$n0
|
||
|
|
sub $i,$i,#8 // i--
|
||
|
|
|
||
|
|
// (*) mul $lo1,$hi1,$m1 // np[0]*m1
|
||
|
|
umulh $hi1,$hi1,$m1
|
||
|
|
mul $nlo,$nj,$m1 // np[1]*m1
|
||
|
|
// (*) adds $lo1,$lo1,$lo0
|
||
|
|
subs xzr,$lo0,#1 // (*)
|
||
|
|
umulh $nhi,$nj,$m1
|
||
|
|
cbz $j,.Linner_skip
|
||
|
|
|
||
|
|
.Linner:
|
||
|
|
ldr $aj,[$ap],#8
|
||
|
|
adc $hi1,$hi1,xzr
|
||
|
|
ldr $tj,[$tp],#8 // tp[j]
|
||
|
|
adds $lo0,$alo,$hi0
|
||
|
|
sub $j,$j,#8 // j--
|
||
|
|
adc $hi0,$ahi,xzr
|
||
|
|
|
||
|
|
adds $lo1,$nlo,$hi1
|
||
|
|
ldr $nj,[$np],#8
|
||
|
|
adc $hi1,$nhi,xzr
|
||
|
|
|
||
|
|
mul $alo,$aj,$m0 // ap[j]*bp[i]
|
||
|
|
adds $lo0,$lo0,$tj
|
||
|
|
umulh $ahi,$aj,$m0
|
||
|
|
adc $hi0,$hi0,xzr
|
||
|
|
|
||
|
|
mul $nlo,$nj,$m1 // np[j]*m1
|
||
|
|
adds $lo1,$lo1,$lo0
|
||
|
|
umulh $nhi,$nj,$m1
|
||
|
|
stur $lo1,[$tp,#-16] // tp[j-1]
|
||
|
|
cbnz $j,.Linner
|
||
|
|
|
||
|
|
.Linner_skip:
|
||
|
|
ldr $tj,[$tp],#8 // tp[j]
|
||
|
|
adc $hi1,$hi1,xzr
|
||
|
|
adds $lo0,$alo,$hi0
|
||
|
|
sub $ap,$ap,$num // rewind $ap
|
||
|
|
adc $hi0,$ahi,xzr
|
||
|
|
|
||
|
|
adds $lo1,$nlo,$hi1
|
||
|
|
sub $np,$np,$num // rewind $np
|
||
|
|
adcs $hi1,$nhi,$ovf
|
||
|
|
adc $ovf,xzr,xzr
|
||
|
|
|
||
|
|
adds $lo0,$lo0,$tj
|
||
|
|
adc $hi0,$hi0,xzr
|
||
|
|
|
||
|
|
adds $lo1,$lo1,$lo0
|
||
|
|
adcs $hi1,$hi1,$hi0
|
||
|
|
adc $ovf,$ovf,xzr // upmost overflow bit
|
||
|
|
stp $lo1,$hi1,[$tp,#-16]
|
||
|
|
|
||
|
|
cbnz $i,.Louter
|
||
|
|
|
||
|
|
// Final step. We see if result is larger than modulus, and
|
||
|
|
// if it is, subtract the modulus. But comparison implies
|
||
|
|
// subtraction. So we subtract modulus, see if it borrowed,
|
||
|
|
// and conditionally copy original value.
|
||
|
|
ldr $tj,[sp] // tp[0]
|
||
|
|
add $tp,sp,#8
|
||
|
|
ldr $nj,[$np],#8 // np[0]
|
||
|
|
subs $j,$num,#8 // j=num-1 and clear borrow
|
||
|
|
mov $ap,$rp
|
||
|
|
.Lsub:
|
||
|
|
sbcs $aj,$tj,$nj // tp[j]-np[j]
|
||
|
|
ldr $tj,[$tp],#8
|
||
|
|
sub $j,$j,#8 // j--
|
||
|
|
ldr $nj,[$np],#8
|
||
|
|
str $aj,[$ap],#8 // rp[j]=tp[j]-np[j]
|
||
|
|
cbnz $j,.Lsub
|
||
|
|
|
||
|
|
sbcs $aj,$tj,$nj
|
||
|
|
sbcs $ovf,$ovf,xzr // did it borrow?
|
||
|
|
str $aj,[$ap],#8 // rp[num-1]
|
||
|
|
|
||
|
|
ldr $tj,[sp] // tp[0]
|
||
|
|
add $tp,sp,#8
|
||
|
|
ldr $aj,[$rp],#8 // rp[0]
|
||
|
|
sub $num,$num,#8 // num--
|
||
|
|
nop
|
||
|
|
.Lcond_copy:
|
||
|
|
sub $num,$num,#8 // num--
|
||
|
|
csel $nj,$tj,$aj,lo // did it borrow?
|
||
|
|
ldr $tj,[$tp],#8
|
||
|
|
ldr $aj,[$rp],#8
|
||
|
|
stur xzr,[$tp,#-16] // wipe tp
|
||
|
|
stur $nj,[$rp,#-16]
|
||
|
|
cbnz $num,.Lcond_copy
|
||
|
|
|
||
|
|
csel $nj,$tj,$aj,lo
|
||
|
|
stur xzr,[$tp,#-8] // wipe tp
|
||
|
|
stur $nj,[$rp,#-8]
|
||
|
|
|
||
|
|
ldp x19,x20,[x29,#16]
|
||
|
|
mov sp,x29
|
||
|
|
ldp x21,x22,[x29,#32]
|
||
|
|
mov x0,#1
|
||
|
|
ldp x23,x24,[x29,#48]
|
||
|
|
ldr x29,[sp],#64
|
||
|
|
AARCH64_VALIDATE_LINK_REGISTER
|
||
|
|
ret
|
||
|
|
.size bn_mul_mont,.-bn_mul_mont
|
||
|
|
___
|
||
|
|
{
|
||
|
|
my ($A0,$A1,$N0,$N1)=map("v$_",(0..3));
|
||
|
|
my ($Z,$Temp)=("v4.16b","v5");
|
||
|
|
my @ACC=map("v$_",(6..13));
|
||
|
|
my ($Bi,$Ni,$M0)=map("v$_",(28..30));
|
||
|
|
my $sBi="s28";
|
||
|
|
my $sM0="s30";
|
||
|
|
my $zero="v14";
|
||
|
|
my $temp="v15";
|
||
|
|
my $ACCTemp="v16";
|
||
|
|
|
||
|
|
my ($rptr,$aptr,$bptr,$nptr,$n0,$num)=map("x$_",(0..5));
|
||
|
|
my ($tinptr,$toutptr,$inner,$outer,$bnptr)=map("x$_",(6..11));
|
||
|
|
|
||
|
|
$code.=<<___;
|
||
|
|
.type bn_mul8x_mont_neon,%function
|
||
|
|
.align 5
|
||
|
|
bn_mul8x_mont_neon:
|
||
|
|
// Not adding AARCH64_SIGN_LINK_REGISTER here because bn_mul8x_mont_neon is jumped to
|
||
|
|
// only from bn_mul_mont which has already signed the return address.
|
||
|
|
stp x29,x30,[sp,#-80]!
|
||
|
|
mov x16,sp
|
||
|
|
stp d8,d9,[sp,#16]
|
||
|
|
stp d10,d11,[sp,#32]
|
||
|
|
stp d12,d13,[sp,#48]
|
||
|
|
stp d14,d15,[sp,#64]
|
||
|
|
lsl $num,$num,#1
|
||
|
|
eor $zero.16b,$zero.16b,$zero.16b
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.LNEON_8n:
|
||
|
|
eor @ACC[0].16b,@ACC[0].16b,@ACC[0].16b
|
||
|
|
sub $toutptr,sp,#128
|
||
|
|
eor @ACC[1].16b,@ACC[1].16b,@ACC[1].16b
|
||
|
|
sub $toutptr,$toutptr,$num,lsl#4
|
||
|
|
eor @ACC[2].16b,@ACC[2].16b,@ACC[2].16b
|
||
|
|
and $toutptr,$toutptr,#-64
|
||
|
|
eor @ACC[3].16b,@ACC[3].16b,@ACC[3].16b
|
||
|
|
mov sp,$toutptr // alloca
|
||
|
|
eor @ACC[4].16b,@ACC[4].16b,@ACC[4].16b
|
||
|
|
add $toutptr,$toutptr,#256
|
||
|
|
eor @ACC[5].16b,@ACC[5].16b,@ACC[5].16b
|
||
|
|
sub $inner,$num,#8
|
||
|
|
eor @ACC[6].16b,@ACC[6].16b,@ACC[6].16b
|
||
|
|
eor @ACC[7].16b,@ACC[7].16b,@ACC[7].16b
|
||
|
|
|
||
|
|
.LNEON_8n_init:
|
||
|
|
st1 {@ACC[0].2d,@ACC[1].2d},[$toutptr],#32
|
||
|
|
subs $inner,$inner,#8
|
||
|
|
st1 {@ACC[2].2d,@ACC[3].2d},[$toutptr],#32
|
||
|
|
st1 {@ACC[4].2d,@ACC[5].2d},[$toutptr],#32
|
||
|
|
st1 {@ACC[6].2d,@ACC[7].2d},[$toutptr],#32
|
||
|
|
bne .LNEON_8n_init
|
||
|
|
|
||
|
|
add $tinptr,sp,#256
|
||
|
|
ld1 {$A0.4s,$A1.4s},[$aptr],#32
|
||
|
|
add $bnptr,sp,#8
|
||
|
|
ldr $sM0,[$n0],#4
|
||
|
|
mov $outer,$num
|
||
|
|
b .LNEON_8n_outer
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.LNEON_8n_outer:
|
||
|
|
ldr $sBi,[$bptr],#4 // *b++
|
||
|
|
uxtl $Bi.4s,$Bi.4h
|
||
|
|
add $toutptr,sp,#128
|
||
|
|
ld1 {$N0.4s,$N1.4s},[$nptr],#32
|
||
|
|
|
||
|
|
umlal @ACC[0].2d,$Bi.2s,$A0.s[0]
|
||
|
|
umlal @ACC[1].2d,$Bi.2s,$A0.s[1]
|
||
|
|
umlal @ACC[2].2d,$Bi.2s,$A0.s[2]
|
||
|
|
shl $Ni.2d,@ACC[0].2d,#16
|
||
|
|
ext $Ni.16b,$Ni.16b,$Ni.16b,#8
|
||
|
|
umlal @ACC[3].2d,$Bi.2s,$A0.s[3]
|
||
|
|
add $Ni.2d,$Ni.2d,@ACC[0].2d
|
||
|
|
umlal @ACC[4].2d,$Bi.2s,$A1.s[0]
|
||
|
|
mul $Ni.2s,$Ni.2s,$M0.2s
|
||
|
|
umlal @ACC[5].2d,$Bi.2s,$A1.s[1]
|
||
|
|
st1 {$Bi.2s},[sp] // put aside smashed b[8*i+0]
|
||
|
|
umlal @ACC[6].2d,$Bi.2s,$A1.s[2]
|
||
|
|
uxtl $Ni.4s,$Ni.4h
|
||
|
|
umlal @ACC[7].2d,$Bi.2s,$A1.s[3]
|
||
|
|
___
|
||
|
|
for ($i=0; $i<7;) {
|
||
|
|
$code.=<<___;
|
||
|
|
ldr $sBi,[$bptr],#4 // *b++
|
||
|
|
umlal @ACC[0].2d,$Ni.2s,$N0.s[0]
|
||
|
|
umlal @ACC[1].2d,$Ni.2s,$N0.s[1]
|
||
|
|
uxtl $Bi.4s,$Bi.4h
|
||
|
|
umlal @ACC[2].2d,$Ni.2s,$N0.s[2]
|
||
|
|
ushr $temp.2d,@ACC[0].2d,#16
|
||
|
|
umlal @ACC[3].2d,$Ni.2s,$N0.s[3]
|
||
|
|
umlal @ACC[4].2d,$Ni.2s,$N1.s[0]
|
||
|
|
ext @ACC[0].16b,@ACC[0].16b,@ACC[0].16b,#8
|
||
|
|
add @ACC[0].2d,@ACC[0].2d,$temp.2d
|
||
|
|
umlal @ACC[5].2d,$Ni.2s,$N1.s[1]
|
||
|
|
ushr @ACC[0].2d,@ACC[0].2d,#16
|
||
|
|
umlal @ACC[6].2d,$Ni.2s,$N1.s[2]
|
||
|
|
umlal @ACC[7].2d,$Ni.2s,$N1.s[3]
|
||
|
|
add $ACCTemp.2d,@ACC[1].2d,@ACC[0].2d
|
||
|
|
ins @ACC[1].d[0],$ACCTemp.d[0]
|
||
|
|
st1 {$Ni.2s},[$bnptr],#8 // put aside smashed m[8*i+$i]
|
||
|
|
___
|
||
|
|
push(@ACC,shift(@ACC)); $i++;
|
||
|
|
$code.=<<___;
|
||
|
|
umlal @ACC[0].2d,$Bi.2s,$A0.s[0]
|
||
|
|
ld1 {@ACC[7].2d},[$tinptr],#16
|
||
|
|
umlal @ACC[1].2d,$Bi.2s,$A0.s[1]
|
||
|
|
umlal @ACC[2].2d,$Bi.2s,$A0.s[2]
|
||
|
|
shl $Ni.2d,@ACC[0].2d,#16
|
||
|
|
ext $Ni.16b,$Ni.16b,$Ni.16b,#8
|
||
|
|
umlal @ACC[3].2d,$Bi.2s,$A0.s[3]
|
||
|
|
add $Ni.2d,$Ni.2d,@ACC[0].2d
|
||
|
|
umlal @ACC[4].2d,$Bi.2s,$A1.s[0]
|
||
|
|
mul $Ni.2s,$Ni.2s,$M0.2s
|
||
|
|
umlal @ACC[5].2d,$Bi.2s,$A1.s[1]
|
||
|
|
st1 {$Bi.2s},[$bnptr],#8 // put aside smashed b[8*i+$i]
|
||
|
|
umlal @ACC[6].2d,$Bi.2s,$A1.s[2]
|
||
|
|
uxtl $Ni.4s,$Ni.4h
|
||
|
|
umlal @ACC[7].2d,$Bi.2s,$A1.s[3]
|
||
|
|
___
|
||
|
|
}
|
||
|
|
$code.=<<___;
|
||
|
|
ld1 {$Bi.2s},[sp] // pull smashed b[8*i+0]
|
||
|
|
umlal @ACC[0].2d,$Ni.2s,$N0.s[0]
|
||
|
|
ld1 {$A0.4s,$A1.4s},[$aptr],#32
|
||
|
|
umlal @ACC[1].2d,$Ni.2s,$N0.s[1]
|
||
|
|
umlal @ACC[2].2d,$Ni.2s,$N0.s[2]
|
||
|
|
mov $Temp.16b,@ACC[0].16b
|
||
|
|
ushr $Temp.2d,$Temp.2d,#16
|
||
|
|
ext @ACC[0].16b,@ACC[0].16b,@ACC[0].16b,#8
|
||
|
|
umlal @ACC[3].2d,$Ni.2s,$N0.s[3]
|
||
|
|
umlal @ACC[4].2d,$Ni.2s,$N1.s[0]
|
||
|
|
add @ACC[0].2d,@ACC[0].2d,$Temp.2d
|
||
|
|
umlal @ACC[5].2d,$Ni.2s,$N1.s[1]
|
||
|
|
ushr @ACC[0].2d,@ACC[0].2d,#16
|
||
|
|
eor $temp.16b,$temp.16b,$temp.16b
|
||
|
|
ins @ACC[0].d[1],$temp.d[0]
|
||
|
|
umlal @ACC[6].2d,$Ni.2s,$N1.s[2]
|
||
|
|
umlal @ACC[7].2d,$Ni.2s,$N1.s[3]
|
||
|
|
add @ACC[1].2d,@ACC[1].2d,@ACC[0].2d
|
||
|
|
st1 {$Ni.2s},[$bnptr],#8 // put aside smashed m[8*i+$i]
|
||
|
|
add $bnptr,sp,#8 // rewind
|
||
|
|
___
|
||
|
|
push(@ACC,shift(@ACC));
|
||
|
|
$code.=<<___;
|
||
|
|
sub $inner,$num,#8
|
||
|
|
b .LNEON_8n_inner
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.LNEON_8n_inner:
|
||
|
|
subs $inner,$inner,#8
|
||
|
|
umlal @ACC[0].2d,$Bi.2s,$A0.s[0]
|
||
|
|
ld1 {@ACC[7].2d},[$tinptr]
|
||
|
|
umlal @ACC[1].2d,$Bi.2s,$A0.s[1]
|
||
|
|
ld1 {$Ni.2s},[$bnptr],#8 // pull smashed m[8*i+0]
|
||
|
|
umlal @ACC[2].2d,$Bi.2s,$A0.s[2]
|
||
|
|
ld1 {$N0.4s,$N1.4s},[$nptr],#32
|
||
|
|
umlal @ACC[3].2d,$Bi.2s,$A0.s[3]
|
||
|
|
b.eq .LInner_jump
|
||
|
|
add $tinptr,$tinptr,#16 // don't advance in last iteration
|
||
|
|
.LInner_jump:
|
||
|
|
umlal @ACC[4].2d,$Bi.2s,$A1.s[0]
|
||
|
|
umlal @ACC[5].2d,$Bi.2s,$A1.s[1]
|
||
|
|
umlal @ACC[6].2d,$Bi.2s,$A1.s[2]
|
||
|
|
umlal @ACC[7].2d,$Bi.2s,$A1.s[3]
|
||
|
|
___
|
||
|
|
for ($i=1; $i<8; $i++) {
|
||
|
|
$code.=<<___;
|
||
|
|
ld1 {$Bi.2s},[$bnptr],#8 // pull smashed b[8*i+$i]
|
||
|
|
umlal @ACC[0].2d,$Ni.2s,$N0.s[0]
|
||
|
|
umlal @ACC[1].2d,$Ni.2s,$N0.s[1]
|
||
|
|
umlal @ACC[2].2d,$Ni.2s,$N0.s[2]
|
||
|
|
umlal @ACC[3].2d,$Ni.2s,$N0.s[3]
|
||
|
|
umlal @ACC[4].2d,$Ni.2s,$N1.s[0]
|
||
|
|
umlal @ACC[5].2d,$Ni.2s,$N1.s[1]
|
||
|
|
umlal @ACC[6].2d,$Ni.2s,$N1.s[2]
|
||
|
|
umlal @ACC[7].2d,$Ni.2s,$N1.s[3]
|
||
|
|
st1 {@ACC[0].2d},[$toutptr],#16
|
||
|
|
___
|
||
|
|
push(@ACC,shift(@ACC));
|
||
|
|
$code.=<<___;
|
||
|
|
umlal @ACC[0].2d,$Bi.2s,$A0.s[0]
|
||
|
|
ld1 {@ACC[7].2d},[$tinptr]
|
||
|
|
umlal @ACC[1].2d,$Bi.2s,$A0.s[1]
|
||
|
|
ld1 {$Ni.2s},[$bnptr],#8 // pull smashed m[8*i+$i]
|
||
|
|
umlal @ACC[2].2d,$Bi.2s,$A0.s[2]
|
||
|
|
b.eq .LInner_jump$i
|
||
|
|
add $tinptr,$tinptr,#16 // don't advance in last iteration
|
||
|
|
.LInner_jump$i:
|
||
|
|
umlal @ACC[3].2d,$Bi.2s,$A0.s[3]
|
||
|
|
umlal @ACC[4].2d,$Bi.2s,$A1.s[0]
|
||
|
|
umlal @ACC[5].2d,$Bi.2s,$A1.s[1]
|
||
|
|
umlal @ACC[6].2d,$Bi.2s,$A1.s[2]
|
||
|
|
umlal @ACC[7].2d,$Bi.2s,$A1.s[3]
|
||
|
|
___
|
||
|
|
}
|
||
|
|
$code.=<<___;
|
||
|
|
b.ne .LInner_after_rewind$i
|
||
|
|
sub $aptr,$aptr,$num,lsl#2 // rewind
|
||
|
|
.LInner_after_rewind$i:
|
||
|
|
umlal @ACC[0].2d,$Ni.2s,$N0.s[0]
|
||
|
|
ld1 {$Bi.2s},[sp] // pull smashed b[8*i+0]
|
||
|
|
umlal @ACC[1].2d,$Ni.2s,$N0.s[1]
|
||
|
|
ld1 {$A0.4s,$A1.4s},[$aptr],#32
|
||
|
|
umlal @ACC[2].2d,$Ni.2s,$N0.s[2]
|
||
|
|
add $bnptr,sp,#8 // rewind
|
||
|
|
umlal @ACC[3].2d,$Ni.2s,$N0.s[3]
|
||
|
|
umlal @ACC[4].2d,$Ni.2s,$N1.s[0]
|
||
|
|
umlal @ACC[5].2d,$Ni.2s,$N1.s[1]
|
||
|
|
umlal @ACC[6].2d,$Ni.2s,$N1.s[2]
|
||
|
|
st1 {@ACC[0].2d},[$toutptr],#16
|
||
|
|
umlal @ACC[7].2d,$Ni.2s,$N1.s[3]
|
||
|
|
|
||
|
|
bne .LNEON_8n_inner
|
||
|
|
___
|
||
|
|
push(@ACC,shift(@ACC));
|
||
|
|
$code.=<<___;
|
||
|
|
add $tinptr,sp,#128
|
||
|
|
st1 {@ACC[0].2d,@ACC[1].2d},[$toutptr],#32
|
||
|
|
eor $N0.16b,$N0.16b,$N0.16b // $N0
|
||
|
|
st1 {@ACC[2].2d,@ACC[3].2d},[$toutptr],#32
|
||
|
|
eor $N1.16b,$N1.16b,$N1.16b // $N1
|
||
|
|
st1 {@ACC[4].2d,@ACC[5].2d},[$toutptr],#32
|
||
|
|
st1 {@ACC[6].2d},[$toutptr]
|
||
|
|
|
||
|
|
subs $outer,$outer,#8
|
||
|
|
ld1 {@ACC[0].2d,@ACC[1].2d},[$tinptr],#32
|
||
|
|
ld1 {@ACC[2].2d,@ACC[3].2d},[$tinptr],#32
|
||
|
|
ld1 {@ACC[4].2d,@ACC[5].2d},[$tinptr],#32
|
||
|
|
ld1 {@ACC[6].2d,@ACC[7].2d},[$tinptr],#32
|
||
|
|
|
||
|
|
b.eq .LInner_8n_jump_2steps
|
||
|
|
sub $nptr,$nptr,$num,lsl#2 // rewind
|
||
|
|
b .LNEON_8n_outer
|
||
|
|
|
||
|
|
.LInner_8n_jump_2steps:
|
||
|
|
add $toutptr,sp,#128
|
||
|
|
st1 {$N0.2d,$N1.2d}, [sp],#32 // start wiping stack frame
|
||
|
|
mov $Temp.16b,@ACC[0].16b
|
||
|
|
ushr $temp.2d,@ACC[0].2d,#16
|
||
|
|
ext @ACC[0].16b,@ACC[0].16b,@ACC[0].16b,#8
|
||
|
|
st1 {$N0.2d,$N1.2d}, [sp],#32
|
||
|
|
add @ACC[0].2d,@ACC[0].2d,$temp.2d
|
||
|
|
st1 {$N0.2d,$N1.2d}, [sp],#32
|
||
|
|
ushr $temp.2d,@ACC[0].2d,#16
|
||
|
|
st1 {$N0.2d,$N1.2d}, [sp],#32
|
||
|
|
zip1 @ACC[0].4h,$Temp.4h,@ACC[0].4h
|
||
|
|
ins $temp.d[1],$zero.d[0]
|
||
|
|
|
||
|
|
mov $inner,$num
|
||
|
|
b .LNEON_tail_entry
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.LNEON_tail:
|
||
|
|
add @ACC[0].2d,@ACC[0].2d,$temp.2d
|
||
|
|
mov $Temp.16b,@ACC[0].16b
|
||
|
|
ushr $temp.2d,@ACC[0].2d,#16
|
||
|
|
ext @ACC[0].16b,@ACC[0].16b,@ACC[0].16b,#8
|
||
|
|
ld1 {@ACC[2].2d,@ACC[3].2d}, [$tinptr],#32
|
||
|
|
add @ACC[0].2d,@ACC[0].2d,$temp.2d
|
||
|
|
ld1 {@ACC[4].2d,@ACC[5].2d}, [$tinptr],#32
|
||
|
|
ushr $temp.2d,@ACC[0].2d,#16
|
||
|
|
ld1 {@ACC[6].2d,@ACC[7].2d}, [$tinptr],#32
|
||
|
|
zip1 @ACC[0].4h,$Temp.4h,@ACC[0].4h
|
||
|
|
ins $temp.d[1],$zero.d[0]
|
||
|
|
|
||
|
|
.LNEON_tail_entry:
|
||
|
|
___
|
||
|
|
for ($i=1; $i<8; $i++) {
|
||
|
|
$code.=<<___;
|
||
|
|
add @ACC[1].2d,@ACC[1].2d,$temp.2d
|
||
|
|
st1 {@ACC[0].s}[0], [$toutptr],#4
|
||
|
|
ushr $temp.2d,@ACC[1].2d,#16
|
||
|
|
mov $Temp.16b,@ACC[1].16b
|
||
|
|
ext @ACC[1].16b,@ACC[1].16b,@ACC[1].16b,#8
|
||
|
|
add @ACC[1].2d,@ACC[1].2d,$temp.2d
|
||
|
|
ushr $temp.2d,@ACC[1].2d,#16
|
||
|
|
zip1 @ACC[1].4h,$Temp.4h,@ACC[1].4h
|
||
|
|
ins $temp.d[1],$zero.d[0]
|
||
|
|
___
|
||
|
|
push(@ACC,shift(@ACC));
|
||
|
|
}
|
||
|
|
push(@ACC,shift(@ACC));
|
||
|
|
$code.=<<___;
|
||
|
|
ld1 {@ACC[0].2d,@ACC[1].2d}, [$tinptr],#32
|
||
|
|
subs $inner,$inner,#8
|
||
|
|
st1 {@ACC[7].s}[0], [$toutptr],#4
|
||
|
|
bne .LNEON_tail
|
||
|
|
|
||
|
|
st1 {$temp.s}[0], [$toutptr],#4 // top-most bit
|
||
|
|
sub $nptr,$nptr,$num,lsl#2 // rewind $nptr
|
||
|
|
subs $aptr,sp,#0 // clear carry flag
|
||
|
|
add $bptr,sp,$num,lsl#2
|
||
|
|
|
||
|
|
.LNEON_sub:
|
||
|
|
ldp w4,w5,[$aptr],#8
|
||
|
|
ldp w6,w7,[$aptr],#8
|
||
|
|
ldp w8,w9,[$nptr],#8
|
||
|
|
ldp w10,w11,[$nptr],#8
|
||
|
|
sbcs w8,w4,w8
|
||
|
|
sbcs w9,w5,w9
|
||
|
|
sbcs w10,w6,w10
|
||
|
|
sbcs w11,w7,w11
|
||
|
|
sub x17,$bptr,$aptr
|
||
|
|
stp w8,w9,[$rptr],#8
|
||
|
|
stp w10,w11,[$rptr],#8
|
||
|
|
cbnz x17,.LNEON_sub
|
||
|
|
|
||
|
|
ldr w10, [$aptr] // load top-most bit
|
||
|
|
mov x11,sp
|
||
|
|
eor v0.16b,v0.16b,v0.16b
|
||
|
|
sub x11,$bptr,x11 // this is num*4
|
||
|
|
eor v1.16b,v1.16b,v1.16b
|
||
|
|
mov $aptr,sp
|
||
|
|
sub $rptr,$rptr,x11 // rewind $rptr
|
||
|
|
mov $nptr,$bptr // second 3/4th of frame
|
||
|
|
sbcs w10,w10,wzr // result is carry flag
|
||
|
|
|
||
|
|
.LNEON_copy_n_zap:
|
||
|
|
ldp w4,w5,[$aptr],#8
|
||
|
|
ldp w6,w7,[$aptr],#8
|
||
|
|
ldp w8,w9,[$rptr],#8
|
||
|
|
ldp w10,w11,[$rptr]
|
||
|
|
sub $rptr,$rptr,#8
|
||
|
|
b.cs .LCopy_1
|
||
|
|
mov w8,w4
|
||
|
|
mov w9,w5
|
||
|
|
mov w10,w6
|
||
|
|
mov w11,w7
|
||
|
|
.LCopy_1:
|
||
|
|
st1 {v0.2d,v1.2d}, [$nptr],#32 // wipe
|
||
|
|
st1 {v0.2d,v1.2d}, [$nptr],#32 // wipe
|
||
|
|
ldp w4,w5,[$aptr],#8
|
||
|
|
ldp w6,w7,[$aptr],#8
|
||
|
|
stp w8,w9,[$rptr],#8
|
||
|
|
stp w10,w11,[$rptr],#8
|
||
|
|
sub $aptr,$aptr,#32
|
||
|
|
ldp w8,w9,[$rptr],#8
|
||
|
|
ldp w10,w11,[$rptr]
|
||
|
|
sub $rptr,$rptr,#8
|
||
|
|
b.cs .LCopy_2
|
||
|
|
mov w8, w4
|
||
|
|
mov w9, w5
|
||
|
|
mov w10, w6
|
||
|
|
mov w11, w7
|
||
|
|
.LCopy_2:
|
||
|
|
st1 {v0.2d,v1.2d}, [$aptr],#32 // wipe
|
||
|
|
st1 {v0.2d,v1.2d}, [$nptr],#32 // wipe
|
||
|
|
sub x17,$bptr,$aptr // preserves carry
|
||
|
|
stp w8,w9,[$rptr],#8
|
||
|
|
stp w10,w11,[$rptr],#8
|
||
|
|
cbnz x17,.LNEON_copy_n_zap
|
||
|
|
|
||
|
|
mov sp,x16
|
||
|
|
ldp d14,d15,[sp,#64]
|
||
|
|
ldp d12,d13,[sp,#48]
|
||
|
|
ldp d10,d11,[sp,#32]
|
||
|
|
ldp d8,d9,[sp,#16]
|
||
|
|
ldr x29,[sp],#80
|
||
|
|
AARCH64_VALIDATE_LINK_REGISTER
|
||
|
|
ret // bx lr
|
||
|
|
|
||
|
|
.size bn_mul8x_mont_neon,.-bn_mul8x_mont_neon
|
||
|
|
___
|
||
|
|
}
|
||
|
|
{
|
||
|
|
########################################################################
|
||
|
|
# Following is ARMv8 adaptation of sqrx8x_mont from x86_64-mont5 module.
|
||
|
|
|
||
|
|
my ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("x$_",(6..13));
|
||
|
|
my ($t0,$t1,$t2,$t3)=map("x$_",(14..17));
|
||
|
|
my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("x$_",(19..26));
|
||
|
|
my ($cnt,$carry,$topmost)=("x27","x28","x30");
|
||
|
|
my ($tp,$ap_end,$na0)=($bp,$np,$carry);
|
||
|
|
|
||
|
|
$code.=<<___;
|
||
|
|
.type __bn_sqr8x_mont,%function
|
||
|
|
.align 5
|
||
|
|
__bn_sqr8x_mont:
|
||
|
|
cmp $ap,$bp
|
||
|
|
b.ne __bn_mul4x_mont
|
||
|
|
.Lsqr8x_mont:
|
||
|
|
// Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_sqr8x_mont is jumped to
|
||
|
|
// only from bn_mul_mont which has already signed the return address.
|
||
|
|
stp x29,x30,[sp,#-128]!
|
||
|
|
add x29,sp,#0
|
||
|
|
stp x19,x20,[sp,#16]
|
||
|
|
stp x21,x22,[sp,#32]
|
||
|
|
stp x23,x24,[sp,#48]
|
||
|
|
stp x25,x26,[sp,#64]
|
||
|
|
stp x27,x28,[sp,#80]
|
||
|
|
stp $rp,$np,[sp,#96] // offload rp and np
|
||
|
|
|
||
|
|
ldp $a0,$a1,[$ap,#8*0]
|
||
|
|
ldp $a2,$a3,[$ap,#8*2]
|
||
|
|
ldp $a4,$a5,[$ap,#8*4]
|
||
|
|
ldp $a6,$a7,[$ap,#8*6]
|
||
|
|
|
||
|
|
sub $tp,sp,$num,lsl#4
|
||
|
|
lsl $num,$num,#3
|
||
|
|
ldr $n0,[$n0] // *n0
|
||
|
|
mov sp,$tp // alloca
|
||
|
|
sub $cnt,$num,#8*8
|
||
|
|
b .Lsqr8x_zero_start
|
||
|
|
|
||
|
|
.Lsqr8x_zero:
|
||
|
|
sub $cnt,$cnt,#8*8
|
||
|
|
stp xzr,xzr,[$tp,#8*0]
|
||
|
|
stp xzr,xzr,[$tp,#8*2]
|
||
|
|
stp xzr,xzr,[$tp,#8*4]
|
||
|
|
stp xzr,xzr,[$tp,#8*6]
|
||
|
|
.Lsqr8x_zero_start:
|
||
|
|
stp xzr,xzr,[$tp,#8*8]
|
||
|
|
stp xzr,xzr,[$tp,#8*10]
|
||
|
|
stp xzr,xzr,[$tp,#8*12]
|
||
|
|
stp xzr,xzr,[$tp,#8*14]
|
||
|
|
add $tp,$tp,#8*16
|
||
|
|
cbnz $cnt,.Lsqr8x_zero
|
||
|
|
|
||
|
|
add $ap_end,$ap,$num
|
||
|
|
add $ap,$ap,#8*8
|
||
|
|
mov $acc0,xzr
|
||
|
|
mov $acc1,xzr
|
||
|
|
mov $acc2,xzr
|
||
|
|
mov $acc3,xzr
|
||
|
|
mov $acc4,xzr
|
||
|
|
mov $acc5,xzr
|
||
|
|
mov $acc6,xzr
|
||
|
|
mov $acc7,xzr
|
||
|
|
mov $tp,sp
|
||
|
|
str $n0,[x29,#112] // offload n0
|
||
|
|
|
||
|
|
// Multiply everything but a[i]*a[i]
|
||
|
|
.align 4
|
||
|
|
.Lsqr8x_outer_loop:
|
||
|
|
// a[1]a[0] (i)
|
||
|
|
// a[2]a[0]
|
||
|
|
// a[3]a[0]
|
||
|
|
// a[4]a[0]
|
||
|
|
// a[5]a[0]
|
||
|
|
// a[6]a[0]
|
||
|
|
// a[7]a[0]
|
||
|
|
// a[2]a[1] (ii)
|
||
|
|
// a[3]a[1]
|
||
|
|
// a[4]a[1]
|
||
|
|
// a[5]a[1]
|
||
|
|
// a[6]a[1]
|
||
|
|
// a[7]a[1]
|
||
|
|
// a[3]a[2] (iii)
|
||
|
|
// a[4]a[2]
|
||
|
|
// a[5]a[2]
|
||
|
|
// a[6]a[2]
|
||
|
|
// a[7]a[2]
|
||
|
|
// a[4]a[3] (iv)
|
||
|
|
// a[5]a[3]
|
||
|
|
// a[6]a[3]
|
||
|
|
// a[7]a[3]
|
||
|
|
// a[5]a[4] (v)
|
||
|
|
// a[6]a[4]
|
||
|
|
// a[7]a[4]
|
||
|
|
// a[6]a[5] (vi)
|
||
|
|
// a[7]a[5]
|
||
|
|
// a[7]a[6] (vii)
|
||
|
|
|
||
|
|
mul $t0,$a1,$a0 // lo(a[1..7]*a[0]) (i)
|
||
|
|
mul $t1,$a2,$a0
|
||
|
|
mul $t2,$a3,$a0
|
||
|
|
mul $t3,$a4,$a0
|
||
|
|
adds $acc1,$acc1,$t0 // t[1]+lo(a[1]*a[0])
|
||
|
|
mul $t0,$a5,$a0
|
||
|
|
adcs $acc2,$acc2,$t1
|
||
|
|
mul $t1,$a6,$a0
|
||
|
|
adcs $acc3,$acc3,$t2
|
||
|
|
mul $t2,$a7,$a0
|
||
|
|
adcs $acc4,$acc4,$t3
|
||
|
|
umulh $t3,$a1,$a0 // hi(a[1..7]*a[0])
|
||
|
|
adcs $acc5,$acc5,$t0
|
||
|
|
umulh $t0,$a2,$a0
|
||
|
|
adcs $acc6,$acc6,$t1
|
||
|
|
umulh $t1,$a3,$a0
|
||
|
|
adcs $acc7,$acc7,$t2
|
||
|
|
umulh $t2,$a4,$a0
|
||
|
|
stp $acc0,$acc1,[$tp],#8*2 // t[0..1]
|
||
|
|
adc $acc0,xzr,xzr // t[8]
|
||
|
|
adds $acc2,$acc2,$t3 // t[2]+lo(a[1]*a[0])
|
||
|
|
umulh $t3,$a5,$a0
|
||
|
|
adcs $acc3,$acc3,$t0
|
||
|
|
umulh $t0,$a6,$a0
|
||
|
|
adcs $acc4,$acc4,$t1
|
||
|
|
umulh $t1,$a7,$a0
|
||
|
|
adcs $acc5,$acc5,$t2
|
||
|
|
mul $t2,$a2,$a1 // lo(a[2..7]*a[1]) (ii)
|
||
|
|
adcs $acc6,$acc6,$t3
|
||
|
|
mul $t3,$a3,$a1
|
||
|
|
adcs $acc7,$acc7,$t0
|
||
|
|
mul $t0,$a4,$a1
|
||
|
|
adc $acc0,$acc0,$t1
|
||
|
|
|
||
|
|
mul $t1,$a5,$a1
|
||
|
|
adds $acc3,$acc3,$t2
|
||
|
|
mul $t2,$a6,$a1
|
||
|
|
adcs $acc4,$acc4,$t3
|
||
|
|
mul $t3,$a7,$a1
|
||
|
|
adcs $acc5,$acc5,$t0
|
||
|
|
umulh $t0,$a2,$a1 // hi(a[2..7]*a[1])
|
||
|
|
adcs $acc6,$acc6,$t1
|
||
|
|
umulh $t1,$a3,$a1
|
||
|
|
adcs $acc7,$acc7,$t2
|
||
|
|
umulh $t2,$a4,$a1
|
||
|
|
adcs $acc0,$acc0,$t3
|
||
|
|
umulh $t3,$a5,$a1
|
||
|
|
stp $acc2,$acc3,[$tp],#8*2 // t[2..3]
|
||
|
|
adc $acc1,xzr,xzr // t[9]
|
||
|
|
adds $acc4,$acc4,$t0
|
||
|
|
umulh $t0,$a6,$a1
|
||
|
|
adcs $acc5,$acc5,$t1
|
||
|
|
umulh $t1,$a7,$a1
|
||
|
|
adcs $acc6,$acc6,$t2
|
||
|
|
mul $t2,$a3,$a2 // lo(a[3..7]*a[2]) (iii)
|
||
|
|
adcs $acc7,$acc7,$t3
|
||
|
|
mul $t3,$a4,$a2
|
||
|
|
adcs $acc0,$acc0,$t0
|
||
|
|
mul $t0,$a5,$a2
|
||
|
|
adc $acc1,$acc1,$t1
|
||
|
|
|
||
|
|
mul $t1,$a6,$a2
|
||
|
|
adds $acc5,$acc5,$t2
|
||
|
|
mul $t2,$a7,$a2
|
||
|
|
adcs $acc6,$acc6,$t3
|
||
|
|
umulh $t3,$a3,$a2 // hi(a[3..7]*a[2])
|
||
|
|
adcs $acc7,$acc7,$t0
|
||
|
|
umulh $t0,$a4,$a2
|
||
|
|
adcs $acc0,$acc0,$t1
|
||
|
|
umulh $t1,$a5,$a2
|
||
|
|
adcs $acc1,$acc1,$t2
|
||
|
|
umulh $t2,$a6,$a2
|
||
|
|
stp $acc4,$acc5,[$tp],#8*2 // t[4..5]
|
||
|
|
adc $acc2,xzr,xzr // t[10]
|
||
|
|
adds $acc6,$acc6,$t3
|
||
|
|
umulh $t3,$a7,$a2
|
||
|
|
adcs $acc7,$acc7,$t0
|
||
|
|
mul $t0,$a4,$a3 // lo(a[4..7]*a[3]) (iv)
|
||
|
|
adcs $acc0,$acc0,$t1
|
||
|
|
mul $t1,$a5,$a3
|
||
|
|
adcs $acc1,$acc1,$t2
|
||
|
|
mul $t2,$a6,$a3
|
||
|
|
adc $acc2,$acc2,$t3
|
||
|
|
|
||
|
|
mul $t3,$a7,$a3
|
||
|
|
adds $acc7,$acc7,$t0
|
||
|
|
umulh $t0,$a4,$a3 // hi(a[4..7]*a[3])
|
||
|
|
adcs $acc0,$acc0,$t1
|
||
|
|
umulh $t1,$a5,$a3
|
||
|
|
adcs $acc1,$acc1,$t2
|
||
|
|
umulh $t2,$a6,$a3
|
||
|
|
adcs $acc2,$acc2,$t3
|
||
|
|
umulh $t3,$a7,$a3
|
||
|
|
stp $acc6,$acc7,[$tp],#8*2 // t[6..7]
|
||
|
|
adc $acc3,xzr,xzr // t[11]
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
mul $t0,$a5,$a4 // lo(a[5..7]*a[4]) (v)
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
mul $t1,$a6,$a4
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
mul $t2,$a7,$a4
|
||
|
|
adc $acc3,$acc3,$t3
|
||
|
|
|
||
|
|
umulh $t3,$a5,$a4 // hi(a[5..7]*a[4])
|
||
|
|
adds $acc1,$acc1,$t0
|
||
|
|
umulh $t0,$a6,$a4
|
||
|
|
adcs $acc2,$acc2,$t1
|
||
|
|
umulh $t1,$a7,$a4
|
||
|
|
adcs $acc3,$acc3,$t2
|
||
|
|
mul $t2,$a6,$a5 // lo(a[6..7]*a[5]) (vi)
|
||
|
|
adc $acc4,xzr,xzr // t[12]
|
||
|
|
adds $acc2,$acc2,$t3
|
||
|
|
mul $t3,$a7,$a5
|
||
|
|
adcs $acc3,$acc3,$t0
|
||
|
|
umulh $t0,$a6,$a5 // hi(a[6..7]*a[5])
|
||
|
|
adc $acc4,$acc4,$t1
|
||
|
|
|
||
|
|
umulh $t1,$a7,$a5
|
||
|
|
adds $acc3,$acc3,$t2
|
||
|
|
mul $t2,$a7,$a6 // lo(a[7]*a[6]) (vii)
|
||
|
|
adcs $acc4,$acc4,$t3
|
||
|
|
umulh $t3,$a7,$a6 // hi(a[7]*a[6])
|
||
|
|
adc $acc5,xzr,xzr // t[13]
|
||
|
|
adds $acc4,$acc4,$t0
|
||
|
|
sub $cnt,$ap_end,$ap // done yet?
|
||
|
|
adc $acc5,$acc5,$t1
|
||
|
|
|
||
|
|
adds $acc5,$acc5,$t2
|
||
|
|
sub $t0,$ap_end,$num // rewinded ap
|
||
|
|
adc $acc6,xzr,xzr // t[14]
|
||
|
|
add $acc6,$acc6,$t3
|
||
|
|
|
||
|
|
cbz $cnt,.Lsqr8x_outer_break
|
||
|
|
|
||
|
|
mov $n0,$a0
|
||
|
|
ldp $a0,$a1,[$tp,#8*0]
|
||
|
|
ldp $a2,$a3,[$tp,#8*2]
|
||
|
|
ldp $a4,$a5,[$tp,#8*4]
|
||
|
|
ldp $a6,$a7,[$tp,#8*6]
|
||
|
|
adds $acc0,$acc0,$a0
|
||
|
|
adcs $acc1,$acc1,$a1
|
||
|
|
ldp $a0,$a1,[$ap,#8*0]
|
||
|
|
adcs $acc2,$acc2,$a2
|
||
|
|
adcs $acc3,$acc3,$a3
|
||
|
|
ldp $a2,$a3,[$ap,#8*2]
|
||
|
|
adcs $acc4,$acc4,$a4
|
||
|
|
adcs $acc5,$acc5,$a5
|
||
|
|
ldp $a4,$a5,[$ap,#8*4]
|
||
|
|
adcs $acc6,$acc6,$a6
|
||
|
|
mov $rp,$ap
|
||
|
|
adcs $acc7,xzr,$a7
|
||
|
|
ldp $a6,$a7,[$ap,#8*6]
|
||
|
|
add $ap,$ap,#8*8
|
||
|
|
//adc $carry,xzr,xzr // moved below
|
||
|
|
mov $cnt,#-8*8
|
||
|
|
|
||
|
|
// a[8]a[0]
|
||
|
|
// a[9]a[0]
|
||
|
|
// a[a]a[0]
|
||
|
|
// a[b]a[0]
|
||
|
|
// a[c]a[0]
|
||
|
|
// a[d]a[0]
|
||
|
|
// a[e]a[0]
|
||
|
|
// a[f]a[0]
|
||
|
|
// a[8]a[1]
|
||
|
|
// a[f]a[1]........................
|
||
|
|
// a[8]a[2]
|
||
|
|
// a[f]a[2]........................
|
||
|
|
// a[8]a[3]
|
||
|
|
// a[f]a[3]........................
|
||
|
|
// a[8]a[4]
|
||
|
|
// a[f]a[4]........................
|
||
|
|
// a[8]a[5]
|
||
|
|
// a[f]a[5]........................
|
||
|
|
// a[8]a[6]
|
||
|
|
// a[f]a[6]........................
|
||
|
|
// a[8]a[7]
|
||
|
|
// a[f]a[7]........................
|
||
|
|
.Lsqr8x_mul:
|
||
|
|
mul $t0,$a0,$n0
|
||
|
|
adc $carry,xzr,xzr // carry bit, modulo-scheduled
|
||
|
|
mul $t1,$a1,$n0
|
||
|
|
add $cnt,$cnt,#8
|
||
|
|
mul $t2,$a2,$n0
|
||
|
|
mul $t3,$a3,$n0
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
mul $t0,$a4,$n0
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
mul $t1,$a5,$n0
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
mul $t2,$a6,$n0
|
||
|
|
adcs $acc3,$acc3,$t3
|
||
|
|
mul $t3,$a7,$n0
|
||
|
|
adcs $acc4,$acc4,$t0
|
||
|
|
umulh $t0,$a0,$n0
|
||
|
|
adcs $acc5,$acc5,$t1
|
||
|
|
umulh $t1,$a1,$n0
|
||
|
|
adcs $acc6,$acc6,$t2
|
||
|
|
umulh $t2,$a2,$n0
|
||
|
|
adcs $acc7,$acc7,$t3
|
||
|
|
umulh $t3,$a3,$n0
|
||
|
|
adc $carry,$carry,xzr
|
||
|
|
str $acc0,[$tp],#8
|
||
|
|
adds $acc0,$acc1,$t0
|
||
|
|
umulh $t0,$a4,$n0
|
||
|
|
adcs $acc1,$acc2,$t1
|
||
|
|
umulh $t1,$a5,$n0
|
||
|
|
adcs $acc2,$acc3,$t2
|
||
|
|
umulh $t2,$a6,$n0
|
||
|
|
adcs $acc3,$acc4,$t3
|
||
|
|
umulh $t3,$a7,$n0
|
||
|
|
ldr $n0,[$rp,$cnt]
|
||
|
|
adcs $acc4,$acc5,$t0
|
||
|
|
adcs $acc5,$acc6,$t1
|
||
|
|
adcs $acc6,$acc7,$t2
|
||
|
|
adcs $acc7,$carry,$t3
|
||
|
|
//adc $carry,xzr,xzr // moved above
|
||
|
|
cbnz $cnt,.Lsqr8x_mul
|
||
|
|
// note that carry flag is guaranteed
|
||
|
|
// to be zero at this point
|
||
|
|
cmp $ap,$ap_end // done yet?
|
||
|
|
b.eq .Lsqr8x_break
|
||
|
|
|
||
|
|
ldp $a0,$a1,[$tp,#8*0]
|
||
|
|
ldp $a2,$a3,[$tp,#8*2]
|
||
|
|
ldp $a4,$a5,[$tp,#8*4]
|
||
|
|
ldp $a6,$a7,[$tp,#8*6]
|
||
|
|
adds $acc0,$acc0,$a0
|
||
|
|
ldur $n0,[$rp,#-8*8]
|
||
|
|
adcs $acc1,$acc1,$a1
|
||
|
|
ldp $a0,$a1,[$ap,#8*0]
|
||
|
|
adcs $acc2,$acc2,$a2
|
||
|
|
adcs $acc3,$acc3,$a3
|
||
|
|
ldp $a2,$a3,[$ap,#8*2]
|
||
|
|
adcs $acc4,$acc4,$a4
|
||
|
|
adcs $acc5,$acc5,$a5
|
||
|
|
ldp $a4,$a5,[$ap,#8*4]
|
||
|
|
adcs $acc6,$acc6,$a6
|
||
|
|
mov $cnt,#-8*8
|
||
|
|
adcs $acc7,$acc7,$a7
|
||
|
|
ldp $a6,$a7,[$ap,#8*6]
|
||
|
|
add $ap,$ap,#8*8
|
||
|
|
//adc $carry,xzr,xzr // moved above
|
||
|
|
b .Lsqr8x_mul
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.Lsqr8x_break:
|
||
|
|
ldp $a0,$a1,[$rp,#8*0]
|
||
|
|
add $ap,$rp,#8*8
|
||
|
|
ldp $a2,$a3,[$rp,#8*2]
|
||
|
|
sub $t0,$ap_end,$ap // is it last iteration?
|
||
|
|
ldp $a4,$a5,[$rp,#8*4]
|
||
|
|
sub $t1,$tp,$t0
|
||
|
|
ldp $a6,$a7,[$rp,#8*6]
|
||
|
|
cbz $t0,.Lsqr8x_outer_loop
|
||
|
|
|
||
|
|
stp $acc0,$acc1,[$tp,#8*0]
|
||
|
|
ldp $acc0,$acc1,[$t1,#8*0]
|
||
|
|
stp $acc2,$acc3,[$tp,#8*2]
|
||
|
|
ldp $acc2,$acc3,[$t1,#8*2]
|
||
|
|
stp $acc4,$acc5,[$tp,#8*4]
|
||
|
|
ldp $acc4,$acc5,[$t1,#8*4]
|
||
|
|
stp $acc6,$acc7,[$tp,#8*6]
|
||
|
|
mov $tp,$t1
|
||
|
|
ldp $acc6,$acc7,[$t1,#8*6]
|
||
|
|
b .Lsqr8x_outer_loop
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.Lsqr8x_outer_break:
|
||
|
|
// Now multiply above result by 2 and add a[n-1]*a[n-1]|...|a[0]*a[0]
|
||
|
|
ldp $a1,$a3,[$t0,#8*0] // recall that $t0 is &a[0]
|
||
|
|
ldp $t1,$t2,[sp,#8*1]
|
||
|
|
ldp $a5,$a7,[$t0,#8*2]
|
||
|
|
add $ap,$t0,#8*4
|
||
|
|
ldp $t3,$t0,[sp,#8*3]
|
||
|
|
|
||
|
|
stp $acc0,$acc1,[$tp,#8*0]
|
||
|
|
mul $acc0,$a1,$a1
|
||
|
|
stp $acc2,$acc3,[$tp,#8*2]
|
||
|
|
umulh $a1,$a1,$a1
|
||
|
|
stp $acc4,$acc5,[$tp,#8*4]
|
||
|
|
mul $a2,$a3,$a3
|
||
|
|
stp $acc6,$acc7,[$tp,#8*6]
|
||
|
|
mov $tp,sp
|
||
|
|
umulh $a3,$a3,$a3
|
||
|
|
adds $acc1,$a1,$t1,lsl#1
|
||
|
|
extr $t1,$t2,$t1,#63
|
||
|
|
sub $cnt,$num,#8*4
|
||
|
|
|
||
|
|
.Lsqr4x_shift_n_add:
|
||
|
|
adcs $acc2,$a2,$t1
|
||
|
|
extr $t2,$t3,$t2,#63
|
||
|
|
sub $cnt,$cnt,#8*4
|
||
|
|
adcs $acc3,$a3,$t2
|
||
|
|
ldp $t1,$t2,[$tp,#8*5]
|
||
|
|
mul $a4,$a5,$a5
|
||
|
|
ldp $a1,$a3,[$ap],#8*2
|
||
|
|
umulh $a5,$a5,$a5
|
||
|
|
mul $a6,$a7,$a7
|
||
|
|
umulh $a7,$a7,$a7
|
||
|
|
extr $t3,$t0,$t3,#63
|
||
|
|
stp $acc0,$acc1,[$tp,#8*0]
|
||
|
|
adcs $acc4,$a4,$t3
|
||
|
|
extr $t0,$t1,$t0,#63
|
||
|
|
stp $acc2,$acc3,[$tp,#8*2]
|
||
|
|
adcs $acc5,$a5,$t0
|
||
|
|
ldp $t3,$t0,[$tp,#8*7]
|
||
|
|
extr $t1,$t2,$t1,#63
|
||
|
|
adcs $acc6,$a6,$t1
|
||
|
|
extr $t2,$t3,$t2,#63
|
||
|
|
adcs $acc7,$a7,$t2
|
||
|
|
ldp $t1,$t2,[$tp,#8*9]
|
||
|
|
mul $a0,$a1,$a1
|
||
|
|
ldp $a5,$a7,[$ap],#8*2
|
||
|
|
umulh $a1,$a1,$a1
|
||
|
|
mul $a2,$a3,$a3
|
||
|
|
umulh $a3,$a3,$a3
|
||
|
|
stp $acc4,$acc5,[$tp,#8*4]
|
||
|
|
extr $t3,$t0,$t3,#63
|
||
|
|
stp $acc6,$acc7,[$tp,#8*6]
|
||
|
|
add $tp,$tp,#8*8
|
||
|
|
adcs $acc0,$a0,$t3
|
||
|
|
extr $t0,$t1,$t0,#63
|
||
|
|
adcs $acc1,$a1,$t0
|
||
|
|
ldp $t3,$t0,[$tp,#8*3]
|
||
|
|
extr $t1,$t2,$t1,#63
|
||
|
|
cbnz $cnt,.Lsqr4x_shift_n_add
|
||
|
|
___
|
||
|
|
my ($np,$np_end)=($ap,$ap_end);
|
||
|
|
$code.=<<___;
|
||
|
|
ldp $np,$n0,[x29,#104] // pull np and n0
|
||
|
|
|
||
|
|
adcs $acc2,$a2,$t1
|
||
|
|
extr $t2,$t3,$t2,#63
|
||
|
|
adcs $acc3,$a3,$t2
|
||
|
|
ldp $t1,$t2,[$tp,#8*5]
|
||
|
|
mul $a4,$a5,$a5
|
||
|
|
umulh $a5,$a5,$a5
|
||
|
|
stp $acc0,$acc1,[$tp,#8*0]
|
||
|
|
mul $a6,$a7,$a7
|
||
|
|
umulh $a7,$a7,$a7
|
||
|
|
stp $acc2,$acc3,[$tp,#8*2]
|
||
|
|
extr $t3,$t0,$t3,#63
|
||
|
|
adcs $acc4,$a4,$t3
|
||
|
|
extr $t0,$t1,$t0,#63
|
||
|
|
ldp $acc0,$acc1,[sp,#8*0]
|
||
|
|
adcs $acc5,$a5,$t0
|
||
|
|
extr $t1,$t2,$t1,#63
|
||
|
|
ldp $a0,$a1,[$np,#8*0]
|
||
|
|
adcs $acc6,$a6,$t1
|
||
|
|
extr $t2,xzr,$t2,#63
|
||
|
|
ldp $a2,$a3,[$np,#8*2]
|
||
|
|
adc $acc7,$a7,$t2
|
||
|
|
ldp $a4,$a5,[$np,#8*4]
|
||
|
|
|
||
|
|
// Reduce by 512 bits per iteration
|
||
|
|
mul $na0,$n0,$acc0 // t[0]*n0
|
||
|
|
ldp $a6,$a7,[$np,#8*6]
|
||
|
|
add $np_end,$np,$num
|
||
|
|
ldp $acc2,$acc3,[sp,#8*2]
|
||
|
|
stp $acc4,$acc5,[$tp,#8*4]
|
||
|
|
ldp $acc4,$acc5,[sp,#8*4]
|
||
|
|
stp $acc6,$acc7,[$tp,#8*6]
|
||
|
|
ldp $acc6,$acc7,[sp,#8*6]
|
||
|
|
add $np,$np,#8*8
|
||
|
|
mov $topmost,xzr // initial top-most carry
|
||
|
|
mov $tp,sp
|
||
|
|
mov $cnt,#8
|
||
|
|
|
||
|
|
.Lsqr8x_reduction:
|
||
|
|
// (*) mul $t0,$a0,$na0 // lo(n[0-7])*lo(t[0]*n0)
|
||
|
|
mul $t1,$a1,$na0
|
||
|
|
sub $cnt,$cnt,#1
|
||
|
|
mul $t2,$a2,$na0
|
||
|
|
str $na0,[$tp],#8 // put aside t[0]*n0 for tail processing
|
||
|
|
mul $t3,$a3,$na0
|
||
|
|
// (*) adds xzr,$acc0,$t0
|
||
|
|
subs xzr,$acc0,#1 // (*)
|
||
|
|
mul $t0,$a4,$na0
|
||
|
|
adcs $acc0,$acc1,$t1
|
||
|
|
mul $t1,$a5,$na0
|
||
|
|
adcs $acc1,$acc2,$t2
|
||
|
|
mul $t2,$a6,$na0
|
||
|
|
adcs $acc2,$acc3,$t3
|
||
|
|
mul $t3,$a7,$na0
|
||
|
|
adcs $acc3,$acc4,$t0
|
||
|
|
umulh $t0,$a0,$na0 // hi(n[0-7])*lo(t[0]*n0)
|
||
|
|
adcs $acc4,$acc5,$t1
|
||
|
|
umulh $t1,$a1,$na0
|
||
|
|
adcs $acc5,$acc6,$t2
|
||
|
|
umulh $t2,$a2,$na0
|
||
|
|
adcs $acc6,$acc7,$t3
|
||
|
|
umulh $t3,$a3,$na0
|
||
|
|
adc $acc7,xzr,xzr
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
umulh $t0,$a4,$na0
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
umulh $t1,$a5,$na0
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
umulh $t2,$a6,$na0
|
||
|
|
adcs $acc3,$acc3,$t3
|
||
|
|
umulh $t3,$a7,$na0
|
||
|
|
mul $na0,$n0,$acc0 // next t[0]*n0
|
||
|
|
adcs $acc4,$acc4,$t0
|
||
|
|
adcs $acc5,$acc5,$t1
|
||
|
|
adcs $acc6,$acc6,$t2
|
||
|
|
adc $acc7,$acc7,$t3
|
||
|
|
cbnz $cnt,.Lsqr8x_reduction
|
||
|
|
|
||
|
|
ldp $t0,$t1,[$tp,#8*0]
|
||
|
|
ldp $t2,$t3,[$tp,#8*2]
|
||
|
|
mov $rp,$tp
|
||
|
|
sub $cnt,$np_end,$np // done yet?
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
ldp $t0,$t1,[$tp,#8*4]
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
adcs $acc3,$acc3,$t3
|
||
|
|
ldp $t2,$t3,[$tp,#8*6]
|
||
|
|
adcs $acc4,$acc4,$t0
|
||
|
|
adcs $acc5,$acc5,$t1
|
||
|
|
adcs $acc6,$acc6,$t2
|
||
|
|
adcs $acc7,$acc7,$t3
|
||
|
|
//adc $carry,xzr,xzr // moved below
|
||
|
|
cbz $cnt,.Lsqr8x8_post_condition
|
||
|
|
|
||
|
|
ldur $n0,[$tp,#-8*8]
|
||
|
|
ldp $a0,$a1,[$np,#8*0]
|
||
|
|
ldp $a2,$a3,[$np,#8*2]
|
||
|
|
ldp $a4,$a5,[$np,#8*4]
|
||
|
|
mov $cnt,#-8*8
|
||
|
|
ldp $a6,$a7,[$np,#8*6]
|
||
|
|
add $np,$np,#8*8
|
||
|
|
|
||
|
|
.Lsqr8x_tail:
|
||
|
|
mul $t0,$a0,$n0
|
||
|
|
adc $carry,xzr,xzr // carry bit, modulo-scheduled
|
||
|
|
mul $t1,$a1,$n0
|
||
|
|
add $cnt,$cnt,#8
|
||
|
|
mul $t2,$a2,$n0
|
||
|
|
mul $t3,$a3,$n0
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
mul $t0,$a4,$n0
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
mul $t1,$a5,$n0
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
mul $t2,$a6,$n0
|
||
|
|
adcs $acc3,$acc3,$t3
|
||
|
|
mul $t3,$a7,$n0
|
||
|
|
adcs $acc4,$acc4,$t0
|
||
|
|
umulh $t0,$a0,$n0
|
||
|
|
adcs $acc5,$acc5,$t1
|
||
|
|
umulh $t1,$a1,$n0
|
||
|
|
adcs $acc6,$acc6,$t2
|
||
|
|
umulh $t2,$a2,$n0
|
||
|
|
adcs $acc7,$acc7,$t3
|
||
|
|
umulh $t3,$a3,$n0
|
||
|
|
adc $carry,$carry,xzr
|
||
|
|
str $acc0,[$tp],#8
|
||
|
|
adds $acc0,$acc1,$t0
|
||
|
|
umulh $t0,$a4,$n0
|
||
|
|
adcs $acc1,$acc2,$t1
|
||
|
|
umulh $t1,$a5,$n0
|
||
|
|
adcs $acc2,$acc3,$t2
|
||
|
|
umulh $t2,$a6,$n0
|
||
|
|
adcs $acc3,$acc4,$t3
|
||
|
|
umulh $t3,$a7,$n0
|
||
|
|
ldr $n0,[$rp,$cnt]
|
||
|
|
adcs $acc4,$acc5,$t0
|
||
|
|
adcs $acc5,$acc6,$t1
|
||
|
|
adcs $acc6,$acc7,$t2
|
||
|
|
adcs $acc7,$carry,$t3
|
||
|
|
//adc $carry,xzr,xzr // moved above
|
||
|
|
cbnz $cnt,.Lsqr8x_tail
|
||
|
|
// note that carry flag is guaranteed
|
||
|
|
// to be zero at this point
|
||
|
|
ldp $a0,$a1,[$tp,#8*0]
|
||
|
|
sub $cnt,$np_end,$np // done yet?
|
||
|
|
sub $t2,$np_end,$num // rewinded np
|
||
|
|
ldp $a2,$a3,[$tp,#8*2]
|
||
|
|
ldp $a4,$a5,[$tp,#8*4]
|
||
|
|
ldp $a6,$a7,[$tp,#8*6]
|
||
|
|
cbz $cnt,.Lsqr8x_tail_break
|
||
|
|
|
||
|
|
ldur $n0,[$rp,#-8*8]
|
||
|
|
adds $acc0,$acc0,$a0
|
||
|
|
adcs $acc1,$acc1,$a1
|
||
|
|
ldp $a0,$a1,[$np,#8*0]
|
||
|
|
adcs $acc2,$acc2,$a2
|
||
|
|
adcs $acc3,$acc3,$a3
|
||
|
|
ldp $a2,$a3,[$np,#8*2]
|
||
|
|
adcs $acc4,$acc4,$a4
|
||
|
|
adcs $acc5,$acc5,$a5
|
||
|
|
ldp $a4,$a5,[$np,#8*4]
|
||
|
|
adcs $acc6,$acc6,$a6
|
||
|
|
mov $cnt,#-8*8
|
||
|
|
adcs $acc7,$acc7,$a7
|
||
|
|
ldp $a6,$a7,[$np,#8*6]
|
||
|
|
add $np,$np,#8*8
|
||
|
|
//adc $carry,xzr,xzr // moved above
|
||
|
|
b .Lsqr8x_tail
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.Lsqr8x_tail_break:
|
||
|
|
ldr $n0,[x29,#112] // pull n0
|
||
|
|
add $cnt,$tp,#8*8 // end of current t[num] window
|
||
|
|
|
||
|
|
subs xzr,$topmost,#1 // "move" top-most carry to carry bit
|
||
|
|
adcs $t0,$acc0,$a0
|
||
|
|
adcs $t1,$acc1,$a1
|
||
|
|
ldp $acc0,$acc1,[$rp,#8*0]
|
||
|
|
adcs $acc2,$acc2,$a2
|
||
|
|
ldp $a0,$a1,[$t2,#8*0] // recall that $t2 is &n[0]
|
||
|
|
adcs $acc3,$acc3,$a3
|
||
|
|
ldp $a2,$a3,[$t2,#8*2]
|
||
|
|
adcs $acc4,$acc4,$a4
|
||
|
|
adcs $acc5,$acc5,$a5
|
||
|
|
ldp $a4,$a5,[$t2,#8*4]
|
||
|
|
adcs $acc6,$acc6,$a6
|
||
|
|
adcs $acc7,$acc7,$a7
|
||
|
|
ldp $a6,$a7,[$t2,#8*6]
|
||
|
|
add $np,$t2,#8*8
|
||
|
|
adc $topmost,xzr,xzr // top-most carry
|
||
|
|
mul $na0,$n0,$acc0
|
||
|
|
stp $t0,$t1,[$tp,#8*0]
|
||
|
|
stp $acc2,$acc3,[$tp,#8*2]
|
||
|
|
ldp $acc2,$acc3,[$rp,#8*2]
|
||
|
|
stp $acc4,$acc5,[$tp,#8*4]
|
||
|
|
ldp $acc4,$acc5,[$rp,#8*4]
|
||
|
|
cmp $cnt,x29 // did we hit the bottom?
|
||
|
|
stp $acc6,$acc7,[$tp,#8*6]
|
||
|
|
mov $tp,$rp // slide the window
|
||
|
|
ldp $acc6,$acc7,[$rp,#8*6]
|
||
|
|
mov $cnt,#8
|
||
|
|
b.ne .Lsqr8x_reduction
|
||
|
|
|
||
|
|
// Final step. We see if result is larger than modulus, and
|
||
|
|
// if it is, subtract the modulus. But comparison implies
|
||
|
|
// subtraction. So we subtract modulus, see if it borrowed,
|
||
|
|
// and conditionally copy original value.
|
||
|
|
ldr $rp,[x29,#96] // pull rp
|
||
|
|
add $tp,$tp,#8*8
|
||
|
|
subs $t0,$acc0,$a0
|
||
|
|
sbcs $t1,$acc1,$a1
|
||
|
|
sub $cnt,$num,#8*8
|
||
|
|
mov $ap_end,$rp // $rp copy
|
||
|
|
|
||
|
|
.Lsqr8x_sub:
|
||
|
|
sbcs $t2,$acc2,$a2
|
||
|
|
ldp $a0,$a1,[$np,#8*0]
|
||
|
|
sbcs $t3,$acc3,$a3
|
||
|
|
stp $t0,$t1,[$rp,#8*0]
|
||
|
|
sbcs $t0,$acc4,$a4
|
||
|
|
ldp $a2,$a3,[$np,#8*2]
|
||
|
|
sbcs $t1,$acc5,$a5
|
||
|
|
stp $t2,$t3,[$rp,#8*2]
|
||
|
|
sbcs $t2,$acc6,$a6
|
||
|
|
ldp $a4,$a5,[$np,#8*4]
|
||
|
|
sbcs $t3,$acc7,$a7
|
||
|
|
ldp $a6,$a7,[$np,#8*6]
|
||
|
|
add $np,$np,#8*8
|
||
|
|
ldp $acc0,$acc1,[$tp,#8*0]
|
||
|
|
sub $cnt,$cnt,#8*8
|
||
|
|
ldp $acc2,$acc3,[$tp,#8*2]
|
||
|
|
ldp $acc4,$acc5,[$tp,#8*4]
|
||
|
|
ldp $acc6,$acc7,[$tp,#8*6]
|
||
|
|
add $tp,$tp,#8*8
|
||
|
|
stp $t0,$t1,[$rp,#8*4]
|
||
|
|
sbcs $t0,$acc0,$a0
|
||
|
|
stp $t2,$t3,[$rp,#8*6]
|
||
|
|
add $rp,$rp,#8*8
|
||
|
|
sbcs $t1,$acc1,$a1
|
||
|
|
cbnz $cnt,.Lsqr8x_sub
|
||
|
|
|
||
|
|
sbcs $t2,$acc2,$a2
|
||
|
|
mov $tp,sp
|
||
|
|
add $ap,sp,$num
|
||
|
|
ldp $a0,$a1,[$ap_end,#8*0]
|
||
|
|
sbcs $t3,$acc3,$a3
|
||
|
|
stp $t0,$t1,[$rp,#8*0]
|
||
|
|
sbcs $t0,$acc4,$a4
|
||
|
|
ldp $a2,$a3,[$ap_end,#8*2]
|
||
|
|
sbcs $t1,$acc5,$a5
|
||
|
|
stp $t2,$t3,[$rp,#8*2]
|
||
|
|
sbcs $t2,$acc6,$a6
|
||
|
|
ldp $acc0,$acc1,[$ap,#8*0]
|
||
|
|
sbcs $t3,$acc7,$a7
|
||
|
|
ldp $acc2,$acc3,[$ap,#8*2]
|
||
|
|
sbcs xzr,$topmost,xzr // did it borrow?
|
||
|
|
ldr x30,[x29,#8] // pull return address
|
||
|
|
stp $t0,$t1,[$rp,#8*4]
|
||
|
|
stp $t2,$t3,[$rp,#8*6]
|
||
|
|
|
||
|
|
sub $cnt,$num,#8*4
|
||
|
|
.Lsqr4x_cond_copy:
|
||
|
|
sub $cnt,$cnt,#8*4
|
||
|
|
csel $t0,$acc0,$a0,lo
|
||
|
|
stp xzr,xzr,[$tp,#8*0]
|
||
|
|
csel $t1,$acc1,$a1,lo
|
||
|
|
ldp $a0,$a1,[$ap_end,#8*4]
|
||
|
|
ldp $acc0,$acc1,[$ap,#8*4]
|
||
|
|
csel $t2,$acc2,$a2,lo
|
||
|
|
stp xzr,xzr,[$tp,#8*2]
|
||
|
|
add $tp,$tp,#8*4
|
||
|
|
csel $t3,$acc3,$a3,lo
|
||
|
|
ldp $a2,$a3,[$ap_end,#8*6]
|
||
|
|
ldp $acc2,$acc3,[$ap,#8*6]
|
||
|
|
add $ap,$ap,#8*4
|
||
|
|
stp $t0,$t1,[$ap_end,#8*0]
|
||
|
|
stp $t2,$t3,[$ap_end,#8*2]
|
||
|
|
add $ap_end,$ap_end,#8*4
|
||
|
|
stp xzr,xzr,[$ap,#8*0]
|
||
|
|
stp xzr,xzr,[$ap,#8*2]
|
||
|
|
cbnz $cnt,.Lsqr4x_cond_copy
|
||
|
|
|
||
|
|
csel $t0,$acc0,$a0,lo
|
||
|
|
stp xzr,xzr,[$tp,#8*0]
|
||
|
|
csel $t1,$acc1,$a1,lo
|
||
|
|
stp xzr,xzr,[$tp,#8*2]
|
||
|
|
csel $t2,$acc2,$a2,lo
|
||
|
|
csel $t3,$acc3,$a3,lo
|
||
|
|
stp $t0,$t1,[$ap_end,#8*0]
|
||
|
|
stp $t2,$t3,[$ap_end,#8*2]
|
||
|
|
|
||
|
|
b .Lsqr8x_done
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.Lsqr8x8_post_condition:
|
||
|
|
adc $carry,xzr,xzr
|
||
|
|
ldr x30,[x29,#8] // pull return address
|
||
|
|
// $acc0-7,$carry hold result, $a0-7 hold modulus
|
||
|
|
subs $a0,$acc0,$a0
|
||
|
|
ldr $ap,[x29,#96] // pull rp
|
||
|
|
sbcs $a1,$acc1,$a1
|
||
|
|
stp xzr,xzr,[sp,#8*0]
|
||
|
|
sbcs $a2,$acc2,$a2
|
||
|
|
stp xzr,xzr,[sp,#8*2]
|
||
|
|
sbcs $a3,$acc3,$a3
|
||
|
|
stp xzr,xzr,[sp,#8*4]
|
||
|
|
sbcs $a4,$acc4,$a4
|
||
|
|
stp xzr,xzr,[sp,#8*6]
|
||
|
|
sbcs $a5,$acc5,$a5
|
||
|
|
stp xzr,xzr,[sp,#8*8]
|
||
|
|
sbcs $a6,$acc6,$a6
|
||
|
|
stp xzr,xzr,[sp,#8*10]
|
||
|
|
sbcs $a7,$acc7,$a7
|
||
|
|
stp xzr,xzr,[sp,#8*12]
|
||
|
|
sbcs $carry,$carry,xzr // did it borrow?
|
||
|
|
stp xzr,xzr,[sp,#8*14]
|
||
|
|
|
||
|
|
// $a0-7 hold result-modulus
|
||
|
|
csel $a0,$acc0,$a0,lo
|
||
|
|
csel $a1,$acc1,$a1,lo
|
||
|
|
csel $a2,$acc2,$a2,lo
|
||
|
|
csel $a3,$acc3,$a3,lo
|
||
|
|
stp $a0,$a1,[$ap,#8*0]
|
||
|
|
csel $a4,$acc4,$a4,lo
|
||
|
|
csel $a5,$acc5,$a5,lo
|
||
|
|
stp $a2,$a3,[$ap,#8*2]
|
||
|
|
csel $a6,$acc6,$a6,lo
|
||
|
|
csel $a7,$acc7,$a7,lo
|
||
|
|
stp $a4,$a5,[$ap,#8*4]
|
||
|
|
stp $a6,$a7,[$ap,#8*6]
|
||
|
|
|
||
|
|
.Lsqr8x_done:
|
||
|
|
ldp x19,x20,[x29,#16]
|
||
|
|
mov sp,x29
|
||
|
|
ldp x21,x22,[x29,#32]
|
||
|
|
mov x0,#1
|
||
|
|
ldp x23,x24,[x29,#48]
|
||
|
|
ldp x25,x26,[x29,#64]
|
||
|
|
ldp x27,x28,[x29,#80]
|
||
|
|
ldr x29,[sp],#128
|
||
|
|
// x30 is loaded earlier
|
||
|
|
AARCH64_VALIDATE_LINK_REGISTER
|
||
|
|
ret
|
||
|
|
.size __bn_sqr8x_mont,.-__bn_sqr8x_mont
|
||
|
|
___
|
||
|
|
}
|
||
|
|
|
||
|
|
{
|
||
|
|
########################################################################
|
||
|
|
# Even though this might look as ARMv8 adaptation of mulx4x_mont from
|
||
|
|
# x86_64-mont5 module, it's different in sense that it performs
|
||
|
|
# reduction 256 bits at a time.
|
||
|
|
|
||
|
|
my ($a0,$a1,$a2,$a3,
|
||
|
|
$t0,$t1,$t2,$t3,
|
||
|
|
$m0,$m1,$m2,$m3,
|
||
|
|
$acc0,$acc1,$acc2,$acc3,$acc4,
|
||
|
|
$bi,$mi,$tp,$ap_end,$cnt) = map("x$_",(6..17,19..28));
|
||
|
|
my $bp_end=$rp;
|
||
|
|
my ($carry,$topmost) = ($rp,"x30");
|
||
|
|
|
||
|
|
$code.=<<___;
|
||
|
|
.type __bn_mul4x_mont,%function
|
||
|
|
.align 5
|
||
|
|
__bn_mul4x_mont:
|
||
|
|
// Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_mul4x_mont is jumped to
|
||
|
|
// only from bn_mul_mont (or __bn_sqr8x_mont from bn_mul_mont) which has already signed the return address.
|
||
|
|
stp x29,x30,[sp,#-128]!
|
||
|
|
add x29,sp,#0
|
||
|
|
stp x19,x20,[sp,#16]
|
||
|
|
stp x21,x22,[sp,#32]
|
||
|
|
stp x23,x24,[sp,#48]
|
||
|
|
stp x25,x26,[sp,#64]
|
||
|
|
stp x27,x28,[sp,#80]
|
||
|
|
|
||
|
|
sub $tp,sp,$num,lsl#3
|
||
|
|
lsl $num,$num,#3
|
||
|
|
ldr $n0,[$n0] // *n0
|
||
|
|
sub sp,$tp,#8*4 // alloca
|
||
|
|
|
||
|
|
add $t0,$bp,$num
|
||
|
|
add $ap_end,$ap,$num
|
||
|
|
stp $rp,$t0,[x29,#96] // offload rp and &b[num]
|
||
|
|
|
||
|
|
ldr $bi,[$bp,#8*0] // b[0]
|
||
|
|
ldp $a0,$a1,[$ap,#8*0] // a[0..3]
|
||
|
|
ldp $a2,$a3,[$ap,#8*2]
|
||
|
|
add $ap,$ap,#8*4
|
||
|
|
mov $acc0,xzr
|
||
|
|
mov $acc1,xzr
|
||
|
|
mov $acc2,xzr
|
||
|
|
mov $acc3,xzr
|
||
|
|
ldp $m0,$m1,[$np,#8*0] // n[0..3]
|
||
|
|
ldp $m2,$m3,[$np,#8*2]
|
||
|
|
adds $np,$np,#8*4 // clear carry bit
|
||
|
|
mov $carry,xzr
|
||
|
|
mov $cnt,#0
|
||
|
|
mov $tp,sp
|
||
|
|
|
||
|
|
.Loop_mul4x_1st_reduction:
|
||
|
|
mul $t0,$a0,$bi // lo(a[0..3]*b[0])
|
||
|
|
adc $carry,$carry,xzr // modulo-scheduled
|
||
|
|
mul $t1,$a1,$bi
|
||
|
|
add $cnt,$cnt,#8
|
||
|
|
mul $t2,$a2,$bi
|
||
|
|
and $cnt,$cnt,#31
|
||
|
|
mul $t3,$a3,$bi
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
umulh $t0,$a0,$bi // hi(a[0..3]*b[0])
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
mul $mi,$acc0,$n0 // t[0]*n0
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
umulh $t1,$a1,$bi
|
||
|
|
adcs $acc3,$acc3,$t3
|
||
|
|
umulh $t2,$a2,$bi
|
||
|
|
adc $acc4,xzr,xzr
|
||
|
|
umulh $t3,$a3,$bi
|
||
|
|
ldr $bi,[$bp,$cnt] // next b[i] (or b[0])
|
||
|
|
adds $acc1,$acc1,$t0
|
||
|
|
// (*) mul $t0,$m0,$mi // lo(n[0..3]*t[0]*n0)
|
||
|
|
str $mi,[$tp],#8 // put aside t[0]*n0 for tail processing
|
||
|
|
adcs $acc2,$acc2,$t1
|
||
|
|
mul $t1,$m1,$mi
|
||
|
|
adcs $acc3,$acc3,$t2
|
||
|
|
mul $t2,$m2,$mi
|
||
|
|
adc $acc4,$acc4,$t3 // can't overflow
|
||
|
|
mul $t3,$m3,$mi
|
||
|
|
// (*) adds xzr,$acc0,$t0
|
||
|
|
subs xzr,$acc0,#1 // (*)
|
||
|
|
umulh $t0,$m0,$mi // hi(n[0..3]*t[0]*n0)
|
||
|
|
adcs $acc0,$acc1,$t1
|
||
|
|
umulh $t1,$m1,$mi
|
||
|
|
adcs $acc1,$acc2,$t2
|
||
|
|
umulh $t2,$m2,$mi
|
||
|
|
adcs $acc2,$acc3,$t3
|
||
|
|
umulh $t3,$m3,$mi
|
||
|
|
adcs $acc3,$acc4,$carry
|
||
|
|
adc $carry,xzr,xzr
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
sub $t0,$ap_end,$ap
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
adcs $acc3,$acc3,$t3
|
||
|
|
//adc $carry,$carry,xzr
|
||
|
|
cbnz $cnt,.Loop_mul4x_1st_reduction
|
||
|
|
|
||
|
|
cbz $t0,.Lmul4x4_post_condition
|
||
|
|
|
||
|
|
ldp $a0,$a1,[$ap,#8*0] // a[4..7]
|
||
|
|
ldp $a2,$a3,[$ap,#8*2]
|
||
|
|
add $ap,$ap,#8*4
|
||
|
|
ldr $mi,[sp] // a[0]*n0
|
||
|
|
ldp $m0,$m1,[$np,#8*0] // n[4..7]
|
||
|
|
ldp $m2,$m3,[$np,#8*2]
|
||
|
|
add $np,$np,#8*4
|
||
|
|
|
||
|
|
.Loop_mul4x_1st_tail:
|
||
|
|
mul $t0,$a0,$bi // lo(a[4..7]*b[i])
|
||
|
|
adc $carry,$carry,xzr // modulo-scheduled
|
||
|
|
mul $t1,$a1,$bi
|
||
|
|
add $cnt,$cnt,#8
|
||
|
|
mul $t2,$a2,$bi
|
||
|
|
and $cnt,$cnt,#31
|
||
|
|
mul $t3,$a3,$bi
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
umulh $t0,$a0,$bi // hi(a[4..7]*b[i])
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
umulh $t1,$a1,$bi
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
umulh $t2,$a2,$bi
|
||
|
|
adcs $acc3,$acc3,$t3
|
||
|
|
umulh $t3,$a3,$bi
|
||
|
|
adc $acc4,xzr,xzr
|
||
|
|
ldr $bi,[$bp,$cnt] // next b[i] (or b[0])
|
||
|
|
adds $acc1,$acc1,$t0
|
||
|
|
mul $t0,$m0,$mi // lo(n[4..7]*a[0]*n0)
|
||
|
|
adcs $acc2,$acc2,$t1
|
||
|
|
mul $t1,$m1,$mi
|
||
|
|
adcs $acc3,$acc3,$t2
|
||
|
|
mul $t2,$m2,$mi
|
||
|
|
adc $acc4,$acc4,$t3 // can't overflow
|
||
|
|
mul $t3,$m3,$mi
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
umulh $t0,$m0,$mi // hi(n[4..7]*a[0]*n0)
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
umulh $t1,$m1,$mi
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
umulh $t2,$m2,$mi
|
||
|
|
adcs $acc3,$acc3,$t3
|
||
|
|
adcs $acc4,$acc4,$carry
|
||
|
|
umulh $t3,$m3,$mi
|
||
|
|
adc $carry,xzr,xzr
|
||
|
|
ldr $mi,[sp,$cnt] // next t[0]*n0
|
||
|
|
str $acc0,[$tp],#8 // result!!!
|
||
|
|
adds $acc0,$acc1,$t0
|
||
|
|
sub $t0,$ap_end,$ap // done yet?
|
||
|
|
adcs $acc1,$acc2,$t1
|
||
|
|
adcs $acc2,$acc3,$t2
|
||
|
|
adcs $acc3,$acc4,$t3
|
||
|
|
//adc $carry,$carry,xzr
|
||
|
|
cbnz $cnt,.Loop_mul4x_1st_tail
|
||
|
|
|
||
|
|
sub $t1,$ap_end,$num // rewinded $ap
|
||
|
|
cbz $t0,.Lmul4x_proceed
|
||
|
|
|
||
|
|
ldp $a0,$a1,[$ap,#8*0]
|
||
|
|
ldp $a2,$a3,[$ap,#8*2]
|
||
|
|
add $ap,$ap,#8*4
|
||
|
|
ldp $m0,$m1,[$np,#8*0]
|
||
|
|
ldp $m2,$m3,[$np,#8*2]
|
||
|
|
add $np,$np,#8*4
|
||
|
|
b .Loop_mul4x_1st_tail
|
||
|
|
|
||
|
|
.align 5
|
||
|
|
.Lmul4x_proceed:
|
||
|
|
ldr $bi,[$bp,#8*4]! // *++b
|
||
|
|
adc $topmost,$carry,xzr
|
||
|
|
ldp $a0,$a1,[$t1,#8*0] // a[0..3]
|
||
|
|
sub $np,$np,$num // rewind np
|
||
|
|
ldp $a2,$a3,[$t1,#8*2]
|
||
|
|
add $ap,$t1,#8*4
|
||
|
|
|
||
|
|
stp $acc0,$acc1,[$tp,#8*0] // result!!!
|
||
|
|
ldp $acc0,$acc1,[sp,#8*4] // t[0..3]
|
||
|
|
stp $acc2,$acc3,[$tp,#8*2] // result!!!
|
||
|
|
ldp $acc2,$acc3,[sp,#8*6]
|
||
|
|
|
||
|
|
ldp $m0,$m1,[$np,#8*0] // n[0..3]
|
||
|
|
mov $tp,sp
|
||
|
|
ldp $m2,$m3,[$np,#8*2]
|
||
|
|
adds $np,$np,#8*4 // clear carry bit
|
||
|
|
mov $carry,xzr
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.Loop_mul4x_reduction:
|
||
|
|
mul $t0,$a0,$bi // lo(a[0..3]*b[4])
|
||
|
|
adc $carry,$carry,xzr // modulo-scheduled
|
||
|
|
mul $t1,$a1,$bi
|
||
|
|
add $cnt,$cnt,#8
|
||
|
|
mul $t2,$a2,$bi
|
||
|
|
and $cnt,$cnt,#31
|
||
|
|
mul $t3,$a3,$bi
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
umulh $t0,$a0,$bi // hi(a[0..3]*b[4])
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
mul $mi,$acc0,$n0 // t[0]*n0
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
umulh $t1,$a1,$bi
|
||
|
|
adcs $acc3,$acc3,$t3
|
||
|
|
umulh $t2,$a2,$bi
|
||
|
|
adc $acc4,xzr,xzr
|
||
|
|
umulh $t3,$a3,$bi
|
||
|
|
ldr $bi,[$bp,$cnt] // next b[i]
|
||
|
|
adds $acc1,$acc1,$t0
|
||
|
|
// (*) mul $t0,$m0,$mi
|
||
|
|
str $mi,[$tp],#8 // put aside t[0]*n0 for tail processing
|
||
|
|
adcs $acc2,$acc2,$t1
|
||
|
|
mul $t1,$m1,$mi // lo(n[0..3]*t[0]*n0
|
||
|
|
adcs $acc3,$acc3,$t2
|
||
|
|
mul $t2,$m2,$mi
|
||
|
|
adc $acc4,$acc4,$t3 // can't overflow
|
||
|
|
mul $t3,$m3,$mi
|
||
|
|
// (*) adds xzr,$acc0,$t0
|
||
|
|
subs xzr,$acc0,#1 // (*)
|
||
|
|
umulh $t0,$m0,$mi // hi(n[0..3]*t[0]*n0
|
||
|
|
adcs $acc0,$acc1,$t1
|
||
|
|
umulh $t1,$m1,$mi
|
||
|
|
adcs $acc1,$acc2,$t2
|
||
|
|
umulh $t2,$m2,$mi
|
||
|
|
adcs $acc2,$acc3,$t3
|
||
|
|
umulh $t3,$m3,$mi
|
||
|
|
adcs $acc3,$acc4,$carry
|
||
|
|
adc $carry,xzr,xzr
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
adcs $acc3,$acc3,$t3
|
||
|
|
//adc $carry,$carry,xzr
|
||
|
|
cbnz $cnt,.Loop_mul4x_reduction
|
||
|
|
|
||
|
|
adc $carry,$carry,xzr
|
||
|
|
ldp $t0,$t1,[$tp,#8*4] // t[4..7]
|
||
|
|
ldp $t2,$t3,[$tp,#8*6]
|
||
|
|
ldp $a0,$a1,[$ap,#8*0] // a[4..7]
|
||
|
|
ldp $a2,$a3,[$ap,#8*2]
|
||
|
|
add $ap,$ap,#8*4
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
adcs $acc3,$acc3,$t3
|
||
|
|
//adc $carry,$carry,xzr
|
||
|
|
|
||
|
|
ldr $mi,[sp] // t[0]*n0
|
||
|
|
ldp $m0,$m1,[$np,#8*0] // n[4..7]
|
||
|
|
ldp $m2,$m3,[$np,#8*2]
|
||
|
|
add $np,$np,#8*4
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.Loop_mul4x_tail:
|
||
|
|
mul $t0,$a0,$bi // lo(a[4..7]*b[4])
|
||
|
|
adc $carry,$carry,xzr // modulo-scheduled
|
||
|
|
mul $t1,$a1,$bi
|
||
|
|
add $cnt,$cnt,#8
|
||
|
|
mul $t2,$a2,$bi
|
||
|
|
and $cnt,$cnt,#31
|
||
|
|
mul $t3,$a3,$bi
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
umulh $t0,$a0,$bi // hi(a[4..7]*b[4])
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
umulh $t1,$a1,$bi
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
umulh $t2,$a2,$bi
|
||
|
|
adcs $acc3,$acc3,$t3
|
||
|
|
umulh $t3,$a3,$bi
|
||
|
|
adc $acc4,xzr,xzr
|
||
|
|
ldr $bi,[$bp,$cnt] // next b[i]
|
||
|
|
adds $acc1,$acc1,$t0
|
||
|
|
mul $t0,$m0,$mi // lo(n[4..7]*t[0]*n0)
|
||
|
|
adcs $acc2,$acc2,$t1
|
||
|
|
mul $t1,$m1,$mi
|
||
|
|
adcs $acc3,$acc3,$t2
|
||
|
|
mul $t2,$m2,$mi
|
||
|
|
adc $acc4,$acc4,$t3 // can't overflow
|
||
|
|
mul $t3,$m3,$mi
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
umulh $t0,$m0,$mi // hi(n[4..7]*t[0]*n0)
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
umulh $t1,$m1,$mi
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
umulh $t2,$m2,$mi
|
||
|
|
adcs $acc3,$acc3,$t3
|
||
|
|
umulh $t3,$m3,$mi
|
||
|
|
adcs $acc4,$acc4,$carry
|
||
|
|
ldr $mi,[sp,$cnt] // next a[0]*n0
|
||
|
|
adc $carry,xzr,xzr
|
||
|
|
str $acc0,[$tp],#8 // result!!!
|
||
|
|
adds $acc0,$acc1,$t0
|
||
|
|
sub $t0,$ap_end,$ap // done yet?
|
||
|
|
adcs $acc1,$acc2,$t1
|
||
|
|
adcs $acc2,$acc3,$t2
|
||
|
|
adcs $acc3,$acc4,$t3
|
||
|
|
//adc $carry,$carry,xzr
|
||
|
|
cbnz $cnt,.Loop_mul4x_tail
|
||
|
|
|
||
|
|
sub $t1,$np,$num // rewinded np?
|
||
|
|
adc $carry,$carry,xzr
|
||
|
|
cbz $t0,.Loop_mul4x_break
|
||
|
|
|
||
|
|
ldp $t0,$t1,[$tp,#8*4]
|
||
|
|
ldp $t2,$t3,[$tp,#8*6]
|
||
|
|
ldp $a0,$a1,[$ap,#8*0]
|
||
|
|
ldp $a2,$a3,[$ap,#8*2]
|
||
|
|
add $ap,$ap,#8*4
|
||
|
|
adds $acc0,$acc0,$t0
|
||
|
|
adcs $acc1,$acc1,$t1
|
||
|
|
adcs $acc2,$acc2,$t2
|
||
|
|
adcs $acc3,$acc3,$t3
|
||
|
|
//adc $carry,$carry,xzr
|
||
|
|
ldp $m0,$m1,[$np,#8*0]
|
||
|
|
ldp $m2,$m3,[$np,#8*2]
|
||
|
|
add $np,$np,#8*4
|
||
|
|
b .Loop_mul4x_tail
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.Loop_mul4x_break:
|
||
|
|
ldp $t2,$t3,[x29,#96] // pull rp and &b[num]
|
||
|
|
adds $acc0,$acc0,$topmost
|
||
|
|
add $bp,$bp,#8*4 // bp++
|
||
|
|
adcs $acc1,$acc1,xzr
|
||
|
|
sub $ap,$ap,$num // rewind ap
|
||
|
|
adcs $acc2,$acc2,xzr
|
||
|
|
stp $acc0,$acc1,[$tp,#8*0] // result!!!
|
||
|
|
adcs $acc3,$acc3,xzr
|
||
|
|
ldp $acc0,$acc1,[sp,#8*4] // t[0..3]
|
||
|
|
adc $topmost,$carry,xzr
|
||
|
|
stp $acc2,$acc3,[$tp,#8*2] // result!!!
|
||
|
|
cmp $bp,$t3 // done yet?
|
||
|
|
ldp $acc2,$acc3,[sp,#8*6]
|
||
|
|
ldp $m0,$m1,[$t1,#8*0] // n[0..3]
|
||
|
|
ldp $m2,$m3,[$t1,#8*2]
|
||
|
|
add $np,$t1,#8*4
|
||
|
|
b.eq .Lmul4x_post
|
||
|
|
|
||
|
|
ldr $bi,[$bp]
|
||
|
|
ldp $a0,$a1,[$ap,#8*0] // a[0..3]
|
||
|
|
ldp $a2,$a3,[$ap,#8*2]
|
||
|
|
adds $ap,$ap,#8*4 // clear carry bit
|
||
|
|
mov $carry,xzr
|
||
|
|
mov $tp,sp
|
||
|
|
b .Loop_mul4x_reduction
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.Lmul4x_post:
|
||
|
|
// Final step. We see if result is larger than modulus, and
|
||
|
|
// if it is, subtract the modulus. But comparison implies
|
||
|
|
// subtraction. So we subtract modulus, see if it borrowed,
|
||
|
|
// and conditionally copy original value.
|
||
|
|
mov $rp,$t2
|
||
|
|
mov $ap_end,$t2 // $rp copy
|
||
|
|
subs $t0,$acc0,$m0
|
||
|
|
add $tp,sp,#8*8
|
||
|
|
sbcs $t1,$acc1,$m1
|
||
|
|
sub $cnt,$num,#8*4
|
||
|
|
|
||
|
|
.Lmul4x_sub:
|
||
|
|
sbcs $t2,$acc2,$m2
|
||
|
|
ldp $m0,$m1,[$np,#8*0]
|
||
|
|
sub $cnt,$cnt,#8*4
|
||
|
|
ldp $acc0,$acc1,[$tp,#8*0]
|
||
|
|
sbcs $t3,$acc3,$m3
|
||
|
|
ldp $m2,$m3,[$np,#8*2]
|
||
|
|
add $np,$np,#8*4
|
||
|
|
ldp $acc2,$acc3,[$tp,#8*2]
|
||
|
|
add $tp,$tp,#8*4
|
||
|
|
stp $t0,$t1,[$rp,#8*0]
|
||
|
|
sbcs $t0,$acc0,$m0
|
||
|
|
stp $t2,$t3,[$rp,#8*2]
|
||
|
|
add $rp,$rp,#8*4
|
||
|
|
sbcs $t1,$acc1,$m1
|
||
|
|
cbnz $cnt,.Lmul4x_sub
|
||
|
|
|
||
|
|
sbcs $t2,$acc2,$m2
|
||
|
|
mov $tp,sp
|
||
|
|
add $ap,sp,#8*4
|
||
|
|
ldp $a0,$a1,[$ap_end,#8*0]
|
||
|
|
sbcs $t3,$acc3,$m3
|
||
|
|
stp $t0,$t1,[$rp,#8*0]
|
||
|
|
ldp $a2,$a3,[$ap_end,#8*2]
|
||
|
|
stp $t2,$t3,[$rp,#8*2]
|
||
|
|
ldp $acc0,$acc1,[$ap,#8*0]
|
||
|
|
ldp $acc2,$acc3,[$ap,#8*2]
|
||
|
|
sbcs xzr,$topmost,xzr // did it borrow?
|
||
|
|
ldr x30,[x29,#8] // pull return address
|
||
|
|
|
||
|
|
sub $cnt,$num,#8*4
|
||
|
|
.Lmul4x_cond_copy:
|
||
|
|
sub $cnt,$cnt,#8*4
|
||
|
|
csel $t0,$acc0,$a0,lo
|
||
|
|
stp xzr,xzr,[$tp,#8*0]
|
||
|
|
csel $t1,$acc1,$a1,lo
|
||
|
|
ldp $a0,$a1,[$ap_end,#8*4]
|
||
|
|
ldp $acc0,$acc1,[$ap,#8*4]
|
||
|
|
csel $t2,$acc2,$a2,lo
|
||
|
|
stp xzr,xzr,[$tp,#8*2]
|
||
|
|
add $tp,$tp,#8*4
|
||
|
|
csel $t3,$acc3,$a3,lo
|
||
|
|
ldp $a2,$a3,[$ap_end,#8*6]
|
||
|
|
ldp $acc2,$acc3,[$ap,#8*6]
|
||
|
|
add $ap,$ap,#8*4
|
||
|
|
stp $t0,$t1,[$ap_end,#8*0]
|
||
|
|
stp $t2,$t3,[$ap_end,#8*2]
|
||
|
|
add $ap_end,$ap_end,#8*4
|
||
|
|
cbnz $cnt,.Lmul4x_cond_copy
|
||
|
|
|
||
|
|
csel $t0,$acc0,$a0,lo
|
||
|
|
stp xzr,xzr,[$tp,#8*0]
|
||
|
|
csel $t1,$acc1,$a1,lo
|
||
|
|
stp xzr,xzr,[$tp,#8*2]
|
||
|
|
csel $t2,$acc2,$a2,lo
|
||
|
|
stp xzr,xzr,[$tp,#8*3]
|
||
|
|
csel $t3,$acc3,$a3,lo
|
||
|
|
stp xzr,xzr,[$tp,#8*4]
|
||
|
|
stp $t0,$t1,[$ap_end,#8*0]
|
||
|
|
stp $t2,$t3,[$ap_end,#8*2]
|
||
|
|
|
||
|
|
b .Lmul4x_done
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.Lmul4x4_post_condition:
|
||
|
|
adc $carry,$carry,xzr
|
||
|
|
ldr $ap,[x29,#96] // pull rp
|
||
|
|
// $acc0-3,$carry hold result, $m0-7 hold modulus
|
||
|
|
subs $a0,$acc0,$m0
|
||
|
|
ldr x30,[x29,#8] // pull return address
|
||
|
|
sbcs $a1,$acc1,$m1
|
||
|
|
stp xzr,xzr,[sp,#8*0]
|
||
|
|
sbcs $a2,$acc2,$m2
|
||
|
|
stp xzr,xzr,[sp,#8*2]
|
||
|
|
sbcs $a3,$acc3,$m3
|
||
|
|
stp xzr,xzr,[sp,#8*4]
|
||
|
|
sbcs xzr,$carry,xzr // did it borrow?
|
||
|
|
stp xzr,xzr,[sp,#8*6]
|
||
|
|
|
||
|
|
// $a0-3 hold result-modulus
|
||
|
|
csel $a0,$acc0,$a0,lo
|
||
|
|
csel $a1,$acc1,$a1,lo
|
||
|
|
csel $a2,$acc2,$a2,lo
|
||
|
|
csel $a3,$acc3,$a3,lo
|
||
|
|
stp $a0,$a1,[$ap,#8*0]
|
||
|
|
stp $a2,$a3,[$ap,#8*2]
|
||
|
|
|
||
|
|
.Lmul4x_done:
|
||
|
|
ldp x19,x20,[x29,#16]
|
||
|
|
mov sp,x29
|
||
|
|
ldp x21,x22,[x29,#32]
|
||
|
|
mov x0,#1
|
||
|
|
ldp x23,x24,[x29,#48]
|
||
|
|
ldp x25,x26,[x29,#64]
|
||
|
|
ldp x27,x28,[x29,#80]
|
||
|
|
ldr x29,[sp],#128
|
||
|
|
// x30 loaded earlier
|
||
|
|
AARCH64_VALIDATE_LINK_REGISTER
|
||
|
|
ret
|
||
|
|
.size __bn_mul4x_mont,.-__bn_mul4x_mont
|
||
|
|
___
|
||
|
|
}
|
||
|
|
$code.=<<___;
|
||
|
|
.asciz "Montgomery Multiplication for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
|
||
|
|
.align 4
|
||
|
|
___
|
||
|
|
|
||
|
|
print $code;
|
||
|
|
|
||
|
|
close STDOUT or die "error closing STDOUT: $!";
|