|
|
#! /usr/bin/env perl |
|
|
# Copyright 2009-2016 The OpenSSL Project Authors. All Rights Reserved. |
|
|
# |
|
|
# Licensed under the OpenSSL license (the "License"). You may not use |
|
|
# this file except in compliance with the License. You can obtain a copy |
|
|
# in the file LICENSE in the source distribution or at |
|
|
# https://www.openssl.org/source/license.html |
|
|
|
|
|
|
|
|
# ==================================================================== |
|
|
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL |
|
|
# project. The module is, however, dual licensed under OpenSSL and |
|
|
# CRYPTOGAMS licenses depending on where you obtain it. For further |
|
|
# details see http://www.openssl.org/~appro/cryptogams/. |
|
|
# ==================================================================== |
|
|
# |
|
|
# This module implements support for Intel AES-NI extension. In |
|
|
# OpenSSL context it's used with Intel engine, but can also be used as |
|
|
# drop-in replacement for crypto/aes/asm/aes-586.pl [see below for |
|
|
# details]. |
|
|
# |
|
|
# Performance. |
|
|
# |
|
|
# To start with see corresponding paragraph in aesni-x86_64.pl... |
|
|
# Instead of filling table similar to one found there I've chosen to |
|
|
# summarize *comparison* results for raw ECB, CTR and CBC benchmarks. |
|
|
# The simplified table below represents 32-bit performance relative |
|
|
# to 64-bit one in every given point. Ratios vary for different |
|
|
# encryption modes, therefore interval values. |
|
|
# |
|
|
# 16-byte 64-byte 256-byte 1-KB 8-KB |
|
|
# 53-67% 67-84% 91-94% 95-98% 97-99.5% |
|
|
# |
|
|
# Lower ratios for smaller block sizes are perfectly understandable, |
|
|
# because function call overhead is higher in 32-bit mode. Largest |
|
|
# 8-KB block performance is virtually same: 32-bit code is less than |
|
|
# 1% slower for ECB, CBC and CCM, and ~3% slower otherwise. |
|
|
|
|
|
# January 2011 |
|
|
# |
|
|
# See aesni-x86_64.pl for details. Unlike x86_64 version this module |
|
|
# interleaves at most 6 aes[enc|dec] instructions, because there are |
|
|
# not enough registers for 8x interleave [which should be optimal for |
|
|
# Sandy Bridge]. Actually, performance results for 6x interleave |
|
|
# factor presented in aesni-x86_64.pl (except for CTR) are for this |
|
|
# module. |
|
|
|
|
|
# April 2011 |
|
|
# |
|
|
# Add aesni_xts_[en|de]crypt. Westmere spends 1.50 cycles processing |
|
|
# one byte out of 8KB with 128-bit key, Sandy Bridge - 1.09. |
|
|
|
|
|
# November 2015 |
|
|
# |
|
|
# Add aesni_ocb_[en|de]crypt. [Removed in BoringSSL] |
|
|
|
|
|
###################################################################### |
|
|
# Current large-block performance in cycles per byte processed with |
|
|
# 128-bit key (less is better). |
|
|
# |
|
|
# CBC en-/decrypt CTR XTS ECB OCB |
|
|
# Westmere 3.77/1.37 1.37 1.52 1.27 |
|
|
# * Bridge 5.07/0.98 0.99 1.09 0.91 1.10 |
|
|
# Haswell 4.44/0.80 0.97 1.03 0.72 0.76 |
|
|
# Skylake 2.68/0.65 0.65 0.66 0.64 0.66 |
|
|
# Silvermont 5.77/3.56 3.67 4.03 3.46 4.03 |
|
|
# Goldmont 3.84/1.39 1.39 1.63 1.31 1.70 |
|
|
# Bulldozer 5.80/0.98 1.05 1.24 0.93 1.23 |
|
|
|
|
|
$PREFIX="aes_hw"; # if $PREFIX is set to "AES", the script |
|
|
# generates drop-in replacement for |
|
|
# crypto/aes/asm/aes-586.pl:-) |
|
|
$AESNI_PREFIX="aes_hw"; |
|
|
$inline=1; # inline _aesni_[en|de]crypt |
|
|
|
|
|
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; |
|
|
push(@INC,"${dir}","${dir}../../../perlasm"); |
|
|
require "x86asm.pl"; |
|
|
|
|
|
$output = pop; |
|
|
open OUT,">$output"; |
|
|
*STDOUT=*OUT; |
|
|
|
|
|
&asm_init($ARGV[0]); |
|
|
|
|
|
&external_label("OPENSSL_ia32cap_P"); |
|
|
&preprocessor_ifdef("BORINGSSL_DISPATCH_TEST") |
|
|
&external_label("BORINGSSL_function_hit"); |
|
|
&preprocessor_endif(); |
|
|
&static_label("key_const"); |
|
|
|
|
|
if ($PREFIX eq $AESNI_PREFIX) { $movekey=\&movups; } |
|
|
else { $movekey=\&movups; } |
|
|
|
|
|
$len="eax"; |
|
|
$rounds="ecx"; |
|
|
$key="edx"; |
|
|
$inp="esi"; |
|
|
$out="edi"; |
|
|
$rounds_="ebx"; # backup copy for $rounds |
|
|
$key_="ebp"; # backup copy for $key |
|
|
|
|
|
$rndkey0="xmm0"; |
|
|
$rndkey1="xmm1"; |
|
|
$inout0="xmm2"; |
|
|
$inout1="xmm3"; |
|
|
$inout2="xmm4"; |
|
|
$inout3="xmm5"; $in1="xmm5"; |
|
|
$inout4="xmm6"; $in0="xmm6"; |
|
|
$inout5="xmm7"; $ivec="xmm7"; |
|
|
|
|
|
# AESNI extension |
|
|
sub aeskeygenassist |
|
|
{ my($dst,$src,$imm)=@_; |
|
|
if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/) |
|
|
{ &data_byte(0x66,0x0f,0x3a,0xdf,0xc0|($1<<3)|$2,$imm); } |
|
|
} |
|
|
sub aescommon |
|
|
{ my($opcodelet,$dst,$src)=@_; |
|
|
if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/) |
|
|
{ &data_byte(0x66,0x0f,0x38,$opcodelet,0xc0|($1<<3)|$2);} |
|
|
} |
|
|
sub aesimc { aescommon(0xdb,@_); } |
|
|
sub aesenc { aescommon(0xdc,@_); } |
|
|
sub aesenclast { aescommon(0xdd,@_); } |
|
|
sub aesdec { aescommon(0xde,@_); } |
|
|
sub aesdeclast { aescommon(0xdf,@_); } |
|
|
|
|
|
# Inline version of internal aesni_[en|de]crypt1 |
|
|
{ my $sn; |
|
|
sub aesni_inline_generate1 |
|
|
{ my ($p,$inout,$ivec)=@_; $inout=$inout0 if (!defined($inout)); |
|
|
$sn++; |
|
|
|
|
|
&$movekey ($rndkey0,&QWP(0,$key)); |
|
|
&$movekey ($rndkey1,&QWP(16,$key)); |
|
|
&xorps ($ivec,$rndkey0) if (defined($ivec)); |
|
|
&lea ($key,&DWP(32,$key)); |
|
|
&xorps ($inout,$ivec) if (defined($ivec)); |
|
|
&xorps ($inout,$rndkey0) if (!defined($ivec)); |
|
|
&set_label("${p}1_loop_$sn"); |
|
|
eval"&aes${p} ($inout,$rndkey1)"; |
|
|
&dec ($rounds); |
|
|
&$movekey ($rndkey1,&QWP(0,$key)); |
|
|
&lea ($key,&DWP(16,$key)); |
|
|
&jnz (&label("${p}1_loop_$sn")); |
|
|
eval"&aes${p}last ($inout,$rndkey1)"; |
|
|
}} |
|
|
|
|
|
sub aesni_generate1 # fully unrolled loop |
|
|
{ my ($p,$inout)=@_; $inout=$inout0 if (!defined($inout)); |
|
|
|
|
|
&function_begin_B("_aesni_${p}rypt1"); |
|
|
&movups ($rndkey0,&QWP(0,$key)); |
|
|
&$movekey ($rndkey1,&QWP(0x10,$key)); |
|
|
&xorps ($inout,$rndkey0); |
|
|
&$movekey ($rndkey0,&QWP(0x20,$key)); |
|
|
&lea ($key,&DWP(0x30,$key)); |
|
|
&cmp ($rounds,11); |
|
|
&jb (&label("${p}128")); |
|
|
&lea ($key,&DWP(0x20,$key)); |
|
|
&je (&label("${p}192")); |
|
|
&lea ($key,&DWP(0x20,$key)); |
|
|
eval"&aes${p} ($inout,$rndkey1)"; |
|
|
&$movekey ($rndkey1,&QWP(-0x40,$key)); |
|
|
eval"&aes${p} ($inout,$rndkey0)"; |
|
|
&$movekey ($rndkey0,&QWP(-0x30,$key)); |
|
|
&set_label("${p}192"); |
|
|
eval"&aes${p} ($inout,$rndkey1)"; |
|
|
&$movekey ($rndkey1,&QWP(-0x20,$key)); |
|
|
eval"&aes${p} ($inout,$rndkey0)"; |
|
|
&$movekey ($rndkey0,&QWP(-0x10,$key)); |
|
|
&set_label("${p}128"); |
|
|
eval"&aes${p} ($inout,$rndkey1)"; |
|
|
&$movekey ($rndkey1,&QWP(0,$key)); |
|
|
eval"&aes${p} ($inout,$rndkey0)"; |
|
|
&$movekey ($rndkey0,&QWP(0x10,$key)); |
|
|
eval"&aes${p} ($inout,$rndkey1)"; |
|
|
&$movekey ($rndkey1,&QWP(0x20,$key)); |
|
|
eval"&aes${p} ($inout,$rndkey0)"; |
|
|
&$movekey ($rndkey0,&QWP(0x30,$key)); |
|
|
eval"&aes${p} ($inout,$rndkey1)"; |
|
|
&$movekey ($rndkey1,&QWP(0x40,$key)); |
|
|
eval"&aes${p} ($inout,$rndkey0)"; |
|
|
&$movekey ($rndkey0,&QWP(0x50,$key)); |
|
|
eval"&aes${p} ($inout,$rndkey1)"; |
|
|
&$movekey ($rndkey1,&QWP(0x60,$key)); |
|
|
eval"&aes${p} ($inout,$rndkey0)"; |
|
|
&$movekey ($rndkey0,&QWP(0x70,$key)); |
|
|
eval"&aes${p} ($inout,$rndkey1)"; |
|
|
eval"&aes${p}last ($inout,$rndkey0)"; |
|
|
&ret(); |
|
|
&function_end_B("_aesni_${p}rypt1"); |
|
|
} |
|
|
|
|
|
# void $PREFIX_encrypt (const void *inp,void *out,const AES_KEY *key); |
|
|
&aesni_generate1("enc") if (!$inline); |
|
|
&function_begin_B("${PREFIX}_encrypt"); |
|
|
&record_function_hit(1); |
|
|
|
|
|
&mov ("eax",&wparam(0)); |
|
|
&mov ($key,&wparam(2)); |
|
|
&movups ($inout0,&QWP(0,"eax")); |
|
|
&mov ($rounds,&DWP(240,$key)); |
|
|
&mov ("eax",&wparam(1)); |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("enc"); } |
|
|
else |
|
|
{ &call ("_aesni_encrypt1"); } |
|
|
&pxor ($rndkey0,$rndkey0); # clear register bank |
|
|
&pxor ($rndkey1,$rndkey1); |
|
|
&movups (&QWP(0,"eax"),$inout0); |
|
|
&pxor ($inout0,$inout0); |
|
|
&ret (); |
|
|
&function_end_B("${PREFIX}_encrypt"); |
|
|
|
|
|
# void $PREFIX_decrypt (const void *inp,void *out,const AES_KEY *key); |
|
|
&aesni_generate1("dec") if(!$inline); |
|
|
&function_begin_B("${PREFIX}_decrypt"); |
|
|
&mov ("eax",&wparam(0)); |
|
|
&mov ($key,&wparam(2)); |
|
|
&movups ($inout0,&QWP(0,"eax")); |
|
|
&mov ($rounds,&DWP(240,$key)); |
|
|
&mov ("eax",&wparam(1)); |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("dec"); } |
|
|
else |
|
|
{ &call ("_aesni_decrypt1"); } |
|
|
&pxor ($rndkey0,$rndkey0); # clear register bank |
|
|
&pxor ($rndkey1,$rndkey1); |
|
|
&movups (&QWP(0,"eax"),$inout0); |
|
|
&pxor ($inout0,$inout0); |
|
|
&ret (); |
|
|
&function_end_B("${PREFIX}_decrypt"); |
|
|
|
|
|
# _aesni_[en|de]cryptN are private interfaces, N denotes interleave |
|
|
# factor. Why 3x subroutine were originally used in loops? Even though |
|
|
# aes[enc|dec] latency was originally 6, it could be scheduled only |
|
|
# every *2nd* cycle. Thus 3x interleave was the one providing optimal |
|
|
# utilization, i.e. when subroutine's throughput is virtually same as |
|
|
# of non-interleaved subroutine [for number of input blocks up to 3]. |
|
|
# This is why it originally made no sense to implement 2x subroutine. |
|
|
# But times change and it became appropriate to spend extra 192 bytes |
|
|
# on 2x subroutine on Atom Silvermont account. For processors that |
|
|
# can schedule aes[enc|dec] every cycle optimal interleave factor |
|
|
# equals to corresponding instructions latency. 8x is optimal for |
|
|
# * Bridge, but it's unfeasible to accommodate such implementation |
|
|
# in XMM registers addressable in 32-bit mode and therefore maximum |
|
|
# of 6x is used instead... |
|
|
|
|
|
sub aesni_generate2 |
|
|
{ my $p=shift; |
|
|
|
|
|
&function_begin_B("_aesni_${p}rypt2"); |
|
|
&$movekey ($rndkey0,&QWP(0,$key)); |
|
|
&shl ($rounds,4); |
|
|
&$movekey ($rndkey1,&QWP(16,$key)); |
|
|
&xorps ($inout0,$rndkey0); |
|
|
&pxor ($inout1,$rndkey0); |
|
|
&$movekey ($rndkey0,&QWP(32,$key)); |
|
|
&lea ($key,&DWP(32,$key,$rounds)); |
|
|
&neg ($rounds); |
|
|
&add ($rounds,16); |
|
|
|
|
|
&set_label("${p}2_loop"); |
|
|
eval"&aes${p} ($inout0,$rndkey1)"; |
|
|
eval"&aes${p} ($inout1,$rndkey1)"; |
|
|
&$movekey ($rndkey1,&QWP(0,$key,$rounds)); |
|
|
&add ($rounds,32); |
|
|
eval"&aes${p} ($inout0,$rndkey0)"; |
|
|
eval"&aes${p} ($inout1,$rndkey0)"; |
|
|
&$movekey ($rndkey0,&QWP(-16,$key,$rounds)); |
|
|
&jnz (&label("${p}2_loop")); |
|
|
eval"&aes${p} ($inout0,$rndkey1)"; |
|
|
eval"&aes${p} ($inout1,$rndkey1)"; |
|
|
eval"&aes${p}last ($inout0,$rndkey0)"; |
|
|
eval"&aes${p}last ($inout1,$rndkey0)"; |
|
|
&ret(); |
|
|
&function_end_B("_aesni_${p}rypt2"); |
|
|
} |
|
|
|
|
|
sub aesni_generate3 |
|
|
{ my $p=shift; |
|
|
|
|
|
&function_begin_B("_aesni_${p}rypt3"); |
|
|
&$movekey ($rndkey0,&QWP(0,$key)); |
|
|
&shl ($rounds,4); |
|
|
&$movekey ($rndkey1,&QWP(16,$key)); |
|
|
&xorps ($inout0,$rndkey0); |
|
|
&pxor ($inout1,$rndkey0); |
|
|
&pxor ($inout2,$rndkey0); |
|
|
&$movekey ($rndkey0,&QWP(32,$key)); |
|
|
&lea ($key,&DWP(32,$key,$rounds)); |
|
|
&neg ($rounds); |
|
|
&add ($rounds,16); |
|
|
|
|
|
&set_label("${p}3_loop"); |
|
|
eval"&aes${p} ($inout0,$rndkey1)"; |
|
|
eval"&aes${p} ($inout1,$rndkey1)"; |
|
|
eval"&aes${p} ($inout2,$rndkey1)"; |
|
|
&$movekey ($rndkey1,&QWP(0,$key,$rounds)); |
|
|
&add ($rounds,32); |
|
|
eval"&aes${p} ($inout0,$rndkey0)"; |
|
|
eval"&aes${p} ($inout1,$rndkey0)"; |
|
|
eval"&aes${p} ($inout2,$rndkey0)"; |
|
|
&$movekey ($rndkey0,&QWP(-16,$key,$rounds)); |
|
|
&jnz (&label("${p}3_loop")); |
|
|
eval"&aes${p} ($inout0,$rndkey1)"; |
|
|
eval"&aes${p} ($inout1,$rndkey1)"; |
|
|
eval"&aes${p} ($inout2,$rndkey1)"; |
|
|
eval"&aes${p}last ($inout0,$rndkey0)"; |
|
|
eval"&aes${p}last ($inout1,$rndkey0)"; |
|
|
eval"&aes${p}last ($inout2,$rndkey0)"; |
|
|
&ret(); |
|
|
&function_end_B("_aesni_${p}rypt3"); |
|
|
} |
|
|
|
|
|
# 4x interleave is implemented to improve small block performance, |
|
|
# most notably [and naturally] 4 block by ~30%. One can argue that one |
|
|
# should have implemented 5x as well, but improvement would be <20%, |
|
|
# so it's not worth it... |
|
|
sub aesni_generate4 |
|
|
{ my $p=shift; |
|
|
|
|
|
&function_begin_B("_aesni_${p}rypt4"); |
|
|
&$movekey ($rndkey0,&QWP(0,$key)); |
|
|
&$movekey ($rndkey1,&QWP(16,$key)); |
|
|
&shl ($rounds,4); |
|
|
&xorps ($inout0,$rndkey0); |
|
|
&pxor ($inout1,$rndkey0); |
|
|
&pxor ($inout2,$rndkey0); |
|
|
&pxor ($inout3,$rndkey0); |
|
|
&$movekey ($rndkey0,&QWP(32,$key)); |
|
|
&lea ($key,&DWP(32,$key,$rounds)); |
|
|
&neg ($rounds); |
|
|
&data_byte (0x0f,0x1f,0x40,0x00); |
|
|
&add ($rounds,16); |
|
|
|
|
|
&set_label("${p}4_loop"); |
|
|
eval"&aes${p} ($inout0,$rndkey1)"; |
|
|
eval"&aes${p} ($inout1,$rndkey1)"; |
|
|
eval"&aes${p} ($inout2,$rndkey1)"; |
|
|
eval"&aes${p} ($inout3,$rndkey1)"; |
|
|
&$movekey ($rndkey1,&QWP(0,$key,$rounds)); |
|
|
&add ($rounds,32); |
|
|
eval"&aes${p} ($inout0,$rndkey0)"; |
|
|
eval"&aes${p} ($inout1,$rndkey0)"; |
|
|
eval"&aes${p} ($inout2,$rndkey0)"; |
|
|
eval"&aes${p} ($inout3,$rndkey0)"; |
|
|
&$movekey ($rndkey0,&QWP(-16,$key,$rounds)); |
|
|
&jnz (&label("${p}4_loop")); |
|
|
|
|
|
eval"&aes${p} ($inout0,$rndkey1)"; |
|
|
eval"&aes${p} ($inout1,$rndkey1)"; |
|
|
eval"&aes${p} ($inout2,$rndkey1)"; |
|
|
eval"&aes${p} ($inout3,$rndkey1)"; |
|
|
eval"&aes${p}last ($inout0,$rndkey0)"; |
|
|
eval"&aes${p}last ($inout1,$rndkey0)"; |
|
|
eval"&aes${p}last ($inout2,$rndkey0)"; |
|
|
eval"&aes${p}last ($inout3,$rndkey0)"; |
|
|
&ret(); |
|
|
&function_end_B("_aesni_${p}rypt4"); |
|
|
} |
|
|
|
|
|
sub aesni_generate6 |
|
|
{ my $p=shift; |
|
|
|
|
|
&function_begin_B("_aesni_${p}rypt6"); |
|
|
&static_label("_aesni_${p}rypt6_enter"); |
|
|
&$movekey ($rndkey0,&QWP(0,$key)); |
|
|
&shl ($rounds,4); |
|
|
&$movekey ($rndkey1,&QWP(16,$key)); |
|
|
&xorps ($inout0,$rndkey0); |
|
|
&pxor ($inout1,$rndkey0); # pxor does better here |
|
|
&pxor ($inout2,$rndkey0); |
|
|
eval"&aes${p} ($inout0,$rndkey1)"; |
|
|
&pxor ($inout3,$rndkey0); |
|
|
&pxor ($inout4,$rndkey0); |
|
|
eval"&aes${p} ($inout1,$rndkey1)"; |
|
|
&lea ($key,&DWP(32,$key,$rounds)); |
|
|
&neg ($rounds); |
|
|
eval"&aes${p} ($inout2,$rndkey1)"; |
|
|
&pxor ($inout5,$rndkey0); |
|
|
&$movekey ($rndkey0,&QWP(0,$key,$rounds)); |
|
|
&add ($rounds,16); |
|
|
&jmp (&label("_aesni_${p}rypt6_inner")); |
|
|
|
|
|
&set_label("${p}6_loop",16); |
|
|
eval"&aes${p} ($inout0,$rndkey1)"; |
|
|
eval"&aes${p} ($inout1,$rndkey1)"; |
|
|
eval"&aes${p} ($inout2,$rndkey1)"; |
|
|
&set_label("_aesni_${p}rypt6_inner"); |
|
|
eval"&aes${p} ($inout3,$rndkey1)"; |
|
|
eval"&aes${p} ($inout4,$rndkey1)"; |
|
|
eval"&aes${p} ($inout5,$rndkey1)"; |
|
|
&set_label("_aesni_${p}rypt6_enter"); |
|
|
&$movekey ($rndkey1,&QWP(0,$key,$rounds)); |
|
|
&add ($rounds,32); |
|
|
eval"&aes${p} ($inout0,$rndkey0)"; |
|
|
eval"&aes${p} ($inout1,$rndkey0)"; |
|
|
eval"&aes${p} ($inout2,$rndkey0)"; |
|
|
eval"&aes${p} ($inout3,$rndkey0)"; |
|
|
eval"&aes${p} ($inout4,$rndkey0)"; |
|
|
eval"&aes${p} ($inout5,$rndkey0)"; |
|
|
&$movekey ($rndkey0,&QWP(-16,$key,$rounds)); |
|
|
&jnz (&label("${p}6_loop")); |
|
|
|
|
|
eval"&aes${p} ($inout0,$rndkey1)"; |
|
|
eval"&aes${p} ($inout1,$rndkey1)"; |
|
|
eval"&aes${p} ($inout2,$rndkey1)"; |
|
|
eval"&aes${p} ($inout3,$rndkey1)"; |
|
|
eval"&aes${p} ($inout4,$rndkey1)"; |
|
|
eval"&aes${p} ($inout5,$rndkey1)"; |
|
|
eval"&aes${p}last ($inout0,$rndkey0)"; |
|
|
eval"&aes${p}last ($inout1,$rndkey0)"; |
|
|
eval"&aes${p}last ($inout2,$rndkey0)"; |
|
|
eval"&aes${p}last ($inout3,$rndkey0)"; |
|
|
eval"&aes${p}last ($inout4,$rndkey0)"; |
|
|
eval"&aes${p}last ($inout5,$rndkey0)"; |
|
|
&ret(); |
|
|
&function_end_B("_aesni_${p}rypt6"); |
|
|
} |
|
|
&aesni_generate2("enc") if ($PREFIX eq $AESNI_PREFIX); |
|
|
&aesni_generate2("dec"); |
|
|
&aesni_generate3("enc") if ($PREFIX eq $AESNI_PREFIX); |
|
|
&aesni_generate3("dec"); |
|
|
&aesni_generate4("enc") if ($PREFIX eq $AESNI_PREFIX); |
|
|
&aesni_generate4("dec"); |
|
|
&aesni_generate6("enc") if ($PREFIX eq $AESNI_PREFIX); |
|
|
&aesni_generate6("dec"); |
|
|
|
|
|
if ($PREFIX eq $AESNI_PREFIX) { |
|
|
###################################################################### |
|
|
# void aes_hw_ecb_encrypt (const void *in, void *out, |
|
|
# size_t length, const AES_KEY *key, |
|
|
# int enc); |
|
|
&function_begin("${PREFIX}_ecb_encrypt"); |
|
|
&mov ($inp,&wparam(0)); |
|
|
&mov ($out,&wparam(1)); |
|
|
&mov ($len,&wparam(2)); |
|
|
&mov ($key,&wparam(3)); |
|
|
&mov ($rounds_,&wparam(4)); |
|
|
&and ($len,-16); |
|
|
&jz (&label("ecb_ret")); |
|
|
&mov ($rounds,&DWP(240,$key)); |
|
|
&test ($rounds_,$rounds_); |
|
|
&jz (&label("ecb_decrypt")); |
|
|
|
|
|
&mov ($key_,$key); # backup $key |
|
|
&mov ($rounds_,$rounds); # backup $rounds |
|
|
&cmp ($len,0x60); |
|
|
&jb (&label("ecb_enc_tail")); |
|
|
|
|
|
&movdqu ($inout0,&QWP(0,$inp)); |
|
|
&movdqu ($inout1,&QWP(0x10,$inp)); |
|
|
&movdqu ($inout2,&QWP(0x20,$inp)); |
|
|
&movdqu ($inout3,&QWP(0x30,$inp)); |
|
|
&movdqu ($inout4,&QWP(0x40,$inp)); |
|
|
&movdqu ($inout5,&QWP(0x50,$inp)); |
|
|
&lea ($inp,&DWP(0x60,$inp)); |
|
|
&sub ($len,0x60); |
|
|
&jmp (&label("ecb_enc_loop6_enter")); |
|
|
|
|
|
&set_label("ecb_enc_loop6",16); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movdqu ($inout0,&QWP(0,$inp)); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&movdqu ($inout1,&QWP(0x10,$inp)); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&movdqu ($inout2,&QWP(0x20,$inp)); |
|
|
&movups (&QWP(0x30,$out),$inout3); |
|
|
&movdqu ($inout3,&QWP(0x30,$inp)); |
|
|
&movups (&QWP(0x40,$out),$inout4); |
|
|
&movdqu ($inout4,&QWP(0x40,$inp)); |
|
|
&movups (&QWP(0x50,$out),$inout5); |
|
|
&lea ($out,&DWP(0x60,$out)); |
|
|
&movdqu ($inout5,&QWP(0x50,$inp)); |
|
|
&lea ($inp,&DWP(0x60,$inp)); |
|
|
&set_label("ecb_enc_loop6_enter"); |
|
|
|
|
|
&call ("_aesni_encrypt6"); |
|
|
|
|
|
&mov ($key,$key_); # restore $key |
|
|
&mov ($rounds,$rounds_); # restore $rounds |
|
|
&sub ($len,0x60); |
|
|
&jnc (&label("ecb_enc_loop6")); |
|
|
|
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&movups (&QWP(0x30,$out),$inout3); |
|
|
&movups (&QWP(0x40,$out),$inout4); |
|
|
&movups (&QWP(0x50,$out),$inout5); |
|
|
&lea ($out,&DWP(0x60,$out)); |
|
|
&add ($len,0x60); |
|
|
&jz (&label("ecb_ret")); |
|
|
|
|
|
&set_label("ecb_enc_tail"); |
|
|
&movups ($inout0,&QWP(0,$inp)); |
|
|
&cmp ($len,0x20); |
|
|
&jb (&label("ecb_enc_one")); |
|
|
&movups ($inout1,&QWP(0x10,$inp)); |
|
|
&je (&label("ecb_enc_two")); |
|
|
&movups ($inout2,&QWP(0x20,$inp)); |
|
|
&cmp ($len,0x40); |
|
|
&jb (&label("ecb_enc_three")); |
|
|
&movups ($inout3,&QWP(0x30,$inp)); |
|
|
&je (&label("ecb_enc_four")); |
|
|
&movups ($inout4,&QWP(0x40,$inp)); |
|
|
&xorps ($inout5,$inout5); |
|
|
&call ("_aesni_encrypt6"); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&movups (&QWP(0x30,$out),$inout3); |
|
|
&movups (&QWP(0x40,$out),$inout4); |
|
|
jmp (&label("ecb_ret")); |
|
|
|
|
|
&set_label("ecb_enc_one",16); |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("enc"); } |
|
|
else |
|
|
{ &call ("_aesni_encrypt1"); } |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&jmp (&label("ecb_ret")); |
|
|
|
|
|
&set_label("ecb_enc_two",16); |
|
|
&call ("_aesni_encrypt2"); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&jmp (&label("ecb_ret")); |
|
|
|
|
|
&set_label("ecb_enc_three",16); |
|
|
&call ("_aesni_encrypt3"); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&jmp (&label("ecb_ret")); |
|
|
|
|
|
&set_label("ecb_enc_four",16); |
|
|
&call ("_aesni_encrypt4"); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&movups (&QWP(0x30,$out),$inout3); |
|
|
&jmp (&label("ecb_ret")); |
|
|
###################################################################### |
|
|
&set_label("ecb_decrypt",16); |
|
|
&mov ($key_,$key); # backup $key |
|
|
&mov ($rounds_,$rounds); # backup $rounds |
|
|
&cmp ($len,0x60); |
|
|
&jb (&label("ecb_dec_tail")); |
|
|
|
|
|
&movdqu ($inout0,&QWP(0,$inp)); |
|
|
&movdqu ($inout1,&QWP(0x10,$inp)); |
|
|
&movdqu ($inout2,&QWP(0x20,$inp)); |
|
|
&movdqu ($inout3,&QWP(0x30,$inp)); |
|
|
&movdqu ($inout4,&QWP(0x40,$inp)); |
|
|
&movdqu ($inout5,&QWP(0x50,$inp)); |
|
|
&lea ($inp,&DWP(0x60,$inp)); |
|
|
&sub ($len,0x60); |
|
|
&jmp (&label("ecb_dec_loop6_enter")); |
|
|
|
|
|
&set_label("ecb_dec_loop6",16); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movdqu ($inout0,&QWP(0,$inp)); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&movdqu ($inout1,&QWP(0x10,$inp)); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&movdqu ($inout2,&QWP(0x20,$inp)); |
|
|
&movups (&QWP(0x30,$out),$inout3); |
|
|
&movdqu ($inout3,&QWP(0x30,$inp)); |
|
|
&movups (&QWP(0x40,$out),$inout4); |
|
|
&movdqu ($inout4,&QWP(0x40,$inp)); |
|
|
&movups (&QWP(0x50,$out),$inout5); |
|
|
&lea ($out,&DWP(0x60,$out)); |
|
|
&movdqu ($inout5,&QWP(0x50,$inp)); |
|
|
&lea ($inp,&DWP(0x60,$inp)); |
|
|
&set_label("ecb_dec_loop6_enter"); |
|
|
|
|
|
&call ("_aesni_decrypt6"); |
|
|
|
|
|
&mov ($key,$key_); # restore $key |
|
|
&mov ($rounds,$rounds_); # restore $rounds |
|
|
&sub ($len,0x60); |
|
|
&jnc (&label("ecb_dec_loop6")); |
|
|
|
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&movups (&QWP(0x30,$out),$inout3); |
|
|
&movups (&QWP(0x40,$out),$inout4); |
|
|
&movups (&QWP(0x50,$out),$inout5); |
|
|
&lea ($out,&DWP(0x60,$out)); |
|
|
&add ($len,0x60); |
|
|
&jz (&label("ecb_ret")); |
|
|
|
|
|
&set_label("ecb_dec_tail"); |
|
|
&movups ($inout0,&QWP(0,$inp)); |
|
|
&cmp ($len,0x20); |
|
|
&jb (&label("ecb_dec_one")); |
|
|
&movups ($inout1,&QWP(0x10,$inp)); |
|
|
&je (&label("ecb_dec_two")); |
|
|
&movups ($inout2,&QWP(0x20,$inp)); |
|
|
&cmp ($len,0x40); |
|
|
&jb (&label("ecb_dec_three")); |
|
|
&movups ($inout3,&QWP(0x30,$inp)); |
|
|
&je (&label("ecb_dec_four")); |
|
|
&movups ($inout4,&QWP(0x40,$inp)); |
|
|
&xorps ($inout5,$inout5); |
|
|
&call ("_aesni_decrypt6"); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&movups (&QWP(0x30,$out),$inout3); |
|
|
&movups (&QWP(0x40,$out),$inout4); |
|
|
&jmp (&label("ecb_ret")); |
|
|
|
|
|
&set_label("ecb_dec_one",16); |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("dec"); } |
|
|
else |
|
|
{ &call ("_aesni_decrypt1"); } |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&jmp (&label("ecb_ret")); |
|
|
|
|
|
&set_label("ecb_dec_two",16); |
|
|
&call ("_aesni_decrypt2"); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&jmp (&label("ecb_ret")); |
|
|
|
|
|
&set_label("ecb_dec_three",16); |
|
|
&call ("_aesni_decrypt3"); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&jmp (&label("ecb_ret")); |
|
|
|
|
|
&set_label("ecb_dec_four",16); |
|
|
&call ("_aesni_decrypt4"); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&movups (&QWP(0x30,$out),$inout3); |
|
|
|
|
|
&set_label("ecb_ret"); |
|
|
&pxor ("xmm0","xmm0"); # clear register bank |
|
|
&pxor ("xmm1","xmm1"); |
|
|
&pxor ("xmm2","xmm2"); |
|
|
&pxor ("xmm3","xmm3"); |
|
|
&pxor ("xmm4","xmm4"); |
|
|
&pxor ("xmm5","xmm5"); |
|
|
&pxor ("xmm6","xmm6"); |
|
|
&pxor ("xmm7","xmm7"); |
|
|
&function_end("${PREFIX}_ecb_encrypt"); |
|
|
|
|
|
###################################################################### |
|
|
# void aes_hw_ccm64_[en|de]crypt_blocks (const void *in, void *out, |
|
|
# size_t blocks, const AES_KEY *key, |
|
|
# const char *ivec,char *cmac); |
|
|
# |
|
|
# Handles only complete blocks, operates on 64-bit counter and |
|
|
# does not update *ivec! Nor does it finalize CMAC value |
|
|
# (see engine/eng_aesni.c for details) |
|
|
# |
|
|
{ my $cmac=$inout1; |
|
|
&function_begin("${PREFIX}_ccm64_encrypt_blocks"); |
|
|
&mov ($inp,&wparam(0)); |
|
|
&mov ($out,&wparam(1)); |
|
|
&mov ($len,&wparam(2)); |
|
|
&mov ($key,&wparam(3)); |
|
|
&mov ($rounds_,&wparam(4)); |
|
|
&mov ($rounds,&wparam(5)); |
|
|
&mov ($key_,"esp"); |
|
|
&sub ("esp",60); |
|
|
&and ("esp",-16); # align stack |
|
|
&mov (&DWP(48,"esp"),$key_); |
|
|
|
|
|
&movdqu ($ivec,&QWP(0,$rounds_)); # load ivec |
|
|
&movdqu ($cmac,&QWP(0,$rounds)); # load cmac |
|
|
&mov ($rounds,&DWP(240,$key)); |
|
|
|
|
|
# compose byte-swap control mask for pshufb on stack |
|
|
&mov (&DWP(0,"esp"),0x0c0d0e0f); |
|
|
&mov (&DWP(4,"esp"),0x08090a0b); |
|
|
&mov (&DWP(8,"esp"),0x04050607); |
|
|
&mov (&DWP(12,"esp"),0x00010203); |
|
|
|
|
|
# compose counter increment vector on stack |
|
|
&mov ($rounds_,1); |
|
|
&xor ($key_,$key_); |
|
|
&mov (&DWP(16,"esp"),$rounds_); |
|
|
&mov (&DWP(20,"esp"),$key_); |
|
|
&mov (&DWP(24,"esp"),$key_); |
|
|
&mov (&DWP(28,"esp"),$key_); |
|
|
|
|
|
&shl ($rounds,4); |
|
|
&mov ($rounds_,16); |
|
|
&lea ($key_,&DWP(0,$key)); |
|
|
&movdqa ($inout3,&QWP(0,"esp")); |
|
|
&movdqa ($inout0,$ivec); |
|
|
&lea ($key,&DWP(32,$key,$rounds)); |
|
|
&sub ($rounds_,$rounds); |
|
|
&pshufb ($ivec,$inout3); |
|
|
|
|
|
&set_label("ccm64_enc_outer"); |
|
|
&$movekey ($rndkey0,&QWP(0,$key_)); |
|
|
&mov ($rounds,$rounds_); |
|
|
&movups ($in0,&QWP(0,$inp)); |
|
|
|
|
|
&xorps ($inout0,$rndkey0); |
|
|
&$movekey ($rndkey1,&QWP(16,$key_)); |
|
|
&xorps ($rndkey0,$in0); |
|
|
&xorps ($cmac,$rndkey0); # cmac^=inp |
|
|
&$movekey ($rndkey0,&QWP(32,$key_)); |
|
|
|
|
|
&set_label("ccm64_enc2_loop"); |
|
|
&aesenc ($inout0,$rndkey1); |
|
|
&aesenc ($cmac,$rndkey1); |
|
|
&$movekey ($rndkey1,&QWP(0,$key,$rounds)); |
|
|
&add ($rounds,32); |
|
|
&aesenc ($inout0,$rndkey0); |
|
|
&aesenc ($cmac,$rndkey0); |
|
|
&$movekey ($rndkey0,&QWP(-16,$key,$rounds)); |
|
|
&jnz (&label("ccm64_enc2_loop")); |
|
|
&aesenc ($inout0,$rndkey1); |
|
|
&aesenc ($cmac,$rndkey1); |
|
|
&paddq ($ivec,&QWP(16,"esp")); |
|
|
&dec ($len); |
|
|
&aesenclast ($inout0,$rndkey0); |
|
|
&aesenclast ($cmac,$rndkey0); |
|
|
|
|
|
&lea ($inp,&DWP(16,$inp)); |
|
|
&xorps ($in0,$inout0); # inp^=E(ivec) |
|
|
&movdqa ($inout0,$ivec); |
|
|
&movups (&QWP(0,$out),$in0); # save output |
|
|
&pshufb ($inout0,$inout3); |
|
|
&lea ($out,&DWP(16,$out)); |
|
|
&jnz (&label("ccm64_enc_outer")); |
|
|
|
|
|
&mov ("esp",&DWP(48,"esp")); |
|
|
&mov ($out,&wparam(5)); |
|
|
&movups (&QWP(0,$out),$cmac); |
|
|
|
|
|
&pxor ("xmm0","xmm0"); # clear register bank |
|
|
&pxor ("xmm1","xmm1"); |
|
|
&pxor ("xmm2","xmm2"); |
|
|
&pxor ("xmm3","xmm3"); |
|
|
&pxor ("xmm4","xmm4"); |
|
|
&pxor ("xmm5","xmm5"); |
|
|
&pxor ("xmm6","xmm6"); |
|
|
&pxor ("xmm7","xmm7"); |
|
|
&function_end("${PREFIX}_ccm64_encrypt_blocks"); |
|
|
|
|
|
&function_begin("${PREFIX}_ccm64_decrypt_blocks"); |
|
|
&mov ($inp,&wparam(0)); |
|
|
&mov ($out,&wparam(1)); |
|
|
&mov ($len,&wparam(2)); |
|
|
&mov ($key,&wparam(3)); |
|
|
&mov ($rounds_,&wparam(4)); |
|
|
&mov ($rounds,&wparam(5)); |
|
|
&mov ($key_,"esp"); |
|
|
&sub ("esp",60); |
|
|
&and ("esp",-16); # align stack |
|
|
&mov (&DWP(48,"esp"),$key_); |
|
|
|
|
|
&movdqu ($ivec,&QWP(0,$rounds_)); # load ivec |
|
|
&movdqu ($cmac,&QWP(0,$rounds)); # load cmac |
|
|
&mov ($rounds,&DWP(240,$key)); |
|
|
|
|
|
# compose byte-swap control mask for pshufb on stack |
|
|
&mov (&DWP(0,"esp"),0x0c0d0e0f); |
|
|
&mov (&DWP(4,"esp"),0x08090a0b); |
|
|
&mov (&DWP(8,"esp"),0x04050607); |
|
|
&mov (&DWP(12,"esp"),0x00010203); |
|
|
|
|
|
# compose counter increment vector on stack |
|
|
&mov ($rounds_,1); |
|
|
&xor ($key_,$key_); |
|
|
&mov (&DWP(16,"esp"),$rounds_); |
|
|
&mov (&DWP(20,"esp"),$key_); |
|
|
&mov (&DWP(24,"esp"),$key_); |
|
|
&mov (&DWP(28,"esp"),$key_); |
|
|
|
|
|
&movdqa ($inout3,&QWP(0,"esp")); # bswap mask |
|
|
&movdqa ($inout0,$ivec); |
|
|
|
|
|
&mov ($key_,$key); |
|
|
&mov ($rounds_,$rounds); |
|
|
|
|
|
&pshufb ($ivec,$inout3); |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("enc"); } |
|
|
else |
|
|
{ &call ("_aesni_encrypt1"); } |
|
|
&shl ($rounds_,4); |
|
|
&mov ($rounds,16); |
|
|
&movups ($in0,&QWP(0,$inp)); # load inp |
|
|
&paddq ($ivec,&QWP(16,"esp")); |
|
|
&lea ($inp,&QWP(16,$inp)); |
|
|
&sub ($rounds,$rounds_); |
|
|
&lea ($key,&DWP(32,$key_,$rounds_)); |
|
|
&mov ($rounds_,$rounds); |
|
|
&jmp (&label("ccm64_dec_outer")); |
|
|
|
|
|
&set_label("ccm64_dec_outer",16); |
|
|
&xorps ($in0,$inout0); # inp ^= E(ivec) |
|
|
&movdqa ($inout0,$ivec); |
|
|
&movups (&QWP(0,$out),$in0); # save output |
|
|
&lea ($out,&DWP(16,$out)); |
|
|
&pshufb ($inout0,$inout3); |
|
|
|
|
|
&sub ($len,1); |
|
|
&jz (&label("ccm64_dec_break")); |
|
|
|
|
|
&$movekey ($rndkey0,&QWP(0,$key_)); |
|
|
&mov ($rounds,$rounds_); |
|
|
&$movekey ($rndkey1,&QWP(16,$key_)); |
|
|
&xorps ($in0,$rndkey0); |
|
|
&xorps ($inout0,$rndkey0); |
|
|
&xorps ($cmac,$in0); # cmac^=out |
|
|
&$movekey ($rndkey0,&QWP(32,$key_)); |
|
|
|
|
|
&set_label("ccm64_dec2_loop"); |
|
|
&aesenc ($inout0,$rndkey1); |
|
|
&aesenc ($cmac,$rndkey1); |
|
|
&$movekey ($rndkey1,&QWP(0,$key,$rounds)); |
|
|
&add ($rounds,32); |
|
|
&aesenc ($inout0,$rndkey0); |
|
|
&aesenc ($cmac,$rndkey0); |
|
|
&$movekey ($rndkey0,&QWP(-16,$key,$rounds)); |
|
|
&jnz (&label("ccm64_dec2_loop")); |
|
|
&movups ($in0,&QWP(0,$inp)); # load inp |
|
|
&paddq ($ivec,&QWP(16,"esp")); |
|
|
&aesenc ($inout0,$rndkey1); |
|
|
&aesenc ($cmac,$rndkey1); |
|
|
&aesenclast ($inout0,$rndkey0); |
|
|
&aesenclast ($cmac,$rndkey0); |
|
|
&lea ($inp,&QWP(16,$inp)); |
|
|
&jmp (&label("ccm64_dec_outer")); |
|
|
|
|
|
&set_label("ccm64_dec_break",16); |
|
|
&mov ($rounds,&DWP(240,$key_)); |
|
|
&mov ($key,$key_); |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("enc",$cmac,$in0); } |
|
|
else |
|
|
{ &call ("_aesni_encrypt1",$cmac); } |
|
|
|
|
|
&mov ("esp",&DWP(48,"esp")); |
|
|
&mov ($out,&wparam(5)); |
|
|
&movups (&QWP(0,$out),$cmac); |
|
|
|
|
|
&pxor ("xmm0","xmm0"); # clear register bank |
|
|
&pxor ("xmm1","xmm1"); |
|
|
&pxor ("xmm2","xmm2"); |
|
|
&pxor ("xmm3","xmm3"); |
|
|
&pxor ("xmm4","xmm4"); |
|
|
&pxor ("xmm5","xmm5"); |
|
|
&pxor ("xmm6","xmm6"); |
|
|
&pxor ("xmm7","xmm7"); |
|
|
&function_end("${PREFIX}_ccm64_decrypt_blocks"); |
|
|
} |
|
|
|
|
|
###################################################################### |
|
|
# void aes_hw_ctr32_encrypt_blocks (const void *in, void *out, |
|
|
# size_t blocks, const AES_KEY *key, |
|
|
# const char *ivec); |
|
|
# |
|
|
# Handles only complete blocks, operates on 32-bit counter and |
|
|
# does not update *ivec! (see crypto/modes/ctr128.c for details) |
|
|
# |
|
|
# stack layout: |
|
|
# 0 pshufb mask |
|
|
# 16 vector addend: 0,6,6,6 |
|
|
# 32 counter-less ivec |
|
|
# 48 1st triplet of counter vector |
|
|
# 64 2nd triplet of counter vector |
|
|
# 80 saved %esp |
|
|
|
|
|
&function_begin("${PREFIX}_ctr32_encrypt_blocks"); |
|
|
&record_function_hit(0); |
|
|
|
|
|
&mov ($inp,&wparam(0)); |
|
|
&mov ($out,&wparam(1)); |
|
|
&mov ($len,&wparam(2)); |
|
|
&mov ($key,&wparam(3)); |
|
|
&mov ($rounds_,&wparam(4)); |
|
|
&mov ($key_,"esp"); |
|
|
&sub ("esp",88); |
|
|
&and ("esp",-16); # align stack |
|
|
&mov (&DWP(80,"esp"),$key_); |
|
|
|
|
|
&cmp ($len,1); |
|
|
&je (&label("ctr32_one_shortcut")); |
|
|
|
|
|
&movdqu ($inout5,&QWP(0,$rounds_)); # load ivec |
|
|
|
|
|
# compose byte-swap control mask for pshufb on stack |
|
|
&mov (&DWP(0,"esp"),0x0c0d0e0f); |
|
|
&mov (&DWP(4,"esp"),0x08090a0b); |
|
|
&mov (&DWP(8,"esp"),0x04050607); |
|
|
&mov (&DWP(12,"esp"),0x00010203); |
|
|
|
|
|
# compose counter increment vector on stack |
|
|
&mov ($rounds,6); |
|
|
&xor ($key_,$key_); |
|
|
&mov (&DWP(16,"esp"),$rounds); |
|
|
&mov (&DWP(20,"esp"),$rounds); |
|
|
&mov (&DWP(24,"esp"),$rounds); |
|
|
&mov (&DWP(28,"esp"),$key_); |
|
|
|
|
|
&pextrd ($rounds_,$inout5,3); # pull 32-bit counter |
|
|
&pinsrd ($inout5,$key_,3); # wipe 32-bit counter |
|
|
|
|
|
&mov ($rounds,&DWP(240,$key)); # key->rounds |
|
|
|
|
|
# compose 2 vectors of 3x32-bit counters |
|
|
&bswap ($rounds_); |
|
|
&pxor ($rndkey0,$rndkey0); |
|
|
&pxor ($rndkey1,$rndkey1); |
|
|
&movdqa ($inout0,&QWP(0,"esp")); # load byte-swap mask |
|
|
&pinsrd ($rndkey0,$rounds_,0); |
|
|
&lea ($key_,&DWP(3,$rounds_)); |
|
|
&pinsrd ($rndkey1,$key_,0); |
|
|
&inc ($rounds_); |
|
|
&pinsrd ($rndkey0,$rounds_,1); |
|
|
&inc ($key_); |
|
|
&pinsrd ($rndkey1,$key_,1); |
|
|
&inc ($rounds_); |
|
|
&pinsrd ($rndkey0,$rounds_,2); |
|
|
&inc ($key_); |
|
|
&pinsrd ($rndkey1,$key_,2); |
|
|
&movdqa (&QWP(48,"esp"),$rndkey0); # save 1st triplet |
|
|
&pshufb ($rndkey0,$inout0); # byte swap |
|
|
&movdqu ($inout4,&QWP(0,$key)); # key[0] |
|
|
&movdqa (&QWP(64,"esp"),$rndkey1); # save 2nd triplet |
|
|
&pshufb ($rndkey1,$inout0); # byte swap |
|
|
|
|
|
&pshufd ($inout0,$rndkey0,3<<6); # place counter to upper dword |
|
|
&pshufd ($inout1,$rndkey0,2<<6); |
|
|
&cmp ($len,6); |
|
|
&jb (&label("ctr32_tail")); |
|
|
&pxor ($inout5,$inout4); # counter-less ivec^key[0] |
|
|
&shl ($rounds,4); |
|
|
&mov ($rounds_,16); |
|
|
&movdqa (&QWP(32,"esp"),$inout5); # save counter-less ivec^key[0] |
|
|
&mov ($key_,$key); # backup $key |
|
|
&sub ($rounds_,$rounds); # backup twisted $rounds |
|
|
&lea ($key,&DWP(32,$key,$rounds)); |
|
|
&sub ($len,6); |
|
|
&jmp (&label("ctr32_loop6")); |
|
|
|
|
|
&set_label("ctr32_loop6",16); |
|
|
# inlining _aesni_encrypt6's prologue gives ~6% improvement... |
|
|
&pshufd ($inout2,$rndkey0,1<<6); |
|
|
&movdqa ($rndkey0,&QWP(32,"esp")); # pull counter-less ivec |
|
|
&pshufd ($inout3,$rndkey1,3<<6); |
|
|
&pxor ($inout0,$rndkey0); # merge counter-less ivec |
|
|
&pshufd ($inout4,$rndkey1,2<<6); |
|
|
&pxor ($inout1,$rndkey0); |
|
|
&pshufd ($inout5,$rndkey1,1<<6); |
|
|
&$movekey ($rndkey1,&QWP(16,$key_)); |
|
|
&pxor ($inout2,$rndkey0); |
|
|
&pxor ($inout3,$rndkey0); |
|
|
&aesenc ($inout0,$rndkey1); |
|
|
&pxor ($inout4,$rndkey0); |
|
|
&pxor ($inout5,$rndkey0); |
|
|
&aesenc ($inout1,$rndkey1); |
|
|
&$movekey ($rndkey0,&QWP(32,$key_)); |
|
|
&mov ($rounds,$rounds_); |
|
|
&aesenc ($inout2,$rndkey1); |
|
|
&aesenc ($inout3,$rndkey1); |
|
|
&aesenc ($inout4,$rndkey1); |
|
|
&aesenc ($inout5,$rndkey1); |
|
|
|
|
|
&call (&label("_aesni_encrypt6_enter")); |
|
|
|
|
|
&movups ($rndkey1,&QWP(0,$inp)); |
|
|
&movups ($rndkey0,&QWP(0x10,$inp)); |
|
|
&xorps ($inout0,$rndkey1); |
|
|
&movups ($rndkey1,&QWP(0x20,$inp)); |
|
|
&xorps ($inout1,$rndkey0); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movdqa ($rndkey0,&QWP(16,"esp")); # load increment |
|
|
&xorps ($inout2,$rndkey1); |
|
|
&movdqa ($rndkey1,&QWP(64,"esp")); # load 2nd triplet |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
|
|
|
&paddd ($rndkey1,$rndkey0); # 2nd triplet increment |
|
|
&paddd ($rndkey0,&QWP(48,"esp")); # 1st triplet increment |
|
|
&movdqa ($inout0,&QWP(0,"esp")); # load byte swap mask |
|
|
|
|
|
&movups ($inout1,&QWP(0x30,$inp)); |
|
|
&movups ($inout2,&QWP(0x40,$inp)); |
|
|
&xorps ($inout3,$inout1); |
|
|
&movups ($inout1,&QWP(0x50,$inp)); |
|
|
&lea ($inp,&DWP(0x60,$inp)); |
|
|
&movdqa (&QWP(48,"esp"),$rndkey0); # save 1st triplet |
|
|
&pshufb ($rndkey0,$inout0); # byte swap |
|
|
&xorps ($inout4,$inout2); |
|
|
&movups (&QWP(0x30,$out),$inout3); |
|
|
&xorps ($inout5,$inout1); |
|
|
&movdqa (&QWP(64,"esp"),$rndkey1); # save 2nd triplet |
|
|
&pshufb ($rndkey1,$inout0); # byte swap |
|
|
&movups (&QWP(0x40,$out),$inout4); |
|
|
&pshufd ($inout0,$rndkey0,3<<6); |
|
|
&movups (&QWP(0x50,$out),$inout5); |
|
|
&lea ($out,&DWP(0x60,$out)); |
|
|
|
|
|
&pshufd ($inout1,$rndkey0,2<<6); |
|
|
&sub ($len,6); |
|
|
&jnc (&label("ctr32_loop6")); |
|
|
|
|
|
&add ($len,6); |
|
|
&jz (&label("ctr32_ret")); |
|
|
&movdqu ($inout5,&QWP(0,$key_)); |
|
|
&mov ($key,$key_); |
|
|
&pxor ($inout5,&QWP(32,"esp")); # restore count-less ivec |
|
|
&mov ($rounds,&DWP(240,$key_)); # restore $rounds |
|
|
|
|
|
&set_label("ctr32_tail"); |
|
|
&por ($inout0,$inout5); |
|
|
&cmp ($len,2); |
|
|
&jb (&label("ctr32_one")); |
|
|
|
|
|
&pshufd ($inout2,$rndkey0,1<<6); |
|
|
&por ($inout1,$inout5); |
|
|
&je (&label("ctr32_two")); |
|
|
|
|
|
&pshufd ($inout3,$rndkey1,3<<6); |
|
|
&por ($inout2,$inout5); |
|
|
&cmp ($len,4); |
|
|
&jb (&label("ctr32_three")); |
|
|
|
|
|
&pshufd ($inout4,$rndkey1,2<<6); |
|
|
&por ($inout3,$inout5); |
|
|
&je (&label("ctr32_four")); |
|
|
|
|
|
&por ($inout4,$inout5); |
|
|
&call ("_aesni_encrypt6"); |
|
|
&movups ($rndkey1,&QWP(0,$inp)); |
|
|
&movups ($rndkey0,&QWP(0x10,$inp)); |
|
|
&xorps ($inout0,$rndkey1); |
|
|
&movups ($rndkey1,&QWP(0x20,$inp)); |
|
|
&xorps ($inout1,$rndkey0); |
|
|
&movups ($rndkey0,&QWP(0x30,$inp)); |
|
|
&xorps ($inout2,$rndkey1); |
|
|
&movups ($rndkey1,&QWP(0x40,$inp)); |
|
|
&xorps ($inout3,$rndkey0); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&xorps ($inout4,$rndkey1); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&movups (&QWP(0x30,$out),$inout3); |
|
|
&movups (&QWP(0x40,$out),$inout4); |
|
|
&jmp (&label("ctr32_ret")); |
|
|
|
|
|
&set_label("ctr32_one_shortcut",16); |
|
|
&movups ($inout0,&QWP(0,$rounds_)); # load ivec |
|
|
&mov ($rounds,&DWP(240,$key)); |
|
|
|
|
|
&set_label("ctr32_one"); |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("enc"); } |
|
|
else |
|
|
{ &call ("_aesni_encrypt1"); } |
|
|
&movups ($in0,&QWP(0,$inp)); |
|
|
&xorps ($in0,$inout0); |
|
|
&movups (&QWP(0,$out),$in0); |
|
|
&jmp (&label("ctr32_ret")); |
|
|
|
|
|
&set_label("ctr32_two",16); |
|
|
&call ("_aesni_encrypt2"); |
|
|
&movups ($inout3,&QWP(0,$inp)); |
|
|
&movups ($inout4,&QWP(0x10,$inp)); |
|
|
&xorps ($inout0,$inout3); |
|
|
&xorps ($inout1,$inout4); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&jmp (&label("ctr32_ret")); |
|
|
|
|
|
&set_label("ctr32_three",16); |
|
|
&call ("_aesni_encrypt3"); |
|
|
&movups ($inout3,&QWP(0,$inp)); |
|
|
&movups ($inout4,&QWP(0x10,$inp)); |
|
|
&xorps ($inout0,$inout3); |
|
|
&movups ($inout5,&QWP(0x20,$inp)); |
|
|
&xorps ($inout1,$inout4); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&xorps ($inout2,$inout5); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&jmp (&label("ctr32_ret")); |
|
|
|
|
|
&set_label("ctr32_four",16); |
|
|
&call ("_aesni_encrypt4"); |
|
|
&movups ($inout4,&QWP(0,$inp)); |
|
|
&movups ($inout5,&QWP(0x10,$inp)); |
|
|
&movups ($rndkey1,&QWP(0x20,$inp)); |
|
|
&xorps ($inout0,$inout4); |
|
|
&movups ($rndkey0,&QWP(0x30,$inp)); |
|
|
&xorps ($inout1,$inout5); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&xorps ($inout2,$rndkey1); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&xorps ($inout3,$rndkey0); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&movups (&QWP(0x30,$out),$inout3); |
|
|
|
|
|
&set_label("ctr32_ret"); |
|
|
&pxor ("xmm0","xmm0"); # clear register bank |
|
|
&pxor ("xmm1","xmm1"); |
|
|
&pxor ("xmm2","xmm2"); |
|
|
&pxor ("xmm3","xmm3"); |
|
|
&pxor ("xmm4","xmm4"); |
|
|
&movdqa (&QWP(32,"esp"),"xmm0"); # clear stack |
|
|
&pxor ("xmm5","xmm5"); |
|
|
&movdqa (&QWP(48,"esp"),"xmm0"); |
|
|
&pxor ("xmm6","xmm6"); |
|
|
&movdqa (&QWP(64,"esp"),"xmm0"); |
|
|
&pxor ("xmm7","xmm7"); |
|
|
&mov ("esp",&DWP(80,"esp")); |
|
|
&function_end("${PREFIX}_ctr32_encrypt_blocks"); |
|
|
|
|
|
###################################################################### |
|
|
# void aes_hw_xts_[en|de]crypt(const char *inp,char *out,size_t len, |
|
|
# const AES_KEY *key1, const AES_KEY *key2 |
|
|
# const unsigned char iv[16]); |
|
|
# |
|
|
{ my ($tweak,$twtmp,$twres,$twmask)=($rndkey1,$rndkey0,$inout0,$inout1); |
|
|
|
|
|
&function_begin("${PREFIX}_xts_encrypt"); |
|
|
&mov ($key,&wparam(4)); # key2 |
|
|
&mov ($inp,&wparam(5)); # clear-text tweak |
|
|
|
|
|
&mov ($rounds,&DWP(240,$key)); # key2->rounds |
|
|
&movups ($inout0,&QWP(0,$inp)); |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("enc"); } |
|
|
else |
|
|
{ &call ("_aesni_encrypt1"); } |
|
|
|
|
|
&mov ($inp,&wparam(0)); |
|
|
&mov ($out,&wparam(1)); |
|
|
&mov ($len,&wparam(2)); |
|
|
&mov ($key,&wparam(3)); # key1 |
|
|
|
|
|
&mov ($key_,"esp"); |
|
|
&sub ("esp",16*7+8); |
|
|
&mov ($rounds,&DWP(240,$key)); # key1->rounds |
|
|
&and ("esp",-16); # align stack |
|
|
|
|
|
&mov (&DWP(16*6+0,"esp"),0x87); # compose the magic constant |
|
|
&mov (&DWP(16*6+4,"esp"),0); |
|
|
&mov (&DWP(16*6+8,"esp"),1); |
|
|
&mov (&DWP(16*6+12,"esp"),0); |
|
|
&mov (&DWP(16*7+0,"esp"),$len); # save original $len |
|
|
&mov (&DWP(16*7+4,"esp"),$key_); # save original %esp |
|
|
|
|
|
&movdqa ($tweak,$inout0); |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&movdqa ($twmask,&QWP(6*16,"esp")); # 0x0...010...87 |
|
|
&pcmpgtd($twtmp,$tweak); # broadcast upper bits |
|
|
|
|
|
&and ($len,-16); |
|
|
&mov ($key_,$key); # backup $key |
|
|
&mov ($rounds_,$rounds); # backup $rounds |
|
|
&sub ($len,16*6); |
|
|
&jc (&label("xts_enc_short")); |
|
|
|
|
|
&shl ($rounds,4); |
|
|
&mov ($rounds_,16); |
|
|
&sub ($rounds_,$rounds); |
|
|
&lea ($key,&DWP(32,$key,$rounds)); |
|
|
&jmp (&label("xts_enc_loop6")); |
|
|
|
|
|
&set_label("xts_enc_loop6",16); |
|
|
for ($i=0;$i<4;$i++) { |
|
|
&pshufd ($twres,$twtmp,0x13); |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&movdqa (&QWP(16*$i,"esp"),$tweak); |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&pand ($twres,$twmask); # isolate carry and residue |
|
|
&pcmpgtd ($twtmp,$tweak); # broadcast upper bits |
|
|
&pxor ($tweak,$twres); |
|
|
} |
|
|
&pshufd ($inout5,$twtmp,0x13); |
|
|
&movdqa (&QWP(16*$i++,"esp"),$tweak); |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&$movekey ($rndkey0,&QWP(0,$key_)); |
|
|
&pand ($inout5,$twmask); # isolate carry and residue |
|
|
&movups ($inout0,&QWP(0,$inp)); # load input |
|
|
&pxor ($inout5,$tweak); |
|
|
|
|
|
# inline _aesni_encrypt6 prologue and flip xor with tweak and key[0] |
|
|
&mov ($rounds,$rounds_); # restore $rounds |
|
|
&movdqu ($inout1,&QWP(16*1,$inp)); |
|
|
&xorps ($inout0,$rndkey0); # input^=rndkey[0] |
|
|
&movdqu ($inout2,&QWP(16*2,$inp)); |
|
|
&pxor ($inout1,$rndkey0); |
|
|
&movdqu ($inout3,&QWP(16*3,$inp)); |
|
|
&pxor ($inout2,$rndkey0); |
|
|
&movdqu ($inout4,&QWP(16*4,$inp)); |
|
|
&pxor ($inout3,$rndkey0); |
|
|
&movdqu ($rndkey1,&QWP(16*5,$inp)); |
|
|
&pxor ($inout4,$rndkey0); |
|
|
&lea ($inp,&DWP(16*6,$inp)); |
|
|
&pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak |
|
|
&movdqa (&QWP(16*$i,"esp"),$inout5); # save last tweak |
|
|
&pxor ($inout5,$rndkey1); |
|
|
|
|
|
&$movekey ($rndkey1,&QWP(16,$key_)); |
|
|
&pxor ($inout1,&QWP(16*1,"esp")); |
|
|
&pxor ($inout2,&QWP(16*2,"esp")); |
|
|
&aesenc ($inout0,$rndkey1); |
|
|
&pxor ($inout3,&QWP(16*3,"esp")); |
|
|
&pxor ($inout4,&QWP(16*4,"esp")); |
|
|
&aesenc ($inout1,$rndkey1); |
|
|
&pxor ($inout5,$rndkey0); |
|
|
&$movekey ($rndkey0,&QWP(32,$key_)); |
|
|
&aesenc ($inout2,$rndkey1); |
|
|
&aesenc ($inout3,$rndkey1); |
|
|
&aesenc ($inout4,$rndkey1); |
|
|
&aesenc ($inout5,$rndkey1); |
|
|
&call (&label("_aesni_encrypt6_enter")); |
|
|
|
|
|
&movdqa ($tweak,&QWP(16*5,"esp")); # last tweak |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak |
|
|
&pcmpgtd ($twtmp,$tweak); # broadcast upper bits |
|
|
&xorps ($inout1,&QWP(16*1,"esp")); |
|
|
&movups (&QWP(16*0,$out),$inout0); # write output |
|
|
&xorps ($inout2,&QWP(16*2,"esp")); |
|
|
&movups (&QWP(16*1,$out),$inout1); |
|
|
&xorps ($inout3,&QWP(16*3,"esp")); |
|
|
&movups (&QWP(16*2,$out),$inout2); |
|
|
&xorps ($inout4,&QWP(16*4,"esp")); |
|
|
&movups (&QWP(16*3,$out),$inout3); |
|
|
&xorps ($inout5,$tweak); |
|
|
&movups (&QWP(16*4,$out),$inout4); |
|
|
&pshufd ($twres,$twtmp,0x13); |
|
|
&movups (&QWP(16*5,$out),$inout5); |
|
|
&lea ($out,&DWP(16*6,$out)); |
|
|
&movdqa ($twmask,&QWP(16*6,"esp")); # 0x0...010...87 |
|
|
|
|
|
&pxor ($twtmp,$twtmp); |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&pand ($twres,$twmask); # isolate carry and residue |
|
|
&pcmpgtd($twtmp,$tweak); # broadcast upper bits |
|
|
&pxor ($tweak,$twres); |
|
|
|
|
|
&sub ($len,16*6); |
|
|
&jnc (&label("xts_enc_loop6")); |
|
|
|
|
|
&mov ($rounds,&DWP(240,$key_)); # restore $rounds |
|
|
&mov ($key,$key_); # restore $key |
|
|
&mov ($rounds_,$rounds); |
|
|
|
|
|
&set_label("xts_enc_short"); |
|
|
&add ($len,16*6); |
|
|
&jz (&label("xts_enc_done6x")); |
|
|
|
|
|
&movdqa ($inout3,$tweak); # put aside previous tweak |
|
|
&cmp ($len,0x20); |
|
|
&jb (&label("xts_enc_one")); |
|
|
|
|
|
&pshufd ($twres,$twtmp,0x13); |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&pand ($twres,$twmask); # isolate carry and residue |
|
|
&pcmpgtd($twtmp,$tweak); # broadcast upper bits |
|
|
&pxor ($tweak,$twres); |
|
|
&je (&label("xts_enc_two")); |
|
|
|
|
|
&pshufd ($twres,$twtmp,0x13); |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&movdqa ($inout4,$tweak); # put aside previous tweak |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&pand ($twres,$twmask); # isolate carry and residue |
|
|
&pcmpgtd($twtmp,$tweak); # broadcast upper bits |
|
|
&pxor ($tweak,$twres); |
|
|
&cmp ($len,0x40); |
|
|
&jb (&label("xts_enc_three")); |
|
|
|
|
|
&pshufd ($twres,$twtmp,0x13); |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&movdqa ($inout5,$tweak); # put aside previous tweak |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&pand ($twres,$twmask); # isolate carry and residue |
|
|
&pcmpgtd($twtmp,$tweak); # broadcast upper bits |
|
|
&pxor ($tweak,$twres); |
|
|
&movdqa (&QWP(16*0,"esp"),$inout3); |
|
|
&movdqa (&QWP(16*1,"esp"),$inout4); |
|
|
&je (&label("xts_enc_four")); |
|
|
|
|
|
&movdqa (&QWP(16*2,"esp"),$inout5); |
|
|
&pshufd ($inout5,$twtmp,0x13); |
|
|
&movdqa (&QWP(16*3,"esp"),$tweak); |
|
|
&paddq ($tweak,$tweak); # &psllq($inout0,1); |
|
|
&pand ($inout5,$twmask); # isolate carry and residue |
|
|
&pxor ($inout5,$tweak); |
|
|
|
|
|
&movdqu ($inout0,&QWP(16*0,$inp)); # load input |
|
|
&movdqu ($inout1,&QWP(16*1,$inp)); |
|
|
&movdqu ($inout2,&QWP(16*2,$inp)); |
|
|
&pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak |
|
|
&movdqu ($inout3,&QWP(16*3,$inp)); |
|
|
&pxor ($inout1,&QWP(16*1,"esp")); |
|
|
&movdqu ($inout4,&QWP(16*4,$inp)); |
|
|
&pxor ($inout2,&QWP(16*2,"esp")); |
|
|
&lea ($inp,&DWP(16*5,$inp)); |
|
|
&pxor ($inout3,&QWP(16*3,"esp")); |
|
|
&movdqa (&QWP(16*4,"esp"),$inout5); # save last tweak |
|
|
&pxor ($inout4,$inout5); |
|
|
|
|
|
&call ("_aesni_encrypt6"); |
|
|
|
|
|
&movaps ($tweak,&QWP(16*4,"esp")); # last tweak |
|
|
&xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak |
|
|
&xorps ($inout1,&QWP(16*1,"esp")); |
|
|
&xorps ($inout2,&QWP(16*2,"esp")); |
|
|
&movups (&QWP(16*0,$out),$inout0); # write output |
|
|
&xorps ($inout3,&QWP(16*3,"esp")); |
|
|
&movups (&QWP(16*1,$out),$inout1); |
|
|
&xorps ($inout4,$tweak); |
|
|
&movups (&QWP(16*2,$out),$inout2); |
|
|
&movups (&QWP(16*3,$out),$inout3); |
|
|
&movups (&QWP(16*4,$out),$inout4); |
|
|
&lea ($out,&DWP(16*5,$out)); |
|
|
&jmp (&label("xts_enc_done")); |
|
|
|
|
|
&set_label("xts_enc_one",16); |
|
|
&movups ($inout0,&QWP(16*0,$inp)); # load input |
|
|
&lea ($inp,&DWP(16*1,$inp)); |
|
|
&xorps ($inout0,$inout3); # input^=tweak |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("enc"); } |
|
|
else |
|
|
{ &call ("_aesni_encrypt1"); } |
|
|
&xorps ($inout0,$inout3); # output^=tweak |
|
|
&movups (&QWP(16*0,$out),$inout0); # write output |
|
|
&lea ($out,&DWP(16*1,$out)); |
|
|
|
|
|
&movdqa ($tweak,$inout3); # last tweak |
|
|
&jmp (&label("xts_enc_done")); |
|
|
|
|
|
&set_label("xts_enc_two",16); |
|
|
&movaps ($inout4,$tweak); # put aside last tweak |
|
|
|
|
|
&movups ($inout0,&QWP(16*0,$inp)); # load input |
|
|
&movups ($inout1,&QWP(16*1,$inp)); |
|
|
&lea ($inp,&DWP(16*2,$inp)); |
|
|
&xorps ($inout0,$inout3); # input^=tweak |
|
|
&xorps ($inout1,$inout4); |
|
|
|
|
|
&call ("_aesni_encrypt2"); |
|
|
|
|
|
&xorps ($inout0,$inout3); # output^=tweak |
|
|
&xorps ($inout1,$inout4); |
|
|
&movups (&QWP(16*0,$out),$inout0); # write output |
|
|
&movups (&QWP(16*1,$out),$inout1); |
|
|
&lea ($out,&DWP(16*2,$out)); |
|
|
|
|
|
&movdqa ($tweak,$inout4); # last tweak |
|
|
&jmp (&label("xts_enc_done")); |
|
|
|
|
|
&set_label("xts_enc_three",16); |
|
|
&movaps ($inout5,$tweak); # put aside last tweak |
|
|
&movups ($inout0,&QWP(16*0,$inp)); # load input |
|
|
&movups ($inout1,&QWP(16*1,$inp)); |
|
|
&movups ($inout2,&QWP(16*2,$inp)); |
|
|
&lea ($inp,&DWP(16*3,$inp)); |
|
|
&xorps ($inout0,$inout3); # input^=tweak |
|
|
&xorps ($inout1,$inout4); |
|
|
&xorps ($inout2,$inout5); |
|
|
|
|
|
&call ("_aesni_encrypt3"); |
|
|
|
|
|
&xorps ($inout0,$inout3); # output^=tweak |
|
|
&xorps ($inout1,$inout4); |
|
|
&xorps ($inout2,$inout5); |
|
|
&movups (&QWP(16*0,$out),$inout0); # write output |
|
|
&movups (&QWP(16*1,$out),$inout1); |
|
|
&movups (&QWP(16*2,$out),$inout2); |
|
|
&lea ($out,&DWP(16*3,$out)); |
|
|
|
|
|
&movdqa ($tweak,$inout5); # last tweak |
|
|
&jmp (&label("xts_enc_done")); |
|
|
|
|
|
&set_label("xts_enc_four",16); |
|
|
&movaps ($inout4,$tweak); # put aside last tweak |
|
|
|
|
|
&movups ($inout0,&QWP(16*0,$inp)); # load input |
|
|
&movups ($inout1,&QWP(16*1,$inp)); |
|
|
&movups ($inout2,&QWP(16*2,$inp)); |
|
|
&xorps ($inout0,&QWP(16*0,"esp")); # input^=tweak |
|
|
&movups ($inout3,&QWP(16*3,$inp)); |
|
|
&lea ($inp,&DWP(16*4,$inp)); |
|
|
&xorps ($inout1,&QWP(16*1,"esp")); |
|
|
&xorps ($inout2,$inout5); |
|
|
&xorps ($inout3,$inout4); |
|
|
|
|
|
&call ("_aesni_encrypt4"); |
|
|
|
|
|
&xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak |
|
|
&xorps ($inout1,&QWP(16*1,"esp")); |
|
|
&xorps ($inout2,$inout5); |
|
|
&movups (&QWP(16*0,$out),$inout0); # write output |
|
|
&xorps ($inout3,$inout4); |
|
|
&movups (&QWP(16*1,$out),$inout1); |
|
|
&movups (&QWP(16*2,$out),$inout2); |
|
|
&movups (&QWP(16*3,$out),$inout3); |
|
|
&lea ($out,&DWP(16*4,$out)); |
|
|
|
|
|
&movdqa ($tweak,$inout4); # last tweak |
|
|
&jmp (&label("xts_enc_done")); |
|
|
|
|
|
&set_label("xts_enc_done6x",16); # $tweak is pre-calculated |
|
|
&mov ($len,&DWP(16*7+0,"esp")); # restore original $len |
|
|
&and ($len,15); |
|
|
&jz (&label("xts_enc_ret")); |
|
|
&movdqa ($inout3,$tweak); |
|
|
&mov (&DWP(16*7+0,"esp"),$len); # save $len%16 |
|
|
&jmp (&label("xts_enc_steal")); |
|
|
|
|
|
&set_label("xts_enc_done",16); |
|
|
&mov ($len,&DWP(16*7+0,"esp")); # restore original $len |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&and ($len,15); |
|
|
&jz (&label("xts_enc_ret")); |
|
|
|
|
|
&pcmpgtd($twtmp,$tweak); # broadcast upper bits |
|
|
&mov (&DWP(16*7+0,"esp"),$len); # save $len%16 |
|
|
&pshufd ($inout3,$twtmp,0x13); |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&pand ($inout3,&QWP(16*6,"esp")); # isolate carry and residue |
|
|
&pxor ($inout3,$tweak); |
|
|
|
|
|
&set_label("xts_enc_steal"); |
|
|
&movz ($rounds,&BP(0,$inp)); |
|
|
&movz ($key,&BP(-16,$out)); |
|
|
&lea ($inp,&DWP(1,$inp)); |
|
|
&mov (&BP(-16,$out),&LB($rounds)); |
|
|
&mov (&BP(0,$out),&LB($key)); |
|
|
&lea ($out,&DWP(1,$out)); |
|
|
&sub ($len,1); |
|
|
&jnz (&label("xts_enc_steal")); |
|
|
|
|
|
&sub ($out,&DWP(16*7+0,"esp")); # rewind $out |
|
|
&mov ($key,$key_); # restore $key |
|
|
&mov ($rounds,$rounds_); # restore $rounds |
|
|
|
|
|
&movups ($inout0,&QWP(-16,$out)); # load input |
|
|
&xorps ($inout0,$inout3); # input^=tweak |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("enc"); } |
|
|
else |
|
|
{ &call ("_aesni_encrypt1"); } |
|
|
&xorps ($inout0,$inout3); # output^=tweak |
|
|
&movups (&QWP(-16,$out),$inout0); # write output |
|
|
|
|
|
&set_label("xts_enc_ret"); |
|
|
&pxor ("xmm0","xmm0"); # clear register bank |
|
|
&pxor ("xmm1","xmm1"); |
|
|
&pxor ("xmm2","xmm2"); |
|
|
&movdqa (&QWP(16*0,"esp"),"xmm0"); # clear stack |
|
|
&pxor ("xmm3","xmm3"); |
|
|
&movdqa (&QWP(16*1,"esp"),"xmm0"); |
|
|
&pxor ("xmm4","xmm4"); |
|
|
&movdqa (&QWP(16*2,"esp"),"xmm0"); |
|
|
&pxor ("xmm5","xmm5"); |
|
|
&movdqa (&QWP(16*3,"esp"),"xmm0"); |
|
|
&pxor ("xmm6","xmm6"); |
|
|
&movdqa (&QWP(16*4,"esp"),"xmm0"); |
|
|
&pxor ("xmm7","xmm7"); |
|
|
&movdqa (&QWP(16*5,"esp"),"xmm0"); |
|
|
&mov ("esp",&DWP(16*7+4,"esp")); # restore %esp |
|
|
&function_end("${PREFIX}_xts_encrypt"); |
|
|
|
|
|
&function_begin("${PREFIX}_xts_decrypt"); |
|
|
&mov ($key,&wparam(4)); # key2 |
|
|
&mov ($inp,&wparam(5)); # clear-text tweak |
|
|
|
|
|
&mov ($rounds,&DWP(240,$key)); # key2->rounds |
|
|
&movups ($inout0,&QWP(0,$inp)); |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("enc"); } |
|
|
else |
|
|
{ &call ("_aesni_encrypt1"); } |
|
|
|
|
|
&mov ($inp,&wparam(0)); |
|
|
&mov ($out,&wparam(1)); |
|
|
&mov ($len,&wparam(2)); |
|
|
&mov ($key,&wparam(3)); # key1 |
|
|
|
|
|
&mov ($key_,"esp"); |
|
|
&sub ("esp",16*7+8); |
|
|
&and ("esp",-16); # align stack |
|
|
|
|
|
&xor ($rounds_,$rounds_); # if(len%16) len-=16; |
|
|
&test ($len,15); |
|
|
&setnz (&LB($rounds_)); |
|
|
&shl ($rounds_,4); |
|
|
&sub ($len,$rounds_); |
|
|
|
|
|
&mov (&DWP(16*6+0,"esp"),0x87); # compose the magic constant |
|
|
&mov (&DWP(16*6+4,"esp"),0); |
|
|
&mov (&DWP(16*6+8,"esp"),1); |
|
|
&mov (&DWP(16*6+12,"esp"),0); |
|
|
&mov (&DWP(16*7+0,"esp"),$len); # save original $len |
|
|
&mov (&DWP(16*7+4,"esp"),$key_); # save original %esp |
|
|
|
|
|
&mov ($rounds,&DWP(240,$key)); # key1->rounds |
|
|
&mov ($key_,$key); # backup $key |
|
|
&mov ($rounds_,$rounds); # backup $rounds |
|
|
|
|
|
&movdqa ($tweak,$inout0); |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&movdqa ($twmask,&QWP(6*16,"esp")); # 0x0...010...87 |
|
|
&pcmpgtd($twtmp,$tweak); # broadcast upper bits |
|
|
|
|
|
&and ($len,-16); |
|
|
&sub ($len,16*6); |
|
|
&jc (&label("xts_dec_short")); |
|
|
|
|
|
&shl ($rounds,4); |
|
|
&mov ($rounds_,16); |
|
|
&sub ($rounds_,$rounds); |
|
|
&lea ($key,&DWP(32,$key,$rounds)); |
|
|
&jmp (&label("xts_dec_loop6")); |
|
|
|
|
|
&set_label("xts_dec_loop6",16); |
|
|
for ($i=0;$i<4;$i++) { |
|
|
&pshufd ($twres,$twtmp,0x13); |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&movdqa (&QWP(16*$i,"esp"),$tweak); |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&pand ($twres,$twmask); # isolate carry and residue |
|
|
&pcmpgtd ($twtmp,$tweak); # broadcast upper bits |
|
|
&pxor ($tweak,$twres); |
|
|
} |
|
|
&pshufd ($inout5,$twtmp,0x13); |
|
|
&movdqa (&QWP(16*$i++,"esp"),$tweak); |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&$movekey ($rndkey0,&QWP(0,$key_)); |
|
|
&pand ($inout5,$twmask); # isolate carry and residue |
|
|
&movups ($inout0,&QWP(0,$inp)); # load input |
|
|
&pxor ($inout5,$tweak); |
|
|
|
|
|
# inline _aesni_encrypt6 prologue and flip xor with tweak and key[0] |
|
|
&mov ($rounds,$rounds_); |
|
|
&movdqu ($inout1,&QWP(16*1,$inp)); |
|
|
&xorps ($inout0,$rndkey0); # input^=rndkey[0] |
|
|
&movdqu ($inout2,&QWP(16*2,$inp)); |
|
|
&pxor ($inout1,$rndkey0); |
|
|
&movdqu ($inout3,&QWP(16*3,$inp)); |
|
|
&pxor ($inout2,$rndkey0); |
|
|
&movdqu ($inout4,&QWP(16*4,$inp)); |
|
|
&pxor ($inout3,$rndkey0); |
|
|
&movdqu ($rndkey1,&QWP(16*5,$inp)); |
|
|
&pxor ($inout4,$rndkey0); |
|
|
&lea ($inp,&DWP(16*6,$inp)); |
|
|
&pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak |
|
|
&movdqa (&QWP(16*$i,"esp"),$inout5); # save last tweak |
|
|
&pxor ($inout5,$rndkey1); |
|
|
|
|
|
&$movekey ($rndkey1,&QWP(16,$key_)); |
|
|
&pxor ($inout1,&QWP(16*1,"esp")); |
|
|
&pxor ($inout2,&QWP(16*2,"esp")); |
|
|
&aesdec ($inout0,$rndkey1); |
|
|
&pxor ($inout3,&QWP(16*3,"esp")); |
|
|
&pxor ($inout4,&QWP(16*4,"esp")); |
|
|
&aesdec ($inout1,$rndkey1); |
|
|
&pxor ($inout5,$rndkey0); |
|
|
&$movekey ($rndkey0,&QWP(32,$key_)); |
|
|
&aesdec ($inout2,$rndkey1); |
|
|
&aesdec ($inout3,$rndkey1); |
|
|
&aesdec ($inout4,$rndkey1); |
|
|
&aesdec ($inout5,$rndkey1); |
|
|
&call (&label("_aesni_decrypt6_enter")); |
|
|
|
|
|
&movdqa ($tweak,&QWP(16*5,"esp")); # last tweak |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak |
|
|
&pcmpgtd ($twtmp,$tweak); # broadcast upper bits |
|
|
&xorps ($inout1,&QWP(16*1,"esp")); |
|
|
&movups (&QWP(16*0,$out),$inout0); # write output |
|
|
&xorps ($inout2,&QWP(16*2,"esp")); |
|
|
&movups (&QWP(16*1,$out),$inout1); |
|
|
&xorps ($inout3,&QWP(16*3,"esp")); |
|
|
&movups (&QWP(16*2,$out),$inout2); |
|
|
&xorps ($inout4,&QWP(16*4,"esp")); |
|
|
&movups (&QWP(16*3,$out),$inout3); |
|
|
&xorps ($inout5,$tweak); |
|
|
&movups (&QWP(16*4,$out),$inout4); |
|
|
&pshufd ($twres,$twtmp,0x13); |
|
|
&movups (&QWP(16*5,$out),$inout5); |
|
|
&lea ($out,&DWP(16*6,$out)); |
|
|
&movdqa ($twmask,&QWP(16*6,"esp")); # 0x0...010...87 |
|
|
|
|
|
&pxor ($twtmp,$twtmp); |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&pand ($twres,$twmask); # isolate carry and residue |
|
|
&pcmpgtd($twtmp,$tweak); # broadcast upper bits |
|
|
&pxor ($tweak,$twres); |
|
|
|
|
|
&sub ($len,16*6); |
|
|
&jnc (&label("xts_dec_loop6")); |
|
|
|
|
|
&mov ($rounds,&DWP(240,$key_)); # restore $rounds |
|
|
&mov ($key,$key_); # restore $key |
|
|
&mov ($rounds_,$rounds); |
|
|
|
|
|
&set_label("xts_dec_short"); |
|
|
&add ($len,16*6); |
|
|
&jz (&label("xts_dec_done6x")); |
|
|
|
|
|
&movdqa ($inout3,$tweak); # put aside previous tweak |
|
|
&cmp ($len,0x20); |
|
|
&jb (&label("xts_dec_one")); |
|
|
|
|
|
&pshufd ($twres,$twtmp,0x13); |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&pand ($twres,$twmask); # isolate carry and residue |
|
|
&pcmpgtd($twtmp,$tweak); # broadcast upper bits |
|
|
&pxor ($tweak,$twres); |
|
|
&je (&label("xts_dec_two")); |
|
|
|
|
|
&pshufd ($twres,$twtmp,0x13); |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&movdqa ($inout4,$tweak); # put aside previous tweak |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&pand ($twres,$twmask); # isolate carry and residue |
|
|
&pcmpgtd($twtmp,$tweak); # broadcast upper bits |
|
|
&pxor ($tweak,$twres); |
|
|
&cmp ($len,0x40); |
|
|
&jb (&label("xts_dec_three")); |
|
|
|
|
|
&pshufd ($twres,$twtmp,0x13); |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&movdqa ($inout5,$tweak); # put aside previous tweak |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&pand ($twres,$twmask); # isolate carry and residue |
|
|
&pcmpgtd($twtmp,$tweak); # broadcast upper bits |
|
|
&pxor ($tweak,$twres); |
|
|
&movdqa (&QWP(16*0,"esp"),$inout3); |
|
|
&movdqa (&QWP(16*1,"esp"),$inout4); |
|
|
&je (&label("xts_dec_four")); |
|
|
|
|
|
&movdqa (&QWP(16*2,"esp"),$inout5); |
|
|
&pshufd ($inout5,$twtmp,0x13); |
|
|
&movdqa (&QWP(16*3,"esp"),$tweak); |
|
|
&paddq ($tweak,$tweak); # &psllq($inout0,1); |
|
|
&pand ($inout5,$twmask); # isolate carry and residue |
|
|
&pxor ($inout5,$tweak); |
|
|
|
|
|
&movdqu ($inout0,&QWP(16*0,$inp)); # load input |
|
|
&movdqu ($inout1,&QWP(16*1,$inp)); |
|
|
&movdqu ($inout2,&QWP(16*2,$inp)); |
|
|
&pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak |
|
|
&movdqu ($inout3,&QWP(16*3,$inp)); |
|
|
&pxor ($inout1,&QWP(16*1,"esp")); |
|
|
&movdqu ($inout4,&QWP(16*4,$inp)); |
|
|
&pxor ($inout2,&QWP(16*2,"esp")); |
|
|
&lea ($inp,&DWP(16*5,$inp)); |
|
|
&pxor ($inout3,&QWP(16*3,"esp")); |
|
|
&movdqa (&QWP(16*4,"esp"),$inout5); # save last tweak |
|
|
&pxor ($inout4,$inout5); |
|
|
|
|
|
&call ("_aesni_decrypt6"); |
|
|
|
|
|
&movaps ($tweak,&QWP(16*4,"esp")); # last tweak |
|
|
&xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak |
|
|
&xorps ($inout1,&QWP(16*1,"esp")); |
|
|
&xorps ($inout2,&QWP(16*2,"esp")); |
|
|
&movups (&QWP(16*0,$out),$inout0); # write output |
|
|
&xorps ($inout3,&QWP(16*3,"esp")); |
|
|
&movups (&QWP(16*1,$out),$inout1); |
|
|
&xorps ($inout4,$tweak); |
|
|
&movups (&QWP(16*2,$out),$inout2); |
|
|
&movups (&QWP(16*3,$out),$inout3); |
|
|
&movups (&QWP(16*4,$out),$inout4); |
|
|
&lea ($out,&DWP(16*5,$out)); |
|
|
&jmp (&label("xts_dec_done")); |
|
|
|
|
|
&set_label("xts_dec_one",16); |
|
|
&movups ($inout0,&QWP(16*0,$inp)); # load input |
|
|
&lea ($inp,&DWP(16*1,$inp)); |
|
|
&xorps ($inout0,$inout3); # input^=tweak |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("dec"); } |
|
|
else |
|
|
{ &call ("_aesni_decrypt1"); } |
|
|
&xorps ($inout0,$inout3); # output^=tweak |
|
|
&movups (&QWP(16*0,$out),$inout0); # write output |
|
|
&lea ($out,&DWP(16*1,$out)); |
|
|
|
|
|
&movdqa ($tweak,$inout3); # last tweak |
|
|
&jmp (&label("xts_dec_done")); |
|
|
|
|
|
&set_label("xts_dec_two",16); |
|
|
&movaps ($inout4,$tweak); # put aside last tweak |
|
|
|
|
|
&movups ($inout0,&QWP(16*0,$inp)); # load input |
|
|
&movups ($inout1,&QWP(16*1,$inp)); |
|
|
&lea ($inp,&DWP(16*2,$inp)); |
|
|
&xorps ($inout0,$inout3); # input^=tweak |
|
|
&xorps ($inout1,$inout4); |
|
|
|
|
|
&call ("_aesni_decrypt2"); |
|
|
|
|
|
&xorps ($inout0,$inout3); # output^=tweak |
|
|
&xorps ($inout1,$inout4); |
|
|
&movups (&QWP(16*0,$out),$inout0); # write output |
|
|
&movups (&QWP(16*1,$out),$inout1); |
|
|
&lea ($out,&DWP(16*2,$out)); |
|
|
|
|
|
&movdqa ($tweak,$inout4); # last tweak |
|
|
&jmp (&label("xts_dec_done")); |
|
|
|
|
|
&set_label("xts_dec_three",16); |
|
|
&movaps ($inout5,$tweak); # put aside last tweak |
|
|
&movups ($inout0,&QWP(16*0,$inp)); # load input |
|
|
&movups ($inout1,&QWP(16*1,$inp)); |
|
|
&movups ($inout2,&QWP(16*2,$inp)); |
|
|
&lea ($inp,&DWP(16*3,$inp)); |
|
|
&xorps ($inout0,$inout3); # input^=tweak |
|
|
&xorps ($inout1,$inout4); |
|
|
&xorps ($inout2,$inout5); |
|
|
|
|
|
&call ("_aesni_decrypt3"); |
|
|
|
|
|
&xorps ($inout0,$inout3); # output^=tweak |
|
|
&xorps ($inout1,$inout4); |
|
|
&xorps ($inout2,$inout5); |
|
|
&movups (&QWP(16*0,$out),$inout0); # write output |
|
|
&movups (&QWP(16*1,$out),$inout1); |
|
|
&movups (&QWP(16*2,$out),$inout2); |
|
|
&lea ($out,&DWP(16*3,$out)); |
|
|
|
|
|
&movdqa ($tweak,$inout5); # last tweak |
|
|
&jmp (&label("xts_dec_done")); |
|
|
|
|
|
&set_label("xts_dec_four",16); |
|
|
&movaps ($inout4,$tweak); # put aside last tweak |
|
|
|
|
|
&movups ($inout0,&QWP(16*0,$inp)); # load input |
|
|
&movups ($inout1,&QWP(16*1,$inp)); |
|
|
&movups ($inout2,&QWP(16*2,$inp)); |
|
|
&xorps ($inout0,&QWP(16*0,"esp")); # input^=tweak |
|
|
&movups ($inout3,&QWP(16*3,$inp)); |
|
|
&lea ($inp,&DWP(16*4,$inp)); |
|
|
&xorps ($inout1,&QWP(16*1,"esp")); |
|
|
&xorps ($inout2,$inout5); |
|
|
&xorps ($inout3,$inout4); |
|
|
|
|
|
&call ("_aesni_decrypt4"); |
|
|
|
|
|
&xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak |
|
|
&xorps ($inout1,&QWP(16*1,"esp")); |
|
|
&xorps ($inout2,$inout5); |
|
|
&movups (&QWP(16*0,$out),$inout0); # write output |
|
|
&xorps ($inout3,$inout4); |
|
|
&movups (&QWP(16*1,$out),$inout1); |
|
|
&movups (&QWP(16*2,$out),$inout2); |
|
|
&movups (&QWP(16*3,$out),$inout3); |
|
|
&lea ($out,&DWP(16*4,$out)); |
|
|
|
|
|
&movdqa ($tweak,$inout4); # last tweak |
|
|
&jmp (&label("xts_dec_done")); |
|
|
|
|
|
&set_label("xts_dec_done6x",16); # $tweak is pre-calculated |
|
|
&mov ($len,&DWP(16*7+0,"esp")); # restore original $len |
|
|
&and ($len,15); |
|
|
&jz (&label("xts_dec_ret")); |
|
|
&mov (&DWP(16*7+0,"esp"),$len); # save $len%16 |
|
|
&jmp (&label("xts_dec_only_one_more")); |
|
|
|
|
|
&set_label("xts_dec_done",16); |
|
|
&mov ($len,&DWP(16*7+0,"esp")); # restore original $len |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&and ($len,15); |
|
|
&jz (&label("xts_dec_ret")); |
|
|
|
|
|
&pcmpgtd($twtmp,$tweak); # broadcast upper bits |
|
|
&mov (&DWP(16*7+0,"esp"),$len); # save $len%16 |
|
|
&pshufd ($twres,$twtmp,0x13); |
|
|
&pxor ($twtmp,$twtmp); |
|
|
&movdqa ($twmask,&QWP(16*6,"esp")); |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&pand ($twres,$twmask); # isolate carry and residue |
|
|
&pcmpgtd($twtmp,$tweak); # broadcast upper bits |
|
|
&pxor ($tweak,$twres); |
|
|
|
|
|
&set_label("xts_dec_only_one_more"); |
|
|
&pshufd ($inout3,$twtmp,0x13); |
|
|
&movdqa ($inout4,$tweak); # put aside previous tweak |
|
|
&paddq ($tweak,$tweak); # &psllq($tweak,1); |
|
|
&pand ($inout3,$twmask); # isolate carry and residue |
|
|
&pxor ($inout3,$tweak); |
|
|
|
|
|
&mov ($key,$key_); # restore $key |
|
|
&mov ($rounds,$rounds_); # restore $rounds |
|
|
|
|
|
&movups ($inout0,&QWP(0,$inp)); # load input |
|
|
&xorps ($inout0,$inout3); # input^=tweak |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("dec"); } |
|
|
else |
|
|
{ &call ("_aesni_decrypt1"); } |
|
|
&xorps ($inout0,$inout3); # output^=tweak |
|
|
&movups (&QWP(0,$out),$inout0); # write output |
|
|
|
|
|
&set_label("xts_dec_steal"); |
|
|
&movz ($rounds,&BP(16,$inp)); |
|
|
&movz ($key,&BP(0,$out)); |
|
|
&lea ($inp,&DWP(1,$inp)); |
|
|
&mov (&BP(0,$out),&LB($rounds)); |
|
|
&mov (&BP(16,$out),&LB($key)); |
|
|
&lea ($out,&DWP(1,$out)); |
|
|
&sub ($len,1); |
|
|
&jnz (&label("xts_dec_steal")); |
|
|
|
|
|
&sub ($out,&DWP(16*7+0,"esp")); # rewind $out |
|
|
&mov ($key,$key_); # restore $key |
|
|
&mov ($rounds,$rounds_); # restore $rounds |
|
|
|
|
|
&movups ($inout0,&QWP(0,$out)); # load input |
|
|
&xorps ($inout0,$inout4); # input^=tweak |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("dec"); } |
|
|
else |
|
|
{ &call ("_aesni_decrypt1"); } |
|
|
&xorps ($inout0,$inout4); # output^=tweak |
|
|
&movups (&QWP(0,$out),$inout0); # write output |
|
|
|
|
|
&set_label("xts_dec_ret"); |
|
|
&pxor ("xmm0","xmm0"); # clear register bank |
|
|
&pxor ("xmm1","xmm1"); |
|
|
&pxor ("xmm2","xmm2"); |
|
|
&movdqa (&QWP(16*0,"esp"),"xmm0"); # clear stack |
|
|
&pxor ("xmm3","xmm3"); |
|
|
&movdqa (&QWP(16*1,"esp"),"xmm0"); |
|
|
&pxor ("xmm4","xmm4"); |
|
|
&movdqa (&QWP(16*2,"esp"),"xmm0"); |
|
|
&pxor ("xmm5","xmm5"); |
|
|
&movdqa (&QWP(16*3,"esp"),"xmm0"); |
|
|
&pxor ("xmm6","xmm6"); |
|
|
&movdqa (&QWP(16*4,"esp"),"xmm0"); |
|
|
&pxor ("xmm7","xmm7"); |
|
|
&movdqa (&QWP(16*5,"esp"),"xmm0"); |
|
|
&mov ("esp",&DWP(16*7+4,"esp")); # restore %esp |
|
|
&function_end("${PREFIX}_xts_decrypt"); |
|
|
} |
|
|
} |
|
|
|
|
|
###################################################################### |
|
|
# void $PREFIX_cbc_encrypt (const void *inp, void *out, |
|
|
# size_t length, const AES_KEY *key, |
|
|
# unsigned char *ivp,const int enc); |
|
|
&function_begin("${PREFIX}_cbc_encrypt"); |
|
|
&mov ($inp,&wparam(0)); |
|
|
&mov ($rounds_,"esp"); |
|
|
&mov ($out,&wparam(1)); |
|
|
&sub ($rounds_,24); |
|
|
&mov ($len,&wparam(2)); |
|
|
&and ($rounds_,-16); |
|
|
&mov ($key,&wparam(3)); |
|
|
&mov ($key_,&wparam(4)); |
|
|
&test ($len,$len); |
|
|
&jz (&label("cbc_abort")); |
|
|
|
|
|
&cmp (&wparam(5),0); |
|
|
&xchg ($rounds_,"esp"); # alloca |
|
|
&movups ($ivec,&QWP(0,$key_)); # load IV |
|
|
&mov ($rounds,&DWP(240,$key)); |
|
|
&mov ($key_,$key); # backup $key |
|
|
&mov (&DWP(16,"esp"),$rounds_); # save original %esp |
|
|
&mov ($rounds_,$rounds); # backup $rounds |
|
|
&je (&label("cbc_decrypt")); |
|
|
|
|
|
&movaps ($inout0,$ivec); |
|
|
&cmp ($len,16); |
|
|
&jb (&label("cbc_enc_tail")); |
|
|
&sub ($len,16); |
|
|
&jmp (&label("cbc_enc_loop")); |
|
|
|
|
|
&set_label("cbc_enc_loop",16); |
|
|
&movups ($ivec,&QWP(0,$inp)); # input actually |
|
|
&lea ($inp,&DWP(16,$inp)); |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("enc",$inout0,$ivec); } |
|
|
else |
|
|
{ &xorps($inout0,$ivec); &call("_aesni_encrypt1"); } |
|
|
&mov ($rounds,$rounds_); # restore $rounds |
|
|
&mov ($key,$key_); # restore $key |
|
|
&movups (&QWP(0,$out),$inout0); # store output |
|
|
&lea ($out,&DWP(16,$out)); |
|
|
&sub ($len,16); |
|
|
&jnc (&label("cbc_enc_loop")); |
|
|
&add ($len,16); |
|
|
&jnz (&label("cbc_enc_tail")); |
|
|
&movaps ($ivec,$inout0); |
|
|
&pxor ($inout0,$inout0); |
|
|
&jmp (&label("cbc_ret")); |
|
|
|
|
|
&set_label("cbc_enc_tail"); |
|
|
&mov ("ecx",$len); # zaps $rounds |
|
|
&data_word(0xA4F3F689); # rep movsb |
|
|
&mov ("ecx",16); # zero tail |
|
|
&sub ("ecx",$len); |
|
|
&xor ("eax","eax"); # zaps $len |
|
|
&data_word(0xAAF3F689); # rep stosb |
|
|
&lea ($out,&DWP(-16,$out)); # rewind $out by 1 block |
|
|
&mov ($rounds,$rounds_); # restore $rounds |
|
|
&mov ($inp,$out); # $inp and $out are the same |
|
|
&mov ($key,$key_); # restore $key |
|
|
&jmp (&label("cbc_enc_loop")); |
|
|
###################################################################### |
|
|
&set_label("cbc_decrypt",16); |
|
|
&cmp ($len,0x50); |
|
|
&jbe (&label("cbc_dec_tail")); |
|
|
&movaps (&QWP(0,"esp"),$ivec); # save IV |
|
|
&sub ($len,0x50); |
|
|
&jmp (&label("cbc_dec_loop6_enter")); |
|
|
|
|
|
&set_label("cbc_dec_loop6",16); |
|
|
&movaps (&QWP(0,"esp"),$rndkey0); # save IV |
|
|
&movups (&QWP(0,$out),$inout5); |
|
|
&lea ($out,&DWP(0x10,$out)); |
|
|
&set_label("cbc_dec_loop6_enter"); |
|
|
&movdqu ($inout0,&QWP(0,$inp)); |
|
|
&movdqu ($inout1,&QWP(0x10,$inp)); |
|
|
&movdqu ($inout2,&QWP(0x20,$inp)); |
|
|
&movdqu ($inout3,&QWP(0x30,$inp)); |
|
|
&movdqu ($inout4,&QWP(0x40,$inp)); |
|
|
&movdqu ($inout5,&QWP(0x50,$inp)); |
|
|
|
|
|
&call ("_aesni_decrypt6"); |
|
|
|
|
|
&movups ($rndkey1,&QWP(0,$inp)); |
|
|
&movups ($rndkey0,&QWP(0x10,$inp)); |
|
|
&xorps ($inout0,&QWP(0,"esp")); # ^=IV |
|
|
&xorps ($inout1,$rndkey1); |
|
|
&movups ($rndkey1,&QWP(0x20,$inp)); |
|
|
&xorps ($inout2,$rndkey0); |
|
|
&movups ($rndkey0,&QWP(0x30,$inp)); |
|
|
&xorps ($inout3,$rndkey1); |
|
|
&movups ($rndkey1,&QWP(0x40,$inp)); |
|
|
&xorps ($inout4,$rndkey0); |
|
|
&movups ($rndkey0,&QWP(0x50,$inp)); # IV |
|
|
&xorps ($inout5,$rndkey1); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&lea ($inp,&DWP(0x60,$inp)); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&mov ($rounds,$rounds_); # restore $rounds |
|
|
&movups (&QWP(0x30,$out),$inout3); |
|
|
&mov ($key,$key_); # restore $key |
|
|
&movups (&QWP(0x40,$out),$inout4); |
|
|
&lea ($out,&DWP(0x50,$out)); |
|
|
&sub ($len,0x60); |
|
|
&ja (&label("cbc_dec_loop6")); |
|
|
|
|
|
&movaps ($inout0,$inout5); |
|
|
&movaps ($ivec,$rndkey0); |
|
|
&add ($len,0x50); |
|
|
&jle (&label("cbc_dec_clear_tail_collected")); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&lea ($out,&DWP(0x10,$out)); |
|
|
&set_label("cbc_dec_tail"); |
|
|
&movups ($inout0,&QWP(0,$inp)); |
|
|
&movaps ($in0,$inout0); |
|
|
&cmp ($len,0x10); |
|
|
&jbe (&label("cbc_dec_one")); |
|
|
|
|
|
&movups ($inout1,&QWP(0x10,$inp)); |
|
|
&movaps ($in1,$inout1); |
|
|
&cmp ($len,0x20); |
|
|
&jbe (&label("cbc_dec_two")); |
|
|
|
|
|
&movups ($inout2,&QWP(0x20,$inp)); |
|
|
&cmp ($len,0x30); |
|
|
&jbe (&label("cbc_dec_three")); |
|
|
|
|
|
&movups ($inout3,&QWP(0x30,$inp)); |
|
|
&cmp ($len,0x40); |
|
|
&jbe (&label("cbc_dec_four")); |
|
|
|
|
|
&movups ($inout4,&QWP(0x40,$inp)); |
|
|
&movaps (&QWP(0,"esp"),$ivec); # save IV |
|
|
&movups ($inout0,&QWP(0,$inp)); |
|
|
&xorps ($inout5,$inout5); |
|
|
&call ("_aesni_decrypt6"); |
|
|
&movups ($rndkey1,&QWP(0,$inp)); |
|
|
&movups ($rndkey0,&QWP(0x10,$inp)); |
|
|
&xorps ($inout0,&QWP(0,"esp")); # ^= IV |
|
|
&xorps ($inout1,$rndkey1); |
|
|
&movups ($rndkey1,&QWP(0x20,$inp)); |
|
|
&xorps ($inout2,$rndkey0); |
|
|
&movups ($rndkey0,&QWP(0x30,$inp)); |
|
|
&xorps ($inout3,$rndkey1); |
|
|
&movups ($ivec,&QWP(0x40,$inp)); # IV |
|
|
&xorps ($inout4,$rndkey0); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&pxor ($inout1,$inout1); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&pxor ($inout2,$inout2); |
|
|
&movups (&QWP(0x30,$out),$inout3); |
|
|
&pxor ($inout3,$inout3); |
|
|
&lea ($out,&DWP(0x40,$out)); |
|
|
&movaps ($inout0,$inout4); |
|
|
&pxor ($inout4,$inout4); |
|
|
&sub ($len,0x50); |
|
|
&jmp (&label("cbc_dec_tail_collected")); |
|
|
|
|
|
&set_label("cbc_dec_one",16); |
|
|
if ($inline) |
|
|
{ &aesni_inline_generate1("dec"); } |
|
|
else |
|
|
{ &call ("_aesni_decrypt1"); } |
|
|
&xorps ($inout0,$ivec); |
|
|
&movaps ($ivec,$in0); |
|
|
&sub ($len,0x10); |
|
|
&jmp (&label("cbc_dec_tail_collected")); |
|
|
|
|
|
&set_label("cbc_dec_two",16); |
|
|
&call ("_aesni_decrypt2"); |
|
|
&xorps ($inout0,$ivec); |
|
|
&xorps ($inout1,$in0); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movaps ($inout0,$inout1); |
|
|
&pxor ($inout1,$inout1); |
|
|
&lea ($out,&DWP(0x10,$out)); |
|
|
&movaps ($ivec,$in1); |
|
|
&sub ($len,0x20); |
|
|
&jmp (&label("cbc_dec_tail_collected")); |
|
|
|
|
|
&set_label("cbc_dec_three",16); |
|
|
&call ("_aesni_decrypt3"); |
|
|
&xorps ($inout0,$ivec); |
|
|
&xorps ($inout1,$in0); |
|
|
&xorps ($inout2,$in1); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&movaps ($inout0,$inout2); |
|
|
&pxor ($inout2,$inout2); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&pxor ($inout1,$inout1); |
|
|
&lea ($out,&DWP(0x20,$out)); |
|
|
&movups ($ivec,&QWP(0x20,$inp)); |
|
|
&sub ($len,0x30); |
|
|
&jmp (&label("cbc_dec_tail_collected")); |
|
|
|
|
|
&set_label("cbc_dec_four",16); |
|
|
&call ("_aesni_decrypt4"); |
|
|
&movups ($rndkey1,&QWP(0x10,$inp)); |
|
|
&movups ($rndkey0,&QWP(0x20,$inp)); |
|
|
&xorps ($inout0,$ivec); |
|
|
&movups ($ivec,&QWP(0x30,$inp)); |
|
|
&xorps ($inout1,$in0); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&xorps ($inout2,$rndkey1); |
|
|
&movups (&QWP(0x10,$out),$inout1); |
|
|
&pxor ($inout1,$inout1); |
|
|
&xorps ($inout3,$rndkey0); |
|
|
&movups (&QWP(0x20,$out),$inout2); |
|
|
&pxor ($inout2,$inout2); |
|
|
&lea ($out,&DWP(0x30,$out)); |
|
|
&movaps ($inout0,$inout3); |
|
|
&pxor ($inout3,$inout3); |
|
|
&sub ($len,0x40); |
|
|
&jmp (&label("cbc_dec_tail_collected")); |
|
|
|
|
|
&set_label("cbc_dec_clear_tail_collected",16); |
|
|
&pxor ($inout1,$inout1); |
|
|
&pxor ($inout2,$inout2); |
|
|
&pxor ($inout3,$inout3); |
|
|
&pxor ($inout4,$inout4); |
|
|
&set_label("cbc_dec_tail_collected"); |
|
|
&and ($len,15); |
|
|
&jnz (&label("cbc_dec_tail_partial")); |
|
|
&movups (&QWP(0,$out),$inout0); |
|
|
&pxor ($rndkey0,$rndkey0); |
|
|
&jmp (&label("cbc_ret")); |
|
|
|
|
|
&set_label("cbc_dec_tail_partial",16); |
|
|
&movaps (&QWP(0,"esp"),$inout0); |
|
|
&pxor ($rndkey0,$rndkey0); |
|
|
&mov ("ecx",16); |
|
|
&mov ($inp,"esp"); |
|
|
&sub ("ecx",$len); |
|
|
&data_word(0xA4F3F689); # rep movsb |
|
|
&movdqa (&QWP(0,"esp"),$inout0); |
|
|
|
|
|
&set_label("cbc_ret"); |
|
|
&mov ("esp",&DWP(16,"esp")); # pull original %esp |
|
|
&mov ($key_,&wparam(4)); |
|
|
&pxor ($inout0,$inout0); |
|
|
&pxor ($rndkey1,$rndkey1); |
|
|
&movups (&QWP(0,$key_),$ivec); # output IV |
|
|
&pxor ($ivec,$ivec); |
|
|
&set_label("cbc_abort"); |
|
|
&function_end("${PREFIX}_cbc_encrypt"); |
|
|
|
|
|
###################################################################### |
|
|
# Mechanical port from aesni-x86_64.pl. |
|
|
# |
|
|
# _aesni_set_encrypt_key is private interface, |
|
|
# input: |
|
|
# "eax" const unsigned char *userKey |
|
|
# $rounds int bits |
|
|
# $key AES_KEY *key |
|
|
# output: |
|
|
# "eax" return code |
|
|
# $round rounds |
|
|
|
|
|
&function_begin_B("_aesni_set_encrypt_key"); |
|
|
&push ("ebp"); |
|
|
&push ("ebx"); |
|
|
&test ("eax","eax"); |
|
|
&jz (&label("bad_pointer")); |
|
|
&test ($key,$key); |
|
|
&jz (&label("bad_pointer")); |
|
|
|
|
|
&call (&label("pic")); |
|
|
&set_label("pic"); |
|
|
&blindpop("ebx"); |
|
|
&lea ("ebx",&DWP(&label("key_const")."-".&label("pic"),"ebx")); |
|
|
|
|
|
&picmeup("ebp","OPENSSL_ia32cap_P","ebx",&label("key_const")); |
|
|
&movups ("xmm0",&QWP(0,"eax")); # pull first 128 bits of *userKey |
|
|
&xorps ("xmm4","xmm4"); # low dword of xmm4 is assumed 0 |
|
|
&mov ("ebp",&DWP(4,"ebp")); |
|
|
&lea ($key,&DWP(16,$key)); |
|
|
&and ("ebp",1<<28|1<<11); # AVX and XOP bits |
|
|
&cmp ($rounds,256); |
|
|
&je (&label("14rounds")); |
|
|
&cmp ($rounds,192); |
|
|
&je (&label("12rounds")); |
|
|
&cmp ($rounds,128); |
|
|
&jne (&label("bad_keybits")); |
|
|
|
|
|
&set_label("10rounds",16); |
|
|
&cmp ("ebp",1<<28); |
|
|
&je (&label("10rounds_alt")); |
|
|
|
|
|
&mov ($rounds,9); |
|
|
&$movekey (&QWP(-16,$key),"xmm0"); # round 0 |
|
|
&aeskeygenassist("xmm1","xmm0",0x01); # round 1 |
|
|
&call (&label("key_128_cold")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x2); # round 2 |
|
|
&call (&label("key_128")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x04); # round 3 |
|
|
&call (&label("key_128")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x08); # round 4 |
|
|
&call (&label("key_128")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x10); # round 5 |
|
|
&call (&label("key_128")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x20); # round 6 |
|
|
&call (&label("key_128")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x40); # round 7 |
|
|
&call (&label("key_128")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x80); # round 8 |
|
|
&call (&label("key_128")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x1b); # round 9 |
|
|
&call (&label("key_128")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x36); # round 10 |
|
|
&call (&label("key_128")); |
|
|
&$movekey (&QWP(0,$key),"xmm0"); |
|
|
&mov (&DWP(80,$key),$rounds); |
|
|
|
|
|
&jmp (&label("good_key")); |
|
|
|
|
|
&set_label("key_128",16); |
|
|
&$movekey (&QWP(0,$key),"xmm0"); |
|
|
&lea ($key,&DWP(16,$key)); |
|
|
&set_label("key_128_cold"); |
|
|
&shufps ("xmm4","xmm0",0b00010000); |
|
|
&xorps ("xmm0","xmm4"); |
|
|
&shufps ("xmm4","xmm0",0b10001100); |
|
|
&xorps ("xmm0","xmm4"); |
|
|
&shufps ("xmm1","xmm1",0b11111111); # critical path |
|
|
&xorps ("xmm0","xmm1"); |
|
|
&ret(); |
|
|
|
|
|
&set_label("10rounds_alt",16); |
|
|
&movdqa ("xmm5",&QWP(0x00,"ebx")); |
|
|
&mov ($rounds,8); |
|
|
&movdqa ("xmm4",&QWP(0x20,"ebx")); |
|
|
&movdqa ("xmm2","xmm0"); |
|
|
&movdqu (&QWP(-16,$key),"xmm0"); |
|
|
|
|
|
&set_label("loop_key128"); |
|
|
&pshufb ("xmm0","xmm5"); |
|
|
&aesenclast ("xmm0","xmm4"); |
|
|
&pslld ("xmm4",1); |
|
|
&lea ($key,&DWP(16,$key)); |
|
|
|
|
|
&movdqa ("xmm3","xmm2"); |
|
|
&pslldq ("xmm2",4); |
|
|
&pxor ("xmm3","xmm2"); |
|
|
&pslldq ("xmm2",4); |
|
|
&pxor ("xmm3","xmm2"); |
|
|
&pslldq ("xmm2",4); |
|
|
&pxor ("xmm2","xmm3"); |
|
|
|
|
|
&pxor ("xmm0","xmm2"); |
|
|
&movdqu (&QWP(-16,$key),"xmm0"); |
|
|
&movdqa ("xmm2","xmm0"); |
|
|
|
|
|
&dec ($rounds); |
|
|
&jnz (&label("loop_key128")); |
|
|
|
|
|
&movdqa ("xmm4",&QWP(0x30,"ebx")); |
|
|
|
|
|
&pshufb ("xmm0","xmm5"); |
|
|
&aesenclast ("xmm0","xmm4"); |
|
|
&pslld ("xmm4",1); |
|
|
|
|
|
&movdqa ("xmm3","xmm2"); |
|
|
&pslldq ("xmm2",4); |
|
|
&pxor ("xmm3","xmm2"); |
|
|
&pslldq ("xmm2",4); |
|
|
&pxor ("xmm3","xmm2"); |
|
|
&pslldq ("xmm2",4); |
|
|
&pxor ("xmm2","xmm3"); |
|
|
|
|
|
&pxor ("xmm0","xmm2"); |
|
|
&movdqu (&QWP(0,$key),"xmm0"); |
|
|
|
|
|
&movdqa ("xmm2","xmm0"); |
|
|
&pshufb ("xmm0","xmm5"); |
|
|
&aesenclast ("xmm0","xmm4"); |
|
|
|
|
|
&movdqa ("xmm3","xmm2"); |
|
|
&pslldq ("xmm2",4); |
|
|
&pxor ("xmm3","xmm2"); |
|
|
&pslldq ("xmm2",4); |
|
|
&pxor ("xmm3","xmm2"); |
|
|
&pslldq ("xmm2",4); |
|
|
&pxor ("xmm2","xmm3"); |
|
|
|
|
|
&pxor ("xmm0","xmm2"); |
|
|
&movdqu (&QWP(16,$key),"xmm0"); |
|
|
|
|
|
&mov ($rounds,9); |
|
|
&mov (&DWP(96,$key),$rounds); |
|
|
|
|
|
&jmp (&label("good_key")); |
|
|
|
|
|
&set_label("12rounds",16); |
|
|
&movq ("xmm2",&QWP(16,"eax")); # remaining 1/3 of *userKey |
|
|
&cmp ("ebp",1<<28); |
|
|
&je (&label("12rounds_alt")); |
|
|
|
|
|
&mov ($rounds,11); |
|
|
&$movekey (&QWP(-16,$key),"xmm0"); # round 0 |
|
|
&aeskeygenassist("xmm1","xmm2",0x01); # round 1,2 |
|
|
&call (&label("key_192a_cold")); |
|
|
&aeskeygenassist("xmm1","xmm2",0x02); # round 2,3 |
|
|
&call (&label("key_192b")); |
|
|
&aeskeygenassist("xmm1","xmm2",0x04); # round 4,5 |
|
|
&call (&label("key_192a")); |
|
|
&aeskeygenassist("xmm1","xmm2",0x08); # round 5,6 |
|
|
&call (&label("key_192b")); |
|
|
&aeskeygenassist("xmm1","xmm2",0x10); # round 7,8 |
|
|
&call (&label("key_192a")); |
|
|
&aeskeygenassist("xmm1","xmm2",0x20); # round 8,9 |
|
|
&call (&label("key_192b")); |
|
|
&aeskeygenassist("xmm1","xmm2",0x40); # round 10,11 |
|
|
&call (&label("key_192a")); |
|
|
&aeskeygenassist("xmm1","xmm2",0x80); # round 11,12 |
|
|
&call (&label("key_192b")); |
|
|
&$movekey (&QWP(0,$key),"xmm0"); |
|
|
&mov (&DWP(48,$key),$rounds); |
|
|
|
|
|
&jmp (&label("good_key")); |
|
|
|
|
|
&set_label("key_192a",16); |
|
|
&$movekey (&QWP(0,$key),"xmm0"); |
|
|
&lea ($key,&DWP(16,$key)); |
|
|
&set_label("key_192a_cold",16); |
|
|
&movaps ("xmm5","xmm2"); |
|
|
&set_label("key_192b_warm"); |
|
|
&shufps ("xmm4","xmm0",0b00010000); |
|
|
&movdqa ("xmm3","xmm2"); |
|
|
&xorps ("xmm0","xmm4"); |
|
|
&shufps ("xmm4","xmm0",0b10001100); |
|
|
&pslldq ("xmm3",4); |
|
|
&xorps ("xmm0","xmm4"); |
|
|
&pshufd ("xmm1","xmm1",0b01010101); # critical path |
|
|
&pxor ("xmm2","xmm3"); |
|
|
&pxor ("xmm0","xmm1"); |
|
|
&pshufd ("xmm3","xmm0",0b11111111); |
|
|
&pxor ("xmm2","xmm3"); |
|
|
&ret(); |
|
|
|
|
|
&set_label("key_192b",16); |
|
|
&movaps ("xmm3","xmm0"); |
|
|
&shufps ("xmm5","xmm0",0b01000100); |
|
|
&$movekey (&QWP(0,$key),"xmm5"); |
|
|
&shufps ("xmm3","xmm2",0b01001110); |
|
|
&$movekey (&QWP(16,$key),"xmm3"); |
|
|
&lea ($key,&DWP(32,$key)); |
|
|
&jmp (&label("key_192b_warm")); |
|
|
|
|
|
&set_label("12rounds_alt",16); |
|
|
&movdqa ("xmm5",&QWP(0x10,"ebx")); |
|
|
&movdqa ("xmm4",&QWP(0x20,"ebx")); |
|
|
&mov ($rounds,8); |
|
|
&movdqu (&QWP(-16,$key),"xmm0"); |
|
|
|
|
|
&set_label("loop_key192"); |
|
|
&movq (&QWP(0,$key),"xmm2"); |
|
|
&movdqa ("xmm1","xmm2"); |
|
|
&pshufb ("xmm2","xmm5"); |
|
|
&aesenclast ("xmm2","xmm4"); |
|
|
&pslld ("xmm4",1); |
|
|
&lea ($key,&DWP(24,$key)); |
|
|
|
|
|
&movdqa ("xmm3","xmm0"); |
|
|
&pslldq ("xmm0",4); |
|
|
&pxor ("xmm3","xmm0"); |
|
|
&pslldq ("xmm0",4); |
|
|
&pxor ("xmm3","xmm0"); |
|
|
&pslldq ("xmm0",4); |
|
|
&pxor ("xmm0","xmm3"); |
|
|
|
|
|
&pshufd ("xmm3","xmm0",0xff); |
|
|
&pxor ("xmm3","xmm1"); |
|
|
&pslldq ("xmm1",4); |
|
|
&pxor ("xmm3","xmm1"); |
|
|
|
|
|
&pxor ("xmm0","xmm2"); |
|
|
&pxor ("xmm2","xmm3"); |
|
|
&movdqu (&QWP(-16,$key),"xmm0"); |
|
|
|
|
|
&dec ($rounds); |
|
|
&jnz (&label("loop_key192")); |
|
|
|
|
|
&mov ($rounds,11); |
|
|
&mov (&DWP(32,$key),$rounds); |
|
|
|
|
|
&jmp (&label("good_key")); |
|
|
|
|
|
&set_label("14rounds",16); |
|
|
&movups ("xmm2",&QWP(16,"eax")); # remaining half of *userKey |
|
|
&lea ($key,&DWP(16,$key)); |
|
|
&cmp ("ebp",1<<28); |
|
|
&je (&label("14rounds_alt")); |
|
|
|
|
|
&mov ($rounds,13); |
|
|
&$movekey (&QWP(-32,$key),"xmm0"); # round 0 |
|
|
&$movekey (&QWP(-16,$key),"xmm2"); # round 1 |
|
|
&aeskeygenassist("xmm1","xmm2",0x01); # round 2 |
|
|
&call (&label("key_256a_cold")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x01); # round 3 |
|
|
&call (&label("key_256b")); |
|
|
&aeskeygenassist("xmm1","xmm2",0x02); # round 4 |
|
|
&call (&label("key_256a")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x02); # round 5 |
|
|
&call (&label("key_256b")); |
|
|
&aeskeygenassist("xmm1","xmm2",0x04); # round 6 |
|
|
&call (&label("key_256a")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x04); # round 7 |
|
|
&call (&label("key_256b")); |
|
|
&aeskeygenassist("xmm1","xmm2",0x08); # round 8 |
|
|
&call (&label("key_256a")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x08); # round 9 |
|
|
&call (&label("key_256b")); |
|
|
&aeskeygenassist("xmm1","xmm2",0x10); # round 10 |
|
|
&call (&label("key_256a")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x10); # round 11 |
|
|
&call (&label("key_256b")); |
|
|
&aeskeygenassist("xmm1","xmm2",0x20); # round 12 |
|
|
&call (&label("key_256a")); |
|
|
&aeskeygenassist("xmm1","xmm0",0x20); # round 13 |
|
|
&call (&label("key_256b")); |
|
|
&aeskeygenassist("xmm1","xmm2",0x40); # round 14 |
|
|
&call (&label("key_256a")); |
|
|
&$movekey (&QWP(0,$key),"xmm0"); |
|
|
&mov (&DWP(16,$key),$rounds); |
|
|
&xor ("eax","eax"); |
|
|
|
|
|
&jmp (&label("good_key")); |
|
|
|
|
|
&set_label("key_256a",16); |
|
|
&$movekey (&QWP(0,$key),"xmm2"); |
|
|
&lea ($key,&DWP(16,$key)); |
|
|
&set_label("key_256a_cold"); |
|
|
&shufps ("xmm4","xmm0",0b00010000); |
|
|
&xorps ("xmm0","xmm4"); |
|
|
&shufps ("xmm4","xmm0",0b10001100); |
|
|
&xorps ("xmm0","xmm4"); |
|
|
&shufps ("xmm1","xmm1",0b11111111); # critical path |
|
|
&xorps ("xmm0","xmm1"); |
|
|
&ret(); |
|
|
|
|
|
&set_label("key_256b",16); |
|
|
&$movekey (&QWP(0,$key),"xmm0"); |
|
|
&lea ($key,&DWP(16,$key)); |
|
|
|
|
|
&shufps ("xmm4","xmm2",0b00010000); |
|
|
&xorps ("xmm2","xmm4"); |
|
|
&shufps ("xmm4","xmm2",0b10001100); |
|
|
&xorps ("xmm2","xmm4"); |
|
|
&shufps ("xmm1","xmm1",0b10101010); # critical path |
|
|
&xorps ("xmm2","xmm1"); |
|
|
&ret(); |
|
|
|
|
|
&set_label("14rounds_alt",16); |
|
|
&movdqa ("xmm5",&QWP(0x00,"ebx")); |
|
|
&movdqa ("xmm4",&QWP(0x20,"ebx")); |
|
|
&mov ($rounds,7); |
|
|
&movdqu (&QWP(-32,$key),"xmm0"); |
|
|
&movdqa ("xmm1","xmm2"); |
|
|
&movdqu (&QWP(-16,$key),"xmm2"); |
|
|
|
|
|
&set_label("loop_key256"); |
|
|
&pshufb ("xmm2","xmm5"); |
|
|
&aesenclast ("xmm2","xmm4"); |
|
|
|
|
|
&movdqa ("xmm3","xmm0"); |
|
|
&pslldq ("xmm0",4); |
|
|
&pxor ("xmm3","xmm0"); |
|
|
&pslldq ("xmm0",4); |
|
|
&pxor ("xmm3","xmm0"); |
|
|
&pslldq ("xmm0",4); |
|
|
&pxor ("xmm0","xmm3"); |
|
|
&pslld ("xmm4",1); |
|
|
|
|
|
&pxor ("xmm0","xmm2"); |
|
|
&movdqu (&QWP(0,$key),"xmm0"); |
|
|
|
|
|
&dec ($rounds); |
|
|
&jz (&label("done_key256")); |
|
|
|
|
|
&pshufd ("xmm2","xmm0",0xff); |
|
|
&pxor ("xmm3","xmm3"); |
|
|
&aesenclast ("xmm2","xmm3"); |
|
|
|
|
|
&movdqa ("xmm3","xmm1"); |
|
|
&pslldq ("xmm1",4); |
|
|
&pxor ("xmm3","xmm1"); |
|
|
&pslldq ("xmm1",4); |
|
|
&pxor ("xmm3","xmm1"); |
|
|
&pslldq ("xmm1",4); |
|
|
&pxor ("xmm1","xmm3"); |
|
|
|
|
|
&pxor ("xmm2","xmm1"); |
|
|
&movdqu (&QWP(16,$key),"xmm2"); |
|
|
&lea ($key,&DWP(32,$key)); |
|
|
&movdqa ("xmm1","xmm2"); |
|
|
&jmp (&label("loop_key256")); |
|
|
|
|
|
&set_label("done_key256"); |
|
|
&mov ($rounds,13); |
|
|
&mov (&DWP(16,$key),$rounds); |
|
|
|
|
|
&set_label("good_key"); |
|
|
&pxor ("xmm0","xmm0"); |
|
|
&pxor ("xmm1","xmm1"); |
|
|
&pxor ("xmm2","xmm2"); |
|
|
&pxor ("xmm3","xmm3"); |
|
|
&pxor ("xmm4","xmm4"); |
|
|
&pxor ("xmm5","xmm5"); |
|
|
&xor ("eax","eax"); |
|
|
&pop ("ebx"); |
|
|
&pop ("ebp"); |
|
|
&ret (); |
|
|
|
|
|
&set_label("bad_pointer",4); |
|
|
&mov ("eax",-1); |
|
|
&pop ("ebx"); |
|
|
&pop ("ebp"); |
|
|
&ret (); |
|
|
&set_label("bad_keybits",4); |
|
|
&pxor ("xmm0","xmm0"); |
|
|
&mov ("eax",-2); |
|
|
&pop ("ebx"); |
|
|
&pop ("ebp"); |
|
|
&ret (); |
|
|
&function_end_B("_aesni_set_encrypt_key"); |
|
|
|
|
|
# int $PREFIX_set_encrypt_key (const unsigned char *userKey, int bits, |
|
|
# AES_KEY *key) |
|
|
&function_begin_B("${PREFIX}_set_encrypt_key"); |
|
|
&record_function_hit(3); |
|
|
|
|
|
&mov ("eax",&wparam(0)); |
|
|
&mov ($rounds,&wparam(1)); |
|
|
&mov ($key,&wparam(2)); |
|
|
&call ("_aesni_set_encrypt_key"); |
|
|
&ret (); |
|
|
&function_end_B("${PREFIX}_set_encrypt_key"); |
|
|
|
|
|
# int $PREFIX_set_decrypt_key (const unsigned char *userKey, int bits, |
|
|
# AES_KEY *key) |
|
|
&function_begin_B("${PREFIX}_set_decrypt_key"); |
|
|
&mov ("eax",&wparam(0)); |
|
|
&mov ($rounds,&wparam(1)); |
|
|
&mov ($key,&wparam(2)); |
|
|
&call ("_aesni_set_encrypt_key"); |
|
|
&mov ($key,&wparam(2)); |
|
|
&shl ($rounds,4); # rounds-1 after _aesni_set_encrypt_key |
|
|
&test ("eax","eax"); |
|
|
&jnz (&label("dec_key_ret")); |
|
|
&lea ("eax",&DWP(16,$key,$rounds)); # end of key schedule |
|
|
|
|
|
&$movekey ("xmm0",&QWP(0,$key)); # just swap |
|
|
&$movekey ("xmm1",&QWP(0,"eax")); |
|
|
&$movekey (&QWP(0,"eax"),"xmm0"); |
|
|
&$movekey (&QWP(0,$key),"xmm1"); |
|
|
&lea ($key,&DWP(16,$key)); |
|
|
&lea ("eax",&DWP(-16,"eax")); |
|
|
|
|
|
&set_label("dec_key_inverse"); |
|
|
&$movekey ("xmm0",&QWP(0,$key)); # swap and inverse |
|
|
&$movekey ("xmm1",&QWP(0,"eax")); |
|
|
&aesimc ("xmm0","xmm0"); |
|
|
&aesimc ("xmm1","xmm1"); |
|
|
&lea ($key,&DWP(16,$key)); |
|
|
&lea ("eax",&DWP(-16,"eax")); |
|
|
&$movekey (&QWP(16,"eax"),"xmm0"); |
|
|
&$movekey (&QWP(-16,$key),"xmm1"); |
|
|
&cmp ("eax",$key); |
|
|
&ja (&label("dec_key_inverse")); |
|
|
|
|
|
&$movekey ("xmm0",&QWP(0,$key)); # inverse middle |
|
|
&aesimc ("xmm0","xmm0"); |
|
|
&$movekey (&QWP(0,$key),"xmm0"); |
|
|
|
|
|
&pxor ("xmm0","xmm0"); |
|
|
&pxor ("xmm1","xmm1"); |
|
|
&xor ("eax","eax"); # return success |
|
|
&set_label("dec_key_ret"); |
|
|
&ret (); |
|
|
&function_end_B("${PREFIX}_set_decrypt_key"); |
|
|
|
|
|
&set_label("key_const",64); |
|
|
&data_word(0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d); |
|
|
&data_word(0x04070605,0x04070605,0x04070605,0x04070605); |
|
|
&data_word(1,1,1,1); |
|
|
&data_word(0x1b,0x1b,0x1b,0x1b); |
|
|
&asciz("AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>"); |
|
|
|
|
|
&asm_finish(); |
|
|
|
|
|
close STDOUT or die "error closing STDOUT: $!";
|
|
|
|
