luna - wolfssl/wolfcrypt/src/aes

   1; /* aes_asm.asm
   2;  *
   3; * Copyright (C) 2006-2026 wolfSSL Inc.
   4;  *
   5;  * This file is part of wolfSSL.
   6;  *
   7;  * wolfSSL is free software; you can redistribute it and/or modify
   8;  * it under the terms of the GNU General Public License as published by
   9;  * the Free Software Foundation; either version 3 of the License, or
  10;  * (at your option) any later version.
  11;  *
  12;  * wolfSSL is distributed in the hope that it will be useful,
  13;  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14;  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15;  * GNU General Public License for more details.
  16;  *
  17;  * You should have received a copy of the GNU General Public License
  18;  * along with this program; if not, write to the Free Software
  19;  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
  20;  */
  21
  22
  23
  24;
  25;
  26;  /* See Intel Advanced Encryption Standard (AES) Instructions Set White Paper
  27;   * by Israel, Intel Mobility Group Development Center, Israel Shay Gueron
  28;   */
  29;
  30;   /* This file is in intel asm syntax, see .s for at&t syntax */
  31;
  32
  33
  34fips_version = 0
  35IFDEF HAVE_FIPS
  36  fips_version = 1
  37  IFDEF HAVE_FIPS_VERSION
  38    fips_version = HAVE_FIPS_VERSION
  39  ENDIF
  40ENDIF
  41
  42IF fips_version GE 2
  43  fipsAb SEGMENT ALIAS(".fipsA$b") 'CODE'
  44ELSE
  45  _text SEGMENT
  46ENDIF
  47
  48
  49;	/*
  50;	AES_CBC_encrypt_AESNI[const	,unsigned	char*in
  51;	unsigned	,char*out
  52;	unsigned	,char	ivec+16
  53;	unsigned	,long	length
  54;	const	,unsigned	char*KS
  55;	int	nr]
  56;	*/
  57AES_CBC_encrypt_AESNI PROC
  58;#	parameter	1:	rdi
  59;#	parameter	2:	rsi
  60;#	parameter	3:	rdx
  61;#	parameter	4:	rcx
  62;#	parameter	5:	r8
  63;#	parameter	6:	r9d
  64
  65; save rdi and rsi to rax and r11, restore before ret
  66	mov rax,rdi
  67	mov r11,rsi
  68
  69; convert to what we had for att&t convention
  70	mov rdi,rcx
  71	mov rsi,rdx
  72	mov rdx,r8
  73	mov rcx,r9
  74	mov r8,[rsp+40]
  75	mov r9d,[rsp+48]
  76
  77	mov	r10,rcx
  78	shr	rcx,4
  79	shl	r10,60
  80	je	NO_PARTS
  81	add	rcx,1
  82NO_PARTS:
  83	sub	rsi,16
  84	movdqa	xmm1,[rdx]
  85LOOP_1:
  86	pxor	xmm1,[rdi]
  87	pxor	xmm1,[r8]
  88	add	rsi,16
  89	add	rdi,16
  90	cmp	r9d,12
  91	aesenc	xmm1,16[r8]
  92	aesenc	xmm1,32[r8]
  93	aesenc	xmm1,48[r8]
  94	aesenc	xmm1,64[r8]
  95	aesenc	xmm1,80[r8]
  96	aesenc	xmm1,96[r8]
  97	aesenc	xmm1,112[r8]
  98	aesenc	xmm1,128[r8]
  99	aesenc	xmm1,144[r8]
 100	movdqa	xmm2,160[r8]
 101	jb	LAST
 102	cmp	r9d,14
 103
 104	aesenc	xmm1,160[r8]
 105	aesenc	xmm1,176[r8]
 106	movdqa	xmm2,192[r8]
 107	jb	LAST
 108	aesenc	xmm1,192[r8]
 109	aesenc	xmm1,208[r8]
 110	movdqa	xmm2,224[r8]
 111LAST:
 112	dec	rcx
 113	aesenclast	xmm1,xmm2
 114	movdqu	[rsi],xmm1
 115	jne	LOOP_1
 116	; restore non volatile rdi,rsi
 117	mov rdi,rax
 118	mov rsi,r11
 119	ret
 120AES_CBC_encrypt_AESNI ENDP
 121
 122
 123; void AES_CBC_decrypt_AESNI_by4(const unsigned char* in,
 124;                          unsigned char* out,
 125;                          unsigned char ivec[16],
 126;                          unsigned long length,
 127;                          const unsigned char* KS,
 128;                          int nr)
 129AES_CBC_decrypt_AESNI_by4 PROC
 130; parameter 1: rdi
 131; parameter 2: rsi
 132; parameter 3: rdx
 133; parameter 4: rcx
 134; parameter 5: r8
 135; parameter 6: r9d
 136
 137        ; save rdi and rsi to rax and r11, restore before ret
 138        mov         rax, rdi
 139        mov         r11, rsi
 140        ; convert to what we had for att&t convention
 141        mov         rdi, rcx
 142        mov         rsi, rdx
 143        mov         rdx, r8
 144        mov         rcx,r9
 145        mov         r8, [rsp+40]
 146        mov         r9d, [rsp+48]
 147        ; on microsoft xmm6-xmm15 are non volatile,
 148        ; let's save on stack and restore at end
 149        sub         rsp, 8+8*16  ; 8 = align stack , 8 xmm6-12,15 16 bytes each
 150        movdqa      [rsp+0], xmm6
 151        movdqa      [rsp+16], xmm7
 152        movdqa      [rsp+32], xmm8
 153        movdqa      [rsp+48], xmm9
 154        movdqa      [rsp+64], xmm10
 155        movdqa      [rsp+80], xmm11
 156        movdqa      [rsp+96], xmm12
 157        movdqa      [rsp+112], xmm15
 158        ; back to our original code, more or less
 159        mov         r10, rcx
 160        shr         rcx, 4
 161        shl         r10, 60
 162        je          DNO_PARTS_4
 163        add         rcx, 1
 164DNO_PARTS_4:
 165        mov         r10, rcx
 166        shl         r10, 62
 167        shr         r10, 62
 168        shr         rcx, 2
 169        movdqu      xmm5, [rdx]
 170        je          DREMAINDER_4
 171        sub         rsi, 64
 172DLOOP_4:
 173        movdqu      xmm1, [rdi]
 174        movdqu      xmm2, 16[rdi]
 175        movdqu      xmm3, 32[rdi]
 176        movdqu      xmm4, 48[rdi]
 177        movdqa      xmm6, xmm1
 178        movdqa      xmm7, xmm2
 179        movdqa      xmm8, xmm3
 180        movdqa      xmm15, xmm4
 181        movdqa      xmm9, [r8]
 182        movdqa      xmm10, 16[r8]
 183        movdqa      xmm11, 32[r8]
 184        movdqa      xmm12, 48[r8]
 185        pxor        xmm1, xmm9
 186        pxor        xmm2, xmm9
 187        pxor        xmm3, xmm9
 188        pxor        xmm4, xmm9
 189        aesdec      xmm1, xmm10
 190        aesdec      xmm2, xmm10
 191        aesdec      xmm3, xmm10
 192        aesdec      xmm4, xmm10
 193        aesdec      xmm1, xmm11
 194        aesdec      xmm2, xmm11
 195        aesdec      xmm3, xmm11
 196        aesdec      xmm4, xmm11
 197        aesdec      xmm1, xmm12
 198        aesdec      xmm2, xmm12
 199        aesdec      xmm3, xmm12
 200        aesdec      xmm4, xmm12
 201        movdqa      xmm9, 64[r8]
 202        movdqa      xmm10, 80[r8]
 203        movdqa      xmm11, 96[r8]
 204        movdqa      xmm12, 112[r8]
 205        aesdec      xmm1, xmm9
 206        aesdec      xmm2, xmm9
 207        aesdec      xmm3, xmm9
 208        aesdec      xmm4, xmm9
 209        aesdec      xmm1, xmm10
 210        aesdec      xmm2, xmm10
 211        aesdec      xmm3, xmm10
 212        aesdec      xmm4, xmm10
 213        aesdec      xmm1, xmm11
 214        aesdec      xmm2, xmm11
 215        aesdec      xmm3, xmm11
 216        aesdec      xmm4, xmm11
 217        aesdec      xmm1, xmm12
 218        aesdec      xmm2, xmm12
 219        aesdec      xmm3, xmm12
 220        aesdec      xmm4, xmm12
 221        movdqa      xmm9, 128[r8]
 222        movdqa      xmm10, 144[r8]
 223        movdqa      xmm11, 160[r8]
 224        cmp         r9d, 12
 225        aesdec      xmm1, xmm9
 226        aesdec      xmm2, xmm9
 227        aesdec      xmm3, xmm9
 228        aesdec      xmm4, xmm9
 229        aesdec      xmm1, xmm10
 230        aesdec      xmm2, xmm10
 231        aesdec      xmm3, xmm10
 232        aesdec      xmm4, xmm10
 233        jb          DLAST_4
 234        movdqa      xmm9, 160[r8]
 235        movdqa      xmm10, 176[r8]
 236        movdqa      xmm11, 192[r8]
 237        cmp         r9d, 14
 238        aesdec      xmm1, xmm9
 239        aesdec      xmm2, xmm9
 240        aesdec      xmm3, xmm9
 241        aesdec      xmm4, xmm9
 242        aesdec      xmm1, xmm10
 243        aesdec      xmm2, xmm10
 244        aesdec      xmm3, xmm10
 245        aesdec      xmm4, xmm10
 246        jb          DLAST_4
 247        movdqa      xmm9, 192[r8]
 248        movdqa      xmm10, 208[r8]
 249        movdqa      xmm11, 224[r8]
 250        aesdec      xmm1, xmm9
 251        aesdec      xmm2, xmm9
 252        aesdec      xmm3, xmm9
 253        aesdec      xmm4, xmm9
 254        aesdec      xmm1, xmm10
 255        aesdec      xmm2, xmm10
 256        aesdec      xmm3, xmm10
 257        aesdec      xmm4, xmm10
 258DLAST_4:
 259        add         rdi, 64
 260        add         rsi, 64
 261        dec         rcx
 262        aesdeclast  xmm1, xmm11
 263        aesdeclast  xmm2, xmm11
 264        aesdeclast  xmm3, xmm11
 265        aesdeclast  xmm4, xmm11
 266        pxor        xmm1, xmm5
 267        pxor        xmm2, xmm6
 268        pxor        xmm3, xmm7
 269        pxor        xmm4, xmm8
 270        movdqu      [rsi], xmm1
 271        movdqu      16[rsi], xmm2
 272        movdqu      32[rsi], xmm3
 273        movdqu      48[rsi], xmm4
 274        movdqa      xmm5, xmm15
 275        jne         DLOOP_4
 276        add         rsi, 64
 277DREMAINDER_4:
 278        cmp         r10, 0
 279        je          DEND_4
 280DLOOP_4_2:
 281        movdqu      xmm1, [rdi]
 282        movdqa      xmm15, xmm1
 283        add         rdi, 16
 284        pxor        xmm1, [r8]
 285        movdqu      xmm2, 160[r8]
 286        cmp         r9d, 12
 287        aesdec      xmm1, 16[r8]
 288        aesdec      xmm1, 32[r8]
 289        aesdec      xmm1, 48[r8]
 290        aesdec      xmm1, 64[r8]
 291        aesdec      xmm1, 80[r8]
 292        aesdec      xmm1, 96[r8]
 293        aesdec      xmm1, 112[r8]
 294        aesdec      xmm1, 128[r8]
 295        aesdec      xmm1, 144[r8]
 296        jb          DLAST_4_2
 297        movdqu      xmm2, 192[r8]
 298        cmp         r9d, 14
 299        aesdec      xmm1, 160[r8]
 300        aesdec      xmm1, 176[r8]
 301        jb          DLAST_4_2
 302        movdqu      xmm2, 224[r8]
 303        aesdec      xmm1, 192[r8]
 304        aesdec      xmm1, 208[r8]
 305DLAST_4_2:
 306        aesdeclast  xmm1, xmm2
 307        pxor        xmm1, xmm5
 308        movdqa      xmm5, xmm15
 309        movdqu      [rsi], xmm1
 310        add         rsi, 16
 311        dec         r10
 312        jne         DLOOP_4_2
 313DEND_4:
 314        ; restore non volatile rdi,rsi
 315        mov         rdi, rax
 316        mov         rsi, r11
 317        ; restore non volatile xmms from stack
 318        movdqa      xmm6, [rsp+0]
 319        movdqa      xmm7, [rsp+16]
 320        movdqa      xmm8, [rsp+32]
 321        movdqa      xmm9, [rsp+48]
 322        movdqa      xmm10, [rsp+64]
 323        movdqa      xmm11, [rsp+80]
 324        movdqa      xmm12, [rsp+96]
 325        movdqa      xmm15, [rsp+112]
 326        add         rsp, 8+8*16 ; 8 = align stack , 8 xmm6-12,15 16 bytes each
 327        ret
 328AES_CBC_decrypt_AESNI_by4 ENDP
 329
 330
 331; void AES_CBC_decrypt_AESNI_by6(const unsigned char *in,
 332;                          unsigned char *out,
 333;                          unsigned char ivec[16],
 334;                          unsigned long length,
 335;                          const unsigned char *KS,
 336;                          int nr)
 337AES_CBC_decrypt_AESNI_by6 PROC
 338; parameter 1: rdi - in
 339; parameter 2: rsi - out
 340; parameter 3: rdx - ivec
 341; parameter 4: rcx - length
 342; parameter 5: r8  - KS
 343; parameter 6: r9d - nr
 344
 345        ; save rdi and rsi to rax and r11, restore before ret
 346        mov         rax, rdi
 347        mov         r11, rsi
 348        ; convert to what we had for att&t convention
 349        mov         rdi, rcx
 350        mov         rsi, rdx
 351        mov         rdx, r8
 352        mov         rcx, r9
 353        mov         r8, [rsp+40]
 354        mov         r9d, [rsp+48]
 355        ; on microsoft xmm6-xmm15 are non volatile,
 356        ; let's save on stack and restore at end
 357        sub         rsp, 8+9*16  ; 8 = align stack , 9 xmm6-14 16 bytes each
 358        movdqa      [rsp+0], xmm6
 359        movdqa      [rsp+16], xmm7
 360        movdqa      [rsp+32], xmm8
 361        movdqa      [rsp+48], xmm9
 362        movdqa      [rsp+64], xmm10
 363        movdqa      [rsp+80], xmm11
 364        movdqa      [rsp+96], xmm12
 365        movdqa      [rsp+112], xmm13
 366        movdqa      [rsp+128], xmm14
 367        ; back to our original code, more or less
 368        mov         r10, rcx
 369        shr         rcx, 4
 370        shl         r10, 60
 371        je          DNO_PARTS_6
 372        add         rcx, 1
 373DNO_PARTS_6:
 374        mov         r12, rax
 375        mov         r13, rdx
 376        mov         r14, rbx
 377        mov         rdx, 0
 378        mov         rax, rcx
 379        mov         rbx, 6
 380        div         rbx
 381        mov         rcx, rax
 382        mov         r10, rdx
 383        mov         rax, r12
 384        mov         rdx, r13
 385        mov         rbx, r14
 386        cmp         rcx, 0
 387        movdqu      xmm7, [rdx]
 388        je          DREMAINDER_6
 389        sub         rsi, 96
 390DLOOP_6:
 391        movdqu      xmm1, [rdi]
 392        movdqu      xmm2, 16[rdi]
 393        movdqu      xmm3, 32[rdi]
 394        movdqu      xmm4, 48[rdi]
 395        movdqu      xmm5, 64[rdi]
 396        movdqu      xmm6, 80[rdi]
 397        movdqa      xmm8, [r8]
 398        movdqa      xmm9, 16[r8]
 399        movdqa      xmm10, 32[r8]
 400        movdqa      xmm11, 48[r8]
 401        pxor        xmm1, xmm8
 402        pxor        xmm2, xmm8
 403        pxor        xmm3, xmm8
 404        pxor        xmm4, xmm8
 405        pxor        xmm5, xmm8
 406        pxor        xmm6, xmm8
 407        aesdec      xmm1, xmm9
 408        aesdec      xmm2, xmm9
 409        aesdec      xmm3, xmm9
 410        aesdec      xmm4, xmm9
 411        aesdec      xmm5, xmm9
 412        aesdec      xmm6, xmm9
 413        aesdec      xmm1, xmm10
 414        aesdec      xmm2, xmm10
 415        aesdec      xmm3, xmm10
 416        aesdec      xmm4, xmm10
 417        aesdec      xmm5, xmm10
 418        aesdec      xmm6, xmm10
 419        aesdec      xmm1, xmm11
 420        aesdec      xmm2, xmm11
 421        aesdec      xmm3, xmm11
 422        aesdec      xmm4, xmm11
 423        aesdec      xmm5, xmm11
 424        aesdec      xmm6, xmm11
 425        movdqa      xmm8, 64[r8]
 426        movdqa      xmm9, 80[r8]
 427        movdqa      xmm10, 96[r8]
 428        movdqa      xmm11, 112[r8]
 429        aesdec      xmm1, xmm8
 430        aesdec      xmm2, xmm8
 431        aesdec      xmm3, xmm8
 432        aesdec      xmm4, xmm8
 433        aesdec      xmm5, xmm8
 434        aesdec      xmm6, xmm8
 435        aesdec      xmm1, xmm9
 436        aesdec      xmm2, xmm9
 437        aesdec      xmm3, xmm9
 438        aesdec      xmm4, xmm9
 439        aesdec      xmm5, xmm9
 440        aesdec      xmm6, xmm9
 441        aesdec      xmm1, xmm10
 442        aesdec      xmm2, xmm10
 443        aesdec      xmm3, xmm10
 444        aesdec      xmm4, xmm10
 445        aesdec      xmm5, xmm10
 446        aesdec      xmm6, xmm10
 447        aesdec      xmm1, xmm11
 448        aesdec      xmm2, xmm11
 449        aesdec      xmm3, xmm11
 450        aesdec      xmm4, xmm11
 451        aesdec      xmm5, xmm11
 452        aesdec      xmm6, xmm11
 453        movdqa      xmm8, 128[r8]
 454        movdqa      xmm9, 144[r8]
 455        movdqa      xmm10, 160[r8]
 456        cmp         r9d, 12
 457        aesdec      xmm1, xmm8
 458        aesdec      xmm2, xmm8
 459        aesdec      xmm3, xmm8
 460        aesdec      xmm4, xmm8
 461        aesdec      xmm5, xmm8
 462        aesdec      xmm6, xmm8
 463        aesdec      xmm1, xmm9
 464        aesdec      xmm2, xmm9
 465        aesdec      xmm3, xmm9
 466        aesdec      xmm4, xmm9
 467        aesdec      xmm5, xmm9
 468        aesdec      xmm6, xmm9
 469        jb          DLAST_6
 470        movdqa      xmm8, 160[r8]
 471        movdqa      xmm9, 176[r8]
 472        movdqa      xmm10, 192[r8]
 473        cmp         r9d, 14
 474        aesdec      xmm1, xmm8
 475        aesdec      xmm2, xmm8
 476        aesdec      xmm3, xmm8
 477        aesdec      xmm4, xmm8
 478        aesdec      xmm5, xmm8
 479        aesdec      xmm6, xmm8
 480        aesdec      xmm1, xmm9
 481        aesdec      xmm2, xmm9
 482        aesdec      xmm3, xmm9
 483        aesdec      xmm4, xmm9
 484        aesdec      xmm5, xmm9
 485        aesdec      xmm6, xmm9
 486        jb          DLAST_6
 487        movdqa      xmm8, 192[r8]
 488        movdqa      xmm9, 208[r8]
 489        movdqa      xmm10, 224[r8]
 490        aesdec      xmm1, xmm8
 491        aesdec      xmm2, xmm8
 492        aesdec      xmm3, xmm8
 493        aesdec      xmm4, xmm8
 494        aesdec      xmm5, xmm8
 495        aesdec      xmm6, xmm8
 496        aesdec      xmm1, xmm9
 497        aesdec      xmm2, xmm9
 498        aesdec      xmm3, xmm9
 499        aesdec      xmm4, xmm9
 500        aesdec      xmm5, xmm9
 501        aesdec      xmm6, xmm9
 502DLAST_6:
 503        add         rsi, 96
 504        aesdeclast  xmm1, xmm10
 505        aesdeclast  xmm2, xmm10
 506        aesdeclast  xmm3, xmm10
 507        aesdeclast  xmm4, xmm10
 508        aesdeclast  xmm5, xmm10
 509        aesdeclast  xmm6, xmm10
 510        movdqu      xmm8, [rdi]
 511        movdqu      xmm9, 16[rdi]
 512        movdqu      xmm10, 32[rdi]
 513        movdqu      xmm11, 48[rdi]
 514        movdqu      xmm12, 64[rdi]
 515        movdqu      xmm13, 80[rdi]
 516        pxor        xmm1, xmm7
 517        pxor        xmm2, xmm8
 518        pxor        xmm3, xmm9
 519        pxor        xmm4, xmm10
 520        pxor        xmm5, xmm11
 521        pxor        xmm6, xmm12
 522        movdqu      xmm7, xmm13
 523        movdqu      [rsi], xmm1
 524        movdqu      16[rsi], xmm2
 525        movdqu      32[rsi], xmm3
 526        movdqu      48[rsi], xmm4
 527        movdqu      64[rsi], xmm5
 528        movdqu      80[rsi], xmm6
 529        add         rdi, 96
 530        dec         rcx
 531        jne         DLOOP_6
 532        add         rsi, 96
 533DREMAINDER_6:
 534        cmp         r10, 0
 535        je          DEND_6
 536DLOOP_6_2:
 537        movdqu      xmm1, [rdi]
 538        movdqa      xmm10, xmm1
 539        add         rdi, 16
 540        pxor        xmm1, [r8]
 541        movdqu      xmm2, 160[r8]
 542        cmp         r9d, 12
 543        aesdec      xmm1, 16[r8]
 544        aesdec      xmm1, 32[r8]
 545        aesdec      xmm1, 48[r8]
 546        aesdec      xmm1, 64[r8]
 547        aesdec      xmm1, 80[r8]
 548        aesdec      xmm1, 96[r8]
 549        aesdec      xmm1, 112[r8]
 550        aesdec      xmm1, 128[r8]
 551        aesdec      xmm1, 144[r8]
 552        jb          DLAST_6_2
 553        movdqu      xmm2, 192[r8]
 554        cmp         r9d, 14
 555        aesdec      xmm1, 160[r8]
 556        aesdec      xmm1, 176[r8]
 557        jb          DLAST_6_2
 558        movdqu      xmm2, 224[r8]
 559        aesdec      xmm1, 192[r8]
 560        aesdec      xmm1, 208[r8]
 561DLAST_6_2:
 562        aesdeclast  xmm1, xmm2
 563        pxor        xmm1, xmm7
 564        movdqa      xmm7, xmm10
 565        movdqu      [rsi], xmm1
 566        add         rsi, 16
 567        dec         r10
 568        jne         DLOOP_6_2
 569DEND_6:
 570        ; restore non volatile rdi,rsi
 571        mov         rdi, rax
 572        mov         rsi, r11
 573        ; restore non volatile xmms from stack
 574        movdqa      xmm6, [rsp+0]
 575        movdqa      xmm7, [rsp+16]
 576        movdqa      xmm8, [rsp+32]
 577        movdqa      xmm9, [rsp+48]
 578        movdqa      xmm10, [rsp+64]
 579        movdqa      xmm11, [rsp+80]
 580        movdqa      xmm12, [rsp+96]
 581        movdqa      xmm13, [rsp+112]
 582        movdqa      xmm14, [rsp+128]
 583        add         rsp, 8+9*16 ; 8 = align stack , 9 xmm6-14 16 bytes each
 584        ret
 585AES_CBC_decrypt_AESNI_by6 ENDP
 586
 587
 588; void AES_CBC_decrypt_AESNI_by8(const unsigned char *in,
 589;                          unsigned char *out,
 590;                          unsigned char ivec[16],
 591;                          unsigned long length,
 592;                          const unsigned char *KS,
 593;                          int nr)
 594AES_CBC_decrypt_AESNI_by8 PROC
 595; parameter 1: rdi - in
 596; parameter 2: rsi - out
 597; parameter 3: rdx - ivec
 598; parameter 4: rcx - length
 599; parameter 5: r8  - KS
 600; parameter 6: r9d - nr
 601
 602        ; save rdi and rsi to rax and r11, restore before ret
 603        mov         rax, rdi
 604        mov         r11, rsi
 605        ; convert to what we had for att&t convention
 606        mov         rdi, rcx
 607        mov         rsi, rdx
 608        mov         rdx, r8
 609        mov         rcx,r9
 610        mov         r8, [rsp+40]
 611        mov         r9d, [rsp+48]
 612        ; on microsoft xmm6-xmm15 are non volatile,
 613        ; let's save on stack and restore at end
 614        sub         rsp, 8+8*16  ; 8 = align stack , 8 xmm6-13 16 bytes each
 615        movdqa      [rsp+0], xmm6
 616        movdqa      [rsp+16], xmm7
 617        movdqa      [rsp+32], xmm8
 618        movdqa      [rsp+48], xmm9
 619        movdqa      [rsp+64], xmm10
 620        movdqa      [rsp+80], xmm11
 621        movdqa      [rsp+96], xmm12
 622        movdqa      [rsp+112], xmm13
 623        ; back to our original code, more or less
 624        mov         r10, rcx
 625        shr         rcx, 4
 626        shl         r10, 60
 627        je          DNO_PARTS_8
 628        add         rcx, 1
 629DNO_PARTS_8:
 630        mov         r10, rcx
 631        shl         r10, 61
 632        shr         r10, 61
 633        shr         rcx, 3
 634        movdqu      xmm9, [rdx]
 635        je          DREMAINDER_8
 636        sub         rsi, 128
 637DLOOP_8:
 638        movdqu      xmm1, [rdi]
 639        movdqu      xmm2, 16[rdi]
 640        movdqu      xmm3, 32[rdi]
 641        movdqu      xmm4, 48[rdi]
 642        movdqu      xmm5, 64[rdi]
 643        movdqu      xmm6, 80[rdi]
 644        movdqu      xmm7, 96[rdi]
 645        movdqu      xmm8, 112[rdi]
 646        movdqa      xmm10, [r8]
 647        movdqa      xmm11, 16[r8]
 648        movdqa      xmm12, 32[r8]
 649        movdqa      xmm13, 48[r8]
 650        pxor        xmm1, xmm10
 651        pxor        xmm2, xmm10
 652        pxor        xmm3, xmm10
 653        pxor        xmm4, xmm10
 654        pxor        xmm5, xmm10
 655        pxor        xmm6, xmm10
 656        pxor        xmm7, xmm10
 657        pxor        xmm8, xmm10
 658        aesdec      xmm1, xmm11
 659        aesdec      xmm2, xmm11
 660        aesdec      xmm3, xmm11
 661        aesdec      xmm4, xmm11
 662        aesdec      xmm5, xmm11
 663        aesdec      xmm6, xmm11
 664        aesdec      xmm7, xmm11
 665        aesdec      xmm8, xmm11
 666        aesdec      xmm1, xmm12
 667        aesdec      xmm2, xmm12
 668        aesdec      xmm3, xmm12
 669        aesdec      xmm4, xmm12
 670        aesdec      xmm5, xmm12
 671        aesdec      xmm6, xmm12
 672        aesdec      xmm7, xmm12
 673        aesdec      xmm8, xmm12
 674        aesdec      xmm1, xmm13
 675        aesdec      xmm2, xmm13
 676        aesdec      xmm3, xmm13
 677        aesdec      xmm4, xmm13
 678        aesdec      xmm5, xmm13
 679        aesdec      xmm6, xmm13
 680        aesdec      xmm7, xmm13
 681        aesdec      xmm8, xmm13
 682        movdqa      xmm10, 64[r8]
 683        movdqa      xmm11, 80[r8]
 684        movdqa      xmm12, 96[r8]
 685        movdqa      xmm13, 112[r8]
 686        aesdec      xmm1, xmm10
 687        aesdec      xmm2, xmm10
 688        aesdec      xmm3, xmm10
 689        aesdec      xmm4, xmm10
 690        aesdec      xmm5, xmm10
 691        aesdec      xmm6, xmm10
 692        aesdec      xmm7, xmm10
 693        aesdec      xmm8, xmm10
 694        aesdec      xmm1, xmm11
 695        aesdec      xmm2, xmm11
 696        aesdec      xmm3, xmm11
 697        aesdec      xmm4, xmm11
 698        aesdec      xmm5, xmm11
 699        aesdec      xmm6, xmm11
 700        aesdec      xmm7, xmm11
 701        aesdec      xmm8, xmm11
 702        aesdec      xmm1, xmm12
 703        aesdec      xmm2, xmm12
 704        aesdec      xmm3, xmm12
 705        aesdec      xmm4, xmm12
 706        aesdec      xmm5, xmm12
 707        aesdec      xmm6, xmm12
 708        aesdec      xmm7, xmm12
 709        aesdec      xmm8, xmm12
 710        aesdec      xmm1, xmm13
 711        aesdec      xmm2, xmm13
 712        aesdec      xmm3, xmm13
 713        aesdec      xmm4, xmm13
 714        aesdec      xmm5, xmm13
 715        aesdec      xmm6, xmm13
 716        aesdec      xmm7, xmm13
 717        aesdec      xmm8, xmm13
 718        movdqa      xmm10, 128[r8]
 719        movdqa      xmm11, 144[r8]
 720        movdqa      xmm12, 160[r8]
 721        cmp         r9d, 12
 722        aesdec      xmm1, xmm10
 723        aesdec      xmm2, xmm10
 724        aesdec      xmm3, xmm10
 725        aesdec      xmm4, xmm10
 726        aesdec      xmm5, xmm10
 727        aesdec      xmm6, xmm10
 728        aesdec      xmm7, xmm10
 729        aesdec      xmm8, xmm10
 730        aesdec      xmm1, xmm11
 731        aesdec      xmm2, xmm11
 732        aesdec      xmm3, xmm11
 733        aesdec      xmm4, xmm11
 734        aesdec      xmm5, xmm11
 735        aesdec      xmm6, xmm11
 736        aesdec      xmm7, xmm11
 737        aesdec      xmm8, xmm11
 738        jb          DLAST_8
 739        movdqa      xmm10, 160[r8]
 740        movdqa      xmm11, 176[r8]
 741        movdqa      xmm12, 192[r8]
 742        cmp         r9d, 14
 743        aesdec      xmm1, xmm10
 744        aesdec      xmm2, xmm10
 745        aesdec      xmm3, xmm10
 746        aesdec      xmm4, xmm10
 747        aesdec      xmm5, xmm10
 748        aesdec      xmm6, xmm10
 749        aesdec      xmm7, xmm10
 750        aesdec      xmm8, xmm10
 751        aesdec      xmm1, xmm11
 752        aesdec      xmm2, xmm11
 753        aesdec      xmm3, xmm11
 754        aesdec      xmm4, xmm11
 755        aesdec      xmm5, xmm11
 756        aesdec      xmm6, xmm11
 757        aesdec      xmm7, xmm11
 758        aesdec      xmm8, xmm11
 759        jb          DLAST_8
 760        movdqa      xmm10, 192[r8]
 761        movdqa      xmm11, 208[r8]
 762        movdqa      xmm12, 224[r8]
 763        aesdec      xmm1, xmm10
 764        aesdec      xmm2, xmm10
 765        aesdec      xmm3, xmm10
 766        aesdec      xmm4, xmm10
 767        aesdec      xmm5, xmm10
 768        aesdec      xmm6, xmm10
 769        aesdec      xmm7, xmm10
 770        aesdec      xmm8, xmm10
 771        aesdec      xmm1, xmm11
 772        aesdec      xmm2, xmm11
 773        aesdec      xmm3, xmm11
 774        aesdec      xmm4, xmm11
 775        aesdec      xmm5, xmm11
 776        aesdec      xmm6, xmm11
 777        aesdec      xmm7, xmm11
 778        aesdec      xmm8, xmm11
 779DLAST_8:
 780        add         rsi, 128
 781        aesdeclast  xmm1, xmm12
 782        aesdeclast  xmm2, xmm12
 783        aesdeclast  xmm3, xmm12
 784        aesdeclast  xmm4, xmm12
 785        aesdeclast  xmm5, xmm12
 786        aesdeclast  xmm6, xmm12
 787        aesdeclast  xmm7, xmm12
 788        aesdeclast  xmm8, xmm12
 789        movdqu      xmm10, [rdi]
 790        movdqu      xmm11, 16[rdi]
 791        movdqu      xmm12, 32[rdi]
 792        movdqu      xmm13, 48[rdi]
 793        pxor        xmm1, xmm9
 794        pxor        xmm2, xmm10
 795        pxor        xmm3, xmm11
 796        pxor        xmm4, xmm12
 797        pxor        xmm5, xmm13
 798        movdqu      xmm10, 64[rdi]
 799        movdqu      xmm11, 80[rdi]
 800        movdqu      xmm12, 96[rdi]
 801        movdqu      xmm9, 112[rdi]
 802        pxor        xmm6, xmm10
 803        pxor        xmm7, xmm11
 804        pxor        xmm8, xmm12
 805        movdqu      [rsi], xmm1
 806        movdqu      16[rsi], xmm2
 807        movdqu      32[rsi], xmm3
 808        movdqu      48[rsi], xmm4
 809        movdqu      64[rsi], xmm5
 810        movdqu      80[rsi], xmm6
 811        movdqu      96[rsi], xmm7
 812        movdqu      112[rsi], xmm8
 813        add         rdi, 128
 814        dec         rcx
 815        jne         DLOOP_8
 816        add         rsi, 128
 817DREMAINDER_8:
 818        cmp         r10, 0 
 819        je          DEND_8
 820DLOOP_8_2:
 821        movdqu      xmm1, [rdi]
 822        movdqa      xmm10, xmm1
 823        add         rdi, 16
 824        pxor        xmm1, [r8]
 825        movdqu      xmm2, 160[r8]
 826        cmp         r9d, 12
 827        aesdec      xmm1, 16[r8]
 828        aesdec      xmm1, 32[r8]
 829        aesdec      xmm1, 48[r8]
 830        aesdec      xmm1, 64[r8]
 831        aesdec      xmm1, 80[r8]
 832        aesdec      xmm1, 96[r8]
 833        aesdec      xmm1, 112[r8]
 834        aesdec      xmm1, 128[r8]
 835        aesdec      xmm1, 144[r8]
 836        jb          DLAST_8_2
 837        movdqu      xmm2, 192[r8]
 838        cmp         r9d, 14
 839        aesdec      xmm1, 160[r8]
 840        aesdec      xmm1, 176[r8]
 841        jb          DLAST_8_2
 842        movdqu      xmm2, 224[r8]
 843        aesdec      xmm1, 192[r8]
 844        aesdec      xmm1, 208[r8]
 845DLAST_8_2:
 846        aesdeclast  xmm1, xmm2
 847        pxor        xmm1, xmm9
 848        movdqa      xmm9, xmm10
 849        movdqu      [rsi], xmm1
 850        add         rsi, 16
 851        dec         r10
 852        jne         DLOOP_8_2
 853DEND_8:
 854        ; restore non volatile rdi,rsi
 855        mov         rdi, rax
 856        mov         rsi, r11
 857        ; restore non volatile xmms from stack
 858        movdqa      xmm6, [rsp+0]
 859        movdqa      xmm7, [rsp+16]
 860        movdqa      xmm8, [rsp+32]
 861        movdqa      xmm9, [rsp+48]
 862        movdqa      xmm10, [rsp+64]
 863        movdqa      xmm11, [rsp+80]
 864        movdqa      xmm12, [rsp+96]
 865        movdqa      xmm13, [rsp+112]
 866        add         rsp, 8+8*16 ; 8 = align stack , 8 xmm6-13 16 bytes each
 867        ret
 868AES_CBC_decrypt_AESNI_by8 ENDP
 869
 870
 871;	/*
 872;	AES_ECB_encrypt_AESNI[const	,unsigned	char*in
 873;	unsigned	,char*out
 874;	unsigned	,long	length
 875;	const	,unsigned	char*KS
 876;	int	nr]
 877;	*/
 878;	.	globl	AES_ECB_encrypt_AESNI
 879AES_ECB_encrypt_AESNI PROC
 880;#	parameter	1:	rdi
 881;#	parameter	2:	rsi
 882;#	parameter	3:	rdx
 883;#	parameter	4:	rcx
 884;#	parameter	5:	r8d
 885
 886; save rdi and rsi to rax and r11, restore before ret
 887	mov rax,rdi
 888	mov r11,rsi
 889
 890; convert to what we had for att&t convention
 891    mov rdi,rcx
 892	mov rsi,rdx
 893	mov rdx,r8
 894	mov rcx,r9
 895	mov r8d,[rsp+40]
 896
 897; on microsoft xmm6-xmm15 are non volaitle, let's save on stack and restore at end
 898	sub rsp,8+4*16  ; 8 = align stack , 4 xmm9-12, 16 bytes each
 899	movdqa [rsp+0], xmm9
 900	movdqa [rsp+16], xmm10
 901	movdqa [rsp+32], xmm11
 902	movdqa [rsp+48], xmm12
 903
 904
 905	mov	r10,rdx
 906	shr	rdx,4
 907	shl	r10,60
 908	je	EECB_NO_PARTS_4
 909	add	rdx,1
 910EECB_NO_PARTS_4:
 911	mov	r10,rdx
 912	shl	r10,62
 913	shr	r10,62
 914	shr	rdx,2
 915	je	EECB_REMAINDER_4
 916	sub	rsi,64
 917EECB_LOOP_4:
 918	movdqu  xmm1,[rdi]
 919	movdqu	xmm2,16[rdi]
 920	movdqu	xmm3,32[rdi]
 921	movdqu	xmm4,48[rdi]
 922	movdqa  xmm9,[rcx]
 923	movdqa	xmm10,16[rcx]
 924	movdqa	xmm11,32[rcx]
 925	movdqa	xmm12,48[rcx]
 926	pxor	xmm1,xmm9
 927	pxor	xmm2,xmm9
 928	pxor	xmm3,xmm9
 929	pxor	xmm4,xmm9
 930	aesenc	xmm1,xmm10
 931	aesenc	xmm2,xmm10
 932	aesenc	xmm3,xmm10
 933	aesenc	xmm4,xmm10
 934	aesenc	xmm1,xmm11
 935	aesenc	xmm2,xmm11
 936	aesenc	xmm3,xmm11
 937	aesenc	xmm4,xmm11
 938	aesenc	xmm1,xmm12
 939	aesenc	xmm2,xmm12
 940	aesenc	xmm3,xmm12
 941	aesenc	xmm4,xmm12
 942	movdqa	xmm9,64[rcx]
 943	movdqa	xmm10,80[rcx]
 944	movdqa	xmm11,96[rcx]
 945	movdqa	xmm12,112[rcx]
 946	aesenc	xmm1,xmm9
 947	aesenc	xmm2,xmm9
 948	aesenc	xmm3,xmm9
 949	aesenc	xmm4,xmm9
 950	aesenc	xmm1,xmm10
 951	aesenc	xmm2,xmm10
 952	aesenc	xmm3,xmm10
 953	aesenc	xmm4,xmm10
 954	aesenc	xmm1,xmm11
 955	aesenc	xmm2,xmm11
 956	aesenc	xmm3,xmm11
 957	aesenc	xmm4,xmm11
 958	aesenc	xmm1,xmm12
 959	aesenc	xmm2,xmm12
 960	aesenc	xmm3,xmm12
 961	aesenc	xmm4,xmm12
 962	movdqa	xmm9,128[rcx]
 963	movdqa	xmm10,144[rcx]
 964	movdqa	xmm11,160[rcx]
 965	cmp	r8d,12
 966	aesenc	xmm1,xmm9
 967	aesenc	xmm2,xmm9
 968	aesenc	xmm3,xmm9
 969	aesenc	xmm4,xmm9
 970	aesenc	xmm1,xmm10
 971	aesenc	xmm2,xmm10
 972	aesenc	xmm3,xmm10
 973	aesenc	xmm4,xmm10
 974	jb	EECB_LAST_4
 975	movdqa	xmm9,160[rcx]
 976	movdqa	xmm10,176[rcx]
 977	movdqa	xmm11,192[rcx]
 978	cmp	r8d,14
 979	aesenc	xmm1,xmm9
 980	aesenc	xmm2,xmm9
 981	aesenc	xmm3,xmm9
 982	aesenc	xmm4,xmm9
 983	aesenc	xmm1,xmm10
 984	aesenc	xmm2,xmm10
 985	aesenc	xmm3,xmm10
 986	aesenc	xmm4,xmm10
 987	jb	EECB_LAST_4
 988	movdqa	xmm9,192[rcx]
 989	movdqa	xmm10,208[rcx]
 990	movdqa	xmm11,224[rcx]
 991	aesenc	xmm1,xmm9
 992	aesenc	xmm2,xmm9
 993	aesenc	xmm3,xmm9
 994	aesenc	xmm4,xmm9
 995	aesenc	xmm1,xmm10
 996	aesenc	xmm2,xmm10
 997	aesenc	xmm3,xmm10
 998	aesenc	xmm4,xmm10
 999EECB_LAST_4:
1000	add	rdi,64
1001	add	rsi,64
1002	dec	rdx
1003	aesenclast	xmm1,xmm11
1004	aesenclast	xmm2,xmm11
1005	aesenclast	xmm3,xmm11
1006	aesenclast	xmm4,xmm11
1007	movdqu	[rsi],xmm1
1008	movdqu	16[rsi],xmm2
1009	movdqu	32[rsi],xmm3
1010	movdqu	48[rsi],xmm4
1011	jne	EECB_LOOP_4
1012	add	rsi,64
1013EECB_REMAINDER_4:
1014	cmp	r10,0
1015	je	EECB_END_4
1016EECB_LOOP_4_2:
1017	movdqu  xmm1,[rdi]
1018	add	rdi,16
1019	pxor	xmm1,[rcx]
1020	movdqu	xmm2,160[rcx]
1021	aesenc	xmm1,16[rcx]
1022	aesenc	xmm1,32[rcx]
1023	aesenc	xmm1,48[rcx]
1024	aesenc	xmm1,64[rcx]
1025	aesenc	xmm1,80[rcx]
1026	aesenc	xmm1,96[rcx]
1027	aesenc	xmm1,112[rcx]
1028	aesenc	xmm1,128[rcx]
1029	aesenc	xmm1,144[rcx]
1030	cmp	r8d,12
1031	jb	EECB_LAST_4_2
1032	movdqu	xmm2,192[rcx]
1033	aesenc	xmm1,160[rcx]
1034	aesenc	xmm1,176[rcx]
1035	cmp	r8d,14
1036	jb	EECB_LAST_4_2
1037	movdqu	xmm2,224[rcx]
1038	aesenc	xmm1,192[rcx]
1039	aesenc	xmm1,208[rcx]
1040EECB_LAST_4_2:
1041	aesenclast	xmm1,xmm2
1042	movdqu	[rsi],xmm1
1043	add	rsi,16
1044	dec	r10
1045	jne	EECB_LOOP_4_2
1046EECB_END_4:
1047	; restore non volatile rdi,rsi
1048	mov rdi,rax
1049	mov rsi,r11
1050	; restore non volatile xmms from stack
1051	movdqa xmm9, [rsp+0]
1052	movdqa xmm10, [rsp+16]
1053	movdqa xmm11, [rsp+32]
1054	movdqa xmm12, [rsp+48]
1055	add rsp,8+4*16 ; 8 = align stack , 4 xmm9-12 16 bytes each
1056	ret
1057AES_ECB_encrypt_AESNI ENDP
1058
1059;	/*
1060;	AES_ECB_decrypt_AESNI[const	,unsigned	char*in
1061;	unsigned	,char*out
1062;	unsigned	,long	length
1063;	const	,unsigned	char*KS
1064;	int	nr]
1065;	*/
1066;	.	globl	AES_ECB_decrypt_AESNI
1067AES_ECB_decrypt_AESNI PROC
1068;#	parameter	1:	rdi
1069;#	parameter	2:	rsi
1070;#	parameter	3:	rdx
1071;#	parameter	4:	rcx
1072;#	parameter	5:	r8d
1073
1074; save rdi and rsi to rax and r11, restore before ret
1075	mov rax,rdi
1076	mov r11,rsi
1077
1078; convert to what we had for att&t convention
1079	mov rdi,rcx
1080	mov rsi,rdx
1081	mov rdx,r8
1082	mov rcx,r9
1083	mov r8d,[rsp+40]
1084
1085; on microsoft xmm6-xmm15 are non volaitle, let's save on stack and restore at end
1086	sub rsp,8+4*16  ; 8 = align stack , 4 xmm9-12, 16 bytes each
1087	movdqa [rsp+0], xmm9
1088	movdqa [rsp+16], xmm10
1089	movdqa [rsp+32], xmm11
1090	movdqa [rsp+48], xmm12
1091
1092	mov	r10,rdx
1093	shr	rdx,4
1094	shl	r10,60
1095	je	DECB_NO_PARTS_4
1096	add	rdx,1
1097DECB_NO_PARTS_4:
1098	mov	r10,rdx
1099	shl	r10,62
1100	shr	r10,62
1101	shr	rdx,2
1102	je	DECB_REMAINDER_4
1103	sub	rsi,64
1104DECB_LOOP_4:
1105	movdqu  xmm1,[rdi]
1106	movdqu	xmm2,16[rdi]
1107	movdqu	xmm3,32[rdi]
1108	movdqu	xmm4,48[rdi]
1109	movdqa  xmm9,[rcx]
1110	movdqa	xmm10,16[rcx]
1111	movdqa	xmm11,32[rcx]
1112	movdqa	xmm12,48[rcx]
1113	pxor	xmm1,xmm9
1114	pxor	xmm2,xmm9
1115	pxor	xmm3,xmm9
1116	pxor	xmm4,xmm9
1117	aesdec	xmm1,xmm10
1118	aesdec	xmm2,xmm10
1119	aesdec	xmm3,xmm10
1120	aesdec	xmm4,xmm10
1121	aesdec	xmm1,xmm11
1122	aesdec	xmm2,xmm11
1123	aesdec	xmm3,xmm11
1124	aesdec	xmm4,xmm11
1125	aesdec	xmm1,xmm12
1126	aesdec	xmm2,xmm12
1127	aesdec	xmm3,xmm12
1128	aesdec	xmm4,xmm12
1129	movdqa	xmm9,64[rcx]
1130	movdqa	xmm10,80[rcx]
1131	movdqa	xmm11,96[rcx]
1132	movdqa	xmm12,112[rcx]
1133	aesdec	xmm1,xmm9
1134	aesdec	xmm2,xmm9
1135	aesdec	xmm3,xmm9
1136	aesdec	xmm4,xmm9
1137	aesdec	xmm1,xmm10
1138	aesdec	xmm2,xmm10
1139	aesdec	xmm3,xmm10
1140	aesdec	xmm4,xmm10
1141	aesdec	xmm1,xmm11
1142	aesdec	xmm2,xmm11
1143	aesdec	xmm3,xmm11
1144	aesdec	xmm4,xmm11
1145	aesdec	xmm1,xmm12
1146	aesdec	xmm2,xmm12
1147	aesdec	xmm3,xmm12
1148	aesdec	xmm4,xmm12
1149	movdqa	xmm9,128[rcx]
1150	movdqa	xmm10,144[rcx]
1151	movdqa	xmm11,160[rcx]
1152	cmp	r8d,12
1153	aesdec	xmm1,xmm9
1154	aesdec	xmm2,xmm9
1155	aesdec	xmm3,xmm9
1156	aesdec	xmm4,xmm9
1157	aesdec	xmm1,xmm10
1158	aesdec	xmm2,xmm10
1159	aesdec	xmm3,xmm10
1160	aesdec	xmm4,xmm10
1161	jb	DECB_LAST_4
1162	movdqa	xmm9,160[rcx]
1163	movdqa	xmm10,176[rcx]
1164	movdqa	xmm11,192[rcx]
1165	cmp	r8d,14
1166	aesdec	xmm1,xmm9
1167	aesdec	xmm2,xmm9
1168	aesdec	xmm3,xmm9
1169	aesdec	xmm4,xmm9
1170	aesdec	xmm1,xmm10
1171	aesdec	xmm2,xmm10
1172	aesdec	xmm3,xmm10
1173	aesdec	xmm4,xmm10
1174	jb	DECB_LAST_4
1175	movdqa	xmm9,192[rcx]
1176	movdqa	xmm10,208[rcx]
1177	movdqa	xmm11,224[rcx]
1178	aesdec	xmm1,xmm9
1179	aesdec	xmm2,xmm9
1180	aesdec	xmm3,xmm9
1181	aesdec	xmm4,xmm9
1182	aesdec	xmm1,xmm10
1183	aesdec	xmm2,xmm10
1184	aesdec	xmm3,xmm10
1185	aesdec	xmm4,xmm10
1186DECB_LAST_4:
1187	add	rdi,64
1188	add	rsi,64
1189	dec	rdx
1190	aesdeclast	xmm1,xmm11
1191	aesdeclast	xmm2,xmm11
1192	aesdeclast	xmm3,xmm11
1193	aesdeclast	xmm4,xmm11
1194	movdqu	[rsi],xmm1
1195	movdqu	16[rsi],xmm2
1196	movdqu	32[rsi],xmm3
1197	movdqu	48[rsi],xmm4
1198	jne	DECB_LOOP_4
1199	add	rsi,64
1200DECB_REMAINDER_4:
1201	cmp	r10,0
1202	je	DECB_END_4
1203DECB_LOOP_4_2:
1204	movdqu  xmm1,[rdi]
1205	add	rdi,16
1206	pxor	xmm1,[rcx]
1207	movdqu	xmm2,160[rcx]
1208	cmp	r8d,12
1209	aesdec	xmm1,16[rcx]
1210	aesdec	xmm1,32[rcx]
1211	aesdec	xmm1,48[rcx]
1212	aesdec	xmm1,64[rcx]
1213	aesdec	xmm1,80[rcx]
1214	aesdec	xmm1,96[rcx]
1215	aesdec	xmm1,112[rcx]
1216	aesdec	xmm1,128[rcx]
1217	aesdec	xmm1,144[rcx]
1218	jb	DECB_LAST_4_2
1219	cmp	r8d,14
1220	movdqu	xmm2,192[rcx]
1221	aesdec	xmm1,160[rcx]
1222	aesdec	xmm1,176[rcx]
1223	jb	DECB_LAST_4_2
1224	movdqu	xmm2,224[rcx]
1225	aesdec	xmm1,192[rcx]
1226	aesdec	xmm1,208[rcx]
1227DECB_LAST_4_2:
1228	aesdeclast	xmm1,xmm2
1229	movdqu	[rsi],xmm1
1230	add	rsi,16
1231	dec	r10
1232	jne	DECB_LOOP_4_2
1233DECB_END_4:
1234	; restore non volatile rdi,rsi
1235	mov rdi,rax
1236	mov rsi,r11
1237	; restore non volatile xmms from stack
1238	movdqa xmm9, [rsp+0]
1239	movdqa xmm10, [rsp+16]
1240	movdqa xmm11, [rsp+32]
1241	movdqa xmm12, [rsp+48]
1242	add rsp,8+4*16 ; 8 = align stack , 4 xmm9-12 16 bytes each
1243	ret
1244AES_ECB_decrypt_AESNI ENDP
1245
1246
1247
1248;	/*
1249;	void	,AES_128_Key_Expansion_AESNI[const	unsigned	char*userkey
1250;	unsigned	char*key_schedule]/
1251;	*/
1252;	.	align	16,0x90
1253;	.	globl	AES_128_Key_Expansion_AESNI
1254AES_128_Key_Expansion_AESNI PROC
1255;#	parameter	1:	rdi
1256;#	parameter	2:	rsi
1257
1258; save rdi and rsi to rax and r11, restore before ret
1259	mov rax,rdi
1260	mov r11,rsi
1261
1262; convert to what we had for att&t convention
1263	mov rdi,rcx
1264	mov rsi,rdx
1265
1266	mov	dword ptr 240[rsi],10
1267
1268	movdqu	xmm1,[rdi]
1269	movdqa	[rsi],xmm1
1270
1271
1272ASSISTS:
1273	aeskeygenassist	xmm2,xmm1,1
1274	call	PREPARE_ROUNDKEY_128
1275	movdqa	16[rsi],xmm1
1276
1277	aeskeygenassist	xmm2,xmm1,2
1278	call	PREPARE_ROUNDKEY_128
1279	movdqa	32[rsi],xmm1
1280
1281	aeskeygenassist	xmm2,xmm1,4
1282	call	PREPARE_ROUNDKEY_128
1283	movdqa	48[rsi],xmm1
1284
1285	aeskeygenassist	xmm2,xmm1,8
1286	call	PREPARE_ROUNDKEY_128
1287	movdqa	64[rsi],xmm1
1288
1289	aeskeygenassist	xmm2,xmm1,16
1290	call	PREPARE_ROUNDKEY_128
1291	movdqa	80[rsi],xmm1
1292
1293	aeskeygenassist	xmm2,xmm1,32
1294	call	PREPARE_ROUNDKEY_128
1295	movdqa	96[rsi],xmm1
1296
1297	aeskeygenassist	xmm2,xmm1,64
1298	call	PREPARE_ROUNDKEY_128
1299	movdqa	112[rsi],xmm1
1300	aeskeygenassist	xmm2,xmm1,80h
1301	call	PREPARE_ROUNDKEY_128
1302	movdqa	128[rsi],xmm1
1303	aeskeygenassist	xmm2,xmm1,1bh
1304	call	PREPARE_ROUNDKEY_128
1305	movdqa	144[rsi],xmm1
1306	aeskeygenassist	xmm2,xmm1,36h
1307	call	PREPARE_ROUNDKEY_128
1308	movdqa	160[rsi],xmm1
1309	; restore non volatile rdi,rsi
1310	mov rdi,rax
1311	mov rsi,r11
1312	ret
1313
1314PREPARE_ROUNDKEY_128:
1315	pshufd	xmm2,xmm2,255
1316	movdqa	xmm3,xmm1
1317	pslldq	xmm3,4
1318	pxor	xmm1,xmm3
1319	pslldq	xmm3,4
1320	pxor	xmm1,xmm3
1321	pslldq	xmm3,4
1322	pxor	xmm1,xmm3
1323	pxor	xmm1,xmm2
1324	ret
1325AES_128_Key_Expansion_AESNI ENDP
1326
1327;	/*
1328;	void	,AES_192_Key_Expansion_AESNI[const	unsigned	char*userkey
1329;	unsigned	char*key]
1330;	*/
1331;	.	globl	AES_192_Key_Expansion_AESNI
1332AES_192_Key_Expansion_AESNI PROC
1333;#	parameter	1:	rdi
1334;#	parameter	2:	rsi
1335
1336; save rdi and rsi to rax and r11, restore before ret
1337	mov rax,rdi
1338	mov r11,rsi
1339
1340; convert to what we had for att&t convention
1341    mov rdi,rcx
1342	mov rsi,rdx
1343
1344; on microsoft xmm6-xmm15 are non volaitle, let's save on stack and restore at end
1345	sub rsp,8+1*16  ; 8 = align stack , 1 xmm6, 16 bytes each
1346	movdqa [rsp+0], xmm6
1347
1348	movdqu  xmm1,[rdi]
1349	movq	xmm3,qword ptr 16[rdi]
1350	movdqa	[rsi],xmm1
1351	movdqa	xmm5,xmm3
1352
1353	aeskeygenassist	xmm2,xmm3,1h
1354	call	PREPARE_ROUNDKEY_192
1355	shufpd	xmm5,xmm1,0
1356	movdqa	16[rsi],xmm5
1357	movdqa	xmm6,xmm1
1358	shufpd	xmm6,xmm3,1
1359	movdqa	32[rsi],xmm6
1360
1361	aeskeygenassist	xmm2,xmm3,2h
1362	call	PREPARE_ROUNDKEY_192
1363	movdqa	48[rsi],xmm1
1364	movdqa	xmm5,xmm3
1365
1366	aeskeygenassist	xmm2,xmm3,4h
1367	call	PREPARE_ROUNDKEY_192
1368	shufpd	xmm5,xmm1,0
1369	movdqa	64[rsi],xmm5
1370	movdqa	xmm6,xmm1
1371	shufpd	xmm6,xmm3,1
1372	movdqa	80[rsi],xmm6
1373
1374	aeskeygenassist	xmm2,xmm3,8h
1375	call	PREPARE_ROUNDKEY_192
1376	movdqa	96[rsi],xmm1
1377	movdqa	xmm5,xmm3
1378
1379	aeskeygenassist	xmm2,xmm3,10h
1380	call	PREPARE_ROUNDKEY_192
1381	shufpd	xmm5,xmm1,0
1382	movdqa	112[rsi],xmm5
1383	movdqa	xmm6,xmm1
1384	shufpd	xmm6,xmm3,1
1385	movdqa	128[rsi],xmm6
1386
1387	aeskeygenassist	xmm2,xmm3,20h
1388	call	PREPARE_ROUNDKEY_192
1389	movdqa	144[rsi],xmm1
1390	movdqa	xmm5,xmm3
1391
1392	aeskeygenassist	xmm2,xmm3,40h
1393	call	PREPARE_ROUNDKEY_192
1394	shufpd	xmm5,xmm1,0
1395	movdqa	160[rsi],xmm5
1396	movdqa	xmm6,xmm1
1397	shufpd	xmm6,xmm3,1
1398	movdqa	176[rsi],xmm6
1399
1400	aeskeygenassist	xmm2,xmm3,80h
1401	call	PREPARE_ROUNDKEY_192
1402	movdqa	192[rsi],xmm1
1403	movdqa	208[rsi],xmm3
1404	; restore non volatile rdi,rsi
1405	mov rdi,rax
1406	mov rsi,r11
1407; restore non volatile xmms from stack
1408	movdqa xmm6, [rsp+0]
1409	add rsp,8+1*16 ; 8 = align stack , 1 xmm6 16 bytes each
1410	ret
1411
1412PREPARE_ROUNDKEY_192:
1413	pshufd	xmm2,xmm2,55h
1414	movdqu	xmm4,xmm1
1415	pslldq	xmm4,4
1416	pxor	xmm1,xmm4
1417
1418	pslldq	xmm4,4
1419	pxor	xmm1,xmm4
1420	pslldq	xmm4,4
1421	pxor	xmm1,xmm4
1422	pxor	xmm1,xmm2
1423	pshufd	xmm2,xmm1,0ffh
1424	movdqu	xmm4,xmm3
1425	pslldq	xmm4,4
1426	pxor	xmm3,xmm4
1427	pxor	xmm3,xmm2
1428	ret
1429AES_192_Key_Expansion_AESNI ENDP
1430
1431;	/*
1432;	void	,AES_256_Key_Expansion_AESNI[const	unsigned	char*userkey
1433;	unsigned	char*key]
1434;	*/
1435;	.	globl	AES_256_Key_Expansion_AESNI
1436AES_256_Key_Expansion_AESNI PROC
1437;#	parameter	1:	rdi
1438;#	parameter	2:	rsi
1439
1440; save rdi and rsi to rax and r11, restore before ret
1441	mov rax,rdi
1442	mov r11,rsi
1443
1444; convert to what we had for att&t convention
1445    mov rdi,rcx
1446	mov rsi,rdx
1447
1448	movdqu  xmm1,[rdi]
1449	movdqu	xmm3,16[rdi]
1450	movdqa	[rsi],xmm1
1451	movdqa	16[rsi],xmm3
1452
1453	aeskeygenassist	xmm2,xmm3,1h
1454	call	MAKE_RK256_a
1455	movdqa	32[rsi],xmm1
1456	aeskeygenassist	xmm2,xmm1,0h
1457	call	MAKE_RK256_b
1458	movdqa	48[rsi],xmm3
1459	aeskeygenassist	xmm2,xmm3,2h
1460	call	MAKE_RK256_a
1461	movdqa	64[rsi],xmm1
1462	aeskeygenassist	xmm2,xmm1,0h
1463	call	MAKE_RK256_b
1464	movdqa	80[rsi],xmm3
1465	aeskeygenassist	xmm2,xmm3,4h
1466	call	MAKE_RK256_a
1467	movdqa	96[rsi],xmm1
1468	aeskeygenassist	xmm2,xmm1,0h
1469	call	MAKE_RK256_b
1470	movdqa	112[rsi],xmm3
1471	aeskeygenassist	xmm2,xmm3,8h
1472	call	MAKE_RK256_a
1473	movdqa	128[rsi],xmm1
1474	aeskeygenassist	xmm2,xmm1,0h
1475	call	MAKE_RK256_b
1476	movdqa	144[rsi],xmm3
1477	aeskeygenassist	xmm2,xmm3,10h
1478	call	MAKE_RK256_a
1479	movdqa	160[rsi],xmm1
1480	aeskeygenassist	xmm2,xmm1,0h
1481	call	MAKE_RK256_b
1482	movdqa	176[rsi],xmm3
1483	aeskeygenassist	xmm2,xmm3,20h
1484	call	MAKE_RK256_a
1485	movdqa	192[rsi],xmm1
1486
1487	aeskeygenassist	xmm2,xmm1,0h
1488	call	MAKE_RK256_b
1489	movdqa	208[rsi],xmm3
1490	aeskeygenassist	xmm2,xmm3,40h
1491	call	MAKE_RK256_a
1492	movdqa	224[rsi],xmm1
1493
1494	; restore non volatile rdi,rsi
1495	mov rdi,rax
1496	mov rsi,r11
1497	ret
1498AES_256_Key_Expansion_AESNI ENDP
1499
1500MAKE_RK256_a:
1501	pshufd	xmm2,xmm2,0ffh
1502	movdqa	xmm4,xmm1
1503	pslldq	xmm4,4
1504	pxor	xmm1,xmm4
1505	pslldq	xmm4,4
1506	pxor	xmm1,xmm4
1507	pslldq	xmm4,4
1508	pxor	xmm1,xmm4
1509	pxor	xmm1,xmm2
1510	ret
1511
1512MAKE_RK256_b:
1513	pshufd	xmm2,xmm2,0aah
1514	movdqa	xmm4,xmm3
1515	pslldq	xmm4,4
1516	pxor	xmm3,xmm4
1517	pslldq	xmm4,4
1518	pxor	xmm3,xmm4
1519	pslldq	xmm4,4
1520	pxor	xmm3,xmm4
1521	pxor	xmm3,xmm2
1522	ret
1523
1524
1525IF fips_version GE 2
1526  fipsAb ENDS
1527ELSE
1528  _text ENDS
1529ENDIF
1530
1531END