luna - wolfssl/wolfcrypt/src/poly1305

   1; /* poly1305_asm.asm */
   2; /*
   3;  * Copyright (C) 2006-2026 wolfSSL Inc.
   4;  *
   5;  * This file is part of wolfSSL.
   6;  *
   7;  * wolfSSL is free software; you can redistribute it and/or modify
   8;  * it under the terms of the GNU General Public License as published by
   9;  * the Free Software Foundation; either version 3 of the License, or
  10;  * (at your option) any later version.
  11;  *
  12;  * wolfSSL is distributed in the hope that it will be useful,
  13;  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14;  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15;  * GNU General Public License for more details.
  16;  *
  17;  * You should have received a copy of the GNU General Public License
  18;  * along with this program; if not, write to the Free Software
  19;  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
  20;  */
  21
  22IF @Version LT 1200
  23; AVX2 instructions not recognized by old versions of MASM
  24IFNDEF NO_AVX2_SUPPORT
  25NO_AVX2_SUPPORT = 1
  26ENDIF
  27; MOVBE instruction not recognized by old versions of MASM
  28IFNDEF NO_MOVBE_SUPPORT
  29NO_MOVBE_SUPPORT = 1
  30ENDIF
  31ENDIF
  32
  33IFNDEF HAVE_INTEL_AVX1
  34HAVE_INTEL_AVX1 = 1
  35ENDIF
  36IFNDEF NO_AVX2_SUPPORT
  37HAVE_INTEL_AVX2 = 1
  38ENDIF
  39
  40IFNDEF _WIN64
  41_WIN64 = 1
  42ENDIF
  43
  44IFDEF HAVE_INTEL_AVX1
  45_TEXT SEGMENT READONLY PARA
  46poly1305_setkey_avx PROC
  47        push	r12
  48        push	r13
  49        mov	r12, 1152921487695413247
  50        mov	r13, 1152921487695413244
  51        mov	rax, QWORD PTR [rdx]
  52        mov	r8, QWORD PTR [rdx+8]
  53        mov	r9, QWORD PTR [rdx+16]
  54        mov	r10, QWORD PTR [rdx+24]
  55        and	rax, r12
  56        and	r8, r13
  57        mov	r12, rax
  58        mov	r13, r8
  59        xor	r11, r11
  60        mov	QWORD PTR [rcx], rax
  61        mov	QWORD PTR [rcx+8], r8
  62        mov	QWORD PTR [rcx+24], r11
  63        mov	QWORD PTR [rcx+32], r11
  64        mov	QWORD PTR [rcx+40], r11
  65        mov	QWORD PTR [rcx+48], r9
  66        mov	QWORD PTR [rcx+56], r10
  67        mov	QWORD PTR [rcx+352], r11
  68        mov	QWORD PTR [rcx+408], r11
  69        mov	QWORD PTR [rcx+360], rax
  70        mov	QWORD PTR [rcx+416], r8
  71        add	r12, rax
  72        add	r13, r8
  73        mov	QWORD PTR [rcx+368], r12
  74        mov	QWORD PTR [rcx+424], r13
  75        add	r12, rax
  76        add	r13, r8
  77        mov	QWORD PTR [rcx+376], r12
  78        mov	QWORD PTR [rcx+432], r13
  79        add	r12, rax
  80        add	r13, r8
  81        mov	QWORD PTR [rcx+384], r12
  82        mov	QWORD PTR [rcx+440], r13
  83        add	r12, rax
  84        add	r13, r8
  85        mov	QWORD PTR [rcx+392], r12
  86        mov	QWORD PTR [rcx+448], r13
  87        add	r12, rax
  88        add	r13, r8
  89        mov	QWORD PTR [rcx+400], r12
  90        mov	QWORD PTR [rcx+456], r13
  91        mov	QWORD PTR [rcx+608], r11
  92        mov	BYTE PTR [rcx+616], 1
  93        pop	r13
  94        pop	r12
  95        ret
  96poly1305_setkey_avx ENDP
  97_TEXT ENDS
  98_TEXT SEGMENT READONLY PARA
  99poly1305_block_avx PROC
 100        push	r15
 101        push	rbx
 102        push	r12
 103        push	r13
 104        push	r14
 105        mov	r15, QWORD PTR [rcx]
 106        mov	rbx, QWORD PTR [rcx+8]
 107        mov	r8, QWORD PTR [rcx+24]
 108        mov	r9, QWORD PTR [rcx+32]
 109        mov	r10, QWORD PTR [rcx+40]
 110        xor	r14, r14
 111        mov	r14b, BYTE PTR [rcx+616]
 112        ; h += m
 113        mov	r11, QWORD PTR [rdx]
 114        mov	r12, QWORD PTR [rdx+8]
 115        add	r8, r11
 116        adc	r9, r12
 117        mov	rax, rbx
 118        adc	r10, r14
 119        ; r[1] * h[0] => rdx, rax ==> t2, t1
 120        mul	r8
 121        mov	r12, rax
 122        mov	r13, rdx
 123        ; r[0] * h[1] => rdx, rax ++> t2, t1
 124        mov	rax, r15
 125        mul	r9
 126        add	r12, rax
 127        mov	rax, r15
 128        adc	r13, rdx
 129        ; r[0] * h[0] => rdx, rax ==> t4, t0
 130        mul	r8
 131        mov	r11, rax
 132        mov	r8, rdx
 133        ; r[1] * h[1] => rdx, rax =+> t3, t2
 134        mov	rax, rbx
 135        mul	r9
 136        ;   r[0] * h[2] +> t2
 137        add	r13, QWORD PTR [rcx+8*r10+352]
 138        mov	r14, rdx
 139        add	r12, r8
 140        adc	r13, rax
 141        ;   r[1] * h[2] +> t3
 142        adc	r14, QWORD PTR [rcx+8*r10+408]
 143        ; r * h in r14, r13, r12, r11
 144        ; h = (r * h) mod 2^130 - 5
 145        mov	r10, r13
 146        and	r13, -4
 147        and	r10, 3
 148        add	r11, r13
 149        mov	r8, r13
 150        adc	r12, r14
 151        adc	r10, 0
 152        shrd	r8, r14, 2
 153        shr	r14, 2
 154        add	r8, r11
 155        adc	r12, r14
 156        mov	r9, r12
 157        adc	r10, 0
 158        ; h in r10, r9, r8
 159        ; Store h to ctx
 160        mov	QWORD PTR [rcx+24], r8
 161        mov	QWORD PTR [rcx+32], r9
 162        mov	QWORD PTR [rcx+40], r10
 163        pop	r14
 164        pop	r13
 165        pop	r12
 166        pop	rbx
 167        pop	r15
 168        ret
 169poly1305_block_avx ENDP
 170_TEXT ENDS
 171_TEXT SEGMENT READONLY PARA
 172poly1305_blocks_avx PROC
 173        push	rdi
 174        push	rsi
 175        push	r15
 176        push	rbx
 177        push	r12
 178        push	r13
 179        push	r14
 180        mov	rdi, rcx
 181        mov	rsi, rdx
 182        mov	rcx, r8
 183        mov	r15, QWORD PTR [rdi]
 184        mov	rbx, QWORD PTR [rdi+8]
 185        mov	r8, QWORD PTR [rdi+24]
 186        mov	r9, QWORD PTR [rdi+32]
 187        mov	r10, QWORD PTR [rdi+40]
 188L_poly1305_avx_blocks_start:
 189        ; h += m
 190        mov	r11, QWORD PTR [rsi]
 191        mov	r12, QWORD PTR [rsi+8]
 192        add	r8, r11
 193        adc	r9, r12
 194        mov	rax, rbx
 195        adc	r10, 0
 196        ; r[1] * h[0] => rdx, rax ==> t2, t1
 197        mul	r8
 198        mov	r12, rax
 199        mov	r13, rdx
 200        ; r[0] * h[1] => rdx, rax ++> t2, t1
 201        mov	rax, r15
 202        mul	r9
 203        add	r12, rax
 204        mov	rax, r15
 205        adc	r13, rdx
 206        ; r[0] * h[0] => rdx, rax ==> t4, t0
 207        mul	r8
 208        mov	r11, rax
 209        mov	r8, rdx
 210        ; r[1] * h[1] => rdx, rax =+> t3, t2
 211        mov	rax, rbx
 212        mul	r9
 213        ;   r[0] * h[2] +> t2
 214        add	r13, QWORD PTR [rdi+8*r10+360]
 215        mov	r14, rdx
 216        add	r12, r8
 217        adc	r13, rax
 218        ;   r[1] * h[2] +> t3
 219        adc	r14, QWORD PTR [rdi+8*r10+416]
 220        ; r * h in r14, r13, r12, r11
 221        ; h = (r * h) mod 2^130 - 5
 222        mov	r10, r13
 223        and	r13, -4
 224        and	r10, 3
 225        add	r11, r13
 226        mov	r8, r13
 227        adc	r12, r14
 228        adc	r10, 0
 229        shrd	r8, r14, 2
 230        shr	r14, 2
 231        add	r8, r11
 232        adc	r12, r14
 233        mov	r9, r12
 234        adc	r10, 0
 235        ; h in r10, r9, r8
 236        ; Next block from message
 237        add	rsi, 16
 238        sub	rcx, 16
 239        jg	L_poly1305_avx_blocks_start
 240        ; Store h to ctx
 241        mov	QWORD PTR [rdi+24], r8
 242        mov	QWORD PTR [rdi+32], r9
 243        mov	QWORD PTR [rdi+40], r10
 244        pop	r14
 245        pop	r13
 246        pop	r12
 247        pop	rbx
 248        pop	r15
 249        pop	rsi
 250        pop	rdi
 251        ret
 252poly1305_blocks_avx ENDP
 253_TEXT ENDS
 254_TEXT SEGMENT READONLY PARA
 255poly1305_final_avx PROC
 256        push	rdi
 257        push	rbx
 258        push	r12
 259        mov	rdi, rcx
 260        mov	rbx, rdx
 261        mov	rax, QWORD PTR [rdi+608]
 262        test	rax, rax
 263        je	L_poly1305_avx_final_no_more
 264        mov	BYTE PTR [rdi+rax+480], 1
 265        jmp	L_poly1305_avx_final_cmp_rem
 266L_poly1305_avx_final_zero_rem:
 267        mov	BYTE PTR [rdi+rax+480], 0
 268L_poly1305_avx_final_cmp_rem:
 269        inc	al
 270        cmp	rax, 16
 271        jl	L_poly1305_avx_final_zero_rem
 272        mov	BYTE PTR [rdi+616], 0
 273        lea	rdx, QWORD PTR [rdi+480]
 274        call	poly1305_block_avx
 275L_poly1305_avx_final_no_more:
 276        mov	rax, QWORD PTR [rdi+24]
 277        mov	rdx, QWORD PTR [rdi+32]
 278        mov	rcx, QWORD PTR [rdi+40]
 279        mov	r11, QWORD PTR [rdi+48]
 280        mov	r12, QWORD PTR [rdi+56]
 281        ; h %= p
 282        ; h = (h + pad)
 283        ; mod 2^130 - 5
 284        mov	r8, rcx
 285        and	rcx, 3
 286        shr	r8, 2
 287        ;   Multiply by 5
 288        lea	r8, QWORD PTR [r8+4*r8+0]
 289        add	rax, r8
 290        adc	rdx, 0
 291        adc	rcx, 0
 292        ; Fixup when between (1 << 130) - 1 and (1 << 130) - 5
 293        mov	r8, rax
 294        mov	r9, rdx
 295        mov	r10, rcx
 296        add	r8, 5
 297        adc	r9, 0
 298        adc	r10, 0
 299        cmp	r10, 4
 300        cmove	rax, r8
 301        cmove	rdx, r9
 302        ; h += pad
 303        add	rax, r11
 304        adc	rdx, r12
 305        mov	QWORD PTR [rbx], rax
 306        mov	QWORD PTR [rbx+8], rdx
 307        ; Zero out r
 308        mov	QWORD PTR [rdi], 0
 309        mov	QWORD PTR [rdi+8], 0
 310        ; Zero out h
 311        mov	QWORD PTR [rdi+24], 0
 312        mov	QWORD PTR [rdi+32], 0
 313        mov	QWORD PTR [rdi+40], 0
 314        ; Zero out pad
 315        mov	QWORD PTR [rdi+48], 0
 316        mov	QWORD PTR [rdi+56], 0
 317        pop	r12
 318        pop	rbx
 319        pop	rdi
 320        ret
 321poly1305_final_avx ENDP
 322_TEXT ENDS
 323ENDIF
 324IFDEF HAVE_INTEL_AVX2
 325_TEXT SEGMENT READONLY PARA
 326poly1305_calc_powers_avx2 PROC
 327        push	r12
 328        push	r13
 329        push	r14
 330        push	r15
 331        push	rdi
 332        push	rsi
 333        push	rbx
 334        push	rbp
 335        mov	r8, QWORD PTR [rcx]
 336        mov	r9, QWORD PTR [rcx+8]
 337        xor	r10, r10
 338        ; Convert to 26 bits in 32
 339        mov	rax, r8
 340        mov	rdx, r8
 341        mov	rsi, r8
 342        mov	rbx, r9
 343        mov	rbp, r9
 344        shr	rdx, 26
 345        shrd	rsi, r9, 52
 346        shr	rbx, 14
 347        shrd	rbp, r10, 40
 348        and	rax, 67108863
 349        and	rdx, 67108863
 350        and	rsi, 67108863
 351        and	rbx, 67108863
 352        and	rbp, 67108863
 353        mov	DWORD PTR [rcx+224], eax
 354        mov	DWORD PTR [rcx+228], edx
 355        mov	DWORD PTR [rcx+232], esi
 356        mov	DWORD PTR [rcx+236], ebx
 357        mov	DWORD PTR [rcx+240], ebp
 358        mov	DWORD PTR [rcx+244], 0
 359        ; Square 128-bit
 360        mov	rax, r9
 361        mul	r8
 362        xor	r14, r14
 363        mov	r12, rax
 364        mov	r13, rdx
 365        add	r12, rax
 366        adc	r13, rdx
 367        adc	r14, 0
 368        mov	rax, r8
 369        mul	rax
 370        mov	r11, rax
 371        mov	rdi, rdx
 372        mov	rax, r9
 373        mul	rax
 374        add	r12, rdi
 375        adc	r13, rax
 376        adc	r14, rdx
 377        ; Reduce 256-bit to 130-bit
 378        mov	rax, r13
 379        mov	rdx, r14
 380        and	rax, -4
 381        and	r13, 3
 382        add	r11, rax
 383        adc	r12, rdx
 384        adc	r13, 0
 385        shrd	rax, rdx, 2
 386        shr	rdx, 2
 387        add	r11, rax
 388        adc	r12, rdx
 389        adc	r13, 0
 390        mov	rax, r13
 391        shr	rax, 2
 392        lea	rax, QWORD PTR [rax+4*rax+0]
 393        and	r13, 3
 394        add	r11, rax
 395        adc	r12, 0
 396        adc	r13, 0
 397        ; Convert to 26 bits in 32
 398        mov	rax, r11
 399        mov	rdx, r11
 400        mov	rsi, r11
 401        mov	rbx, r12
 402        mov	rbp, r12
 403        shr	rdx, 26
 404        shrd	rsi, r12, 52
 405        shr	rbx, 14
 406        shrd	rbp, r13, 40
 407        and	rax, 67108863
 408        and	rdx, 67108863
 409        and	rsi, 67108863
 410        and	rbx, 67108863
 411        and	rbp, 67108863
 412        mov	DWORD PTR [rcx+256], eax
 413        mov	DWORD PTR [rcx+260], edx
 414        mov	DWORD PTR [rcx+264], esi
 415        mov	DWORD PTR [rcx+268], ebx
 416        mov	DWORD PTR [rcx+272], ebp
 417        mov	DWORD PTR [rcx+276], 0
 418        ; Multiply 128-bit by 130-bit
 419        ;   r1[0] * r2[0]
 420        mov	rax, r8
 421        mul	r11
 422        mov	r14, rax
 423        mov	r15, rdx
 424        ;   r1[0] * r2[1]
 425        mov	rax, r8
 426        mul	r12
 427        mov	rdi, 0
 428        add	r15, rax
 429        adc	rdi, rdx
 430        ;   r1[1] * r2[0]
 431        mov	rax, r9
 432        mul	r11
 433        mov	rsi, 0
 434        add	r15, rax
 435        adc	rdi, rdx
 436        adc	rsi, 0
 437        ;   r1[0] * r2[2]
 438        mov	rax, r8
 439        mul	r13
 440        add	rdi, rax
 441        adc	rsi, rdx
 442        ;   r1[1] * r2[1]
 443        mov	rax, r9
 444        mul	r12
 445        mov	rbx, 0
 446        add	rdi, rax
 447        adc	rsi, rdx
 448        adc	rbx, 0
 449        ;   r1[1] * r2[2]
 450        mov	rax, r9
 451        mul	r13
 452        add	rsi, rax
 453        adc	rbx, rdx
 454        ; Reduce 260-bit to 130-bit
 455        mov	rax, rdi
 456        mov	rdx, rsi
 457        mov	rbx, rbx
 458        and	rax, -4
 459        and	rdi, 3
 460        add	r14, rax
 461        adc	r15, rdx
 462        adc	rdi, rbx
 463        shrd	rax, rdx, 2
 464        shrd	rdx, rbx, 2
 465        shr	rbx, 2
 466        add	r14, rax
 467        adc	r15, rdx
 468        adc	rdi, rbx
 469        mov	rax, rdi
 470        and	rdi, 3
 471        shr	rax, 2
 472        lea	rax, QWORD PTR [rax+4*rax+0]
 473        add	r14, rax
 474        adc	r15, 0
 475        adc	rdi, 0
 476        ; Convert to 26 bits in 32
 477        mov	rax, r14
 478        mov	rdx, r14
 479        mov	rsi, r14
 480        mov	rbx, r15
 481        mov	rbp, r15
 482        shr	rdx, 26
 483        shrd	rsi, r15, 52
 484        shr	rbx, 14
 485        shrd	rbp, rdi, 40
 486        and	rax, 67108863
 487        and	rdx, 67108863
 488        and	rsi, 67108863
 489        and	rbx, 67108863
 490        and	rbp, 67108863
 491        mov	DWORD PTR [rcx+288], eax
 492        mov	DWORD PTR [rcx+292], edx
 493        mov	DWORD PTR [rcx+296], esi
 494        mov	DWORD PTR [rcx+300], ebx
 495        mov	DWORD PTR [rcx+304], ebp
 496        mov	DWORD PTR [rcx+308], 0
 497        ; Square 130-bit
 498        mov	rax, r12
 499        mul	r11
 500        xor	r14, r14
 501        mov	r9, rax
 502        mov	r10, rdx
 503        add	r9, rax
 504        adc	r10, rdx
 505        adc	r14, 0
 506        mov	rax, r11
 507        mul	rax
 508        mov	r8, rax
 509        mov	rdi, rdx
 510        mov	rax, r12
 511        mul	rax
 512        add	r9, rdi
 513        adc	r10, rax
 514        adc	r14, rdx
 515        mov	rax, r13
 516        mul	rax
 517        mov	r15, rax
 518        mov	rax, r13
 519        mul	r11
 520        add	r10, rax
 521        adc	r14, rdx
 522        adc	r15, 0
 523        add	r10, rax
 524        adc	r14, rdx
 525        adc	r15, 0
 526        mov	rax, r13
 527        mul	r12
 528        add	r14, rax
 529        adc	r15, rdx
 530        add	r14, rax
 531        adc	r15, rdx
 532        ; Reduce 260-bit to 130-bit
 533        mov	rax, r10
 534        mov	rdx, r14
 535        mov	rdi, r15
 536        and	rax, -4
 537        and	r10, 3
 538        add	r8, rax
 539        adc	r9, rdx
 540        adc	r10, rdi
 541        shrd	rax, rdx, 2
 542        shrd	rdx, rdi, 2
 543        shr	rdi, 2
 544        add	r8, rax
 545        adc	r9, rdx
 546        adc	r10, rdi
 547        mov	rax, r10
 548        and	r10, 3
 549        shr	rax, 2
 550        lea	rax, QWORD PTR [rax+4*rax+0]
 551        add	r8, rax
 552        adc	r9, 0
 553        adc	r10, 0
 554        ; Convert to 26 bits in 32
 555        mov	rax, r8
 556        mov	rdx, r8
 557        mov	rsi, r8
 558        mov	rbx, r9
 559        mov	rbp, r9
 560        shr	rdx, 26
 561        shrd	rsi, r9, 52
 562        shr	rbx, 14
 563        shrd	rbp, r10, 40
 564        and	rax, 67108863
 565        and	rdx, 67108863
 566        and	rsi, 67108863
 567        and	rbx, 67108863
 568        and	rbp, 67108863
 569        mov	DWORD PTR [rcx+320], eax
 570        mov	DWORD PTR [rcx+324], edx
 571        mov	DWORD PTR [rcx+328], esi
 572        mov	DWORD PTR [rcx+332], ebx
 573        mov	DWORD PTR [rcx+336], ebp
 574        mov	DWORD PTR [rcx+340], 0
 575        pop	rbp
 576        pop	rbx
 577        pop	rsi
 578        pop	rdi
 579        pop	r15
 580        pop	r14
 581        pop	r13
 582        pop	r12
 583        ret
 584poly1305_calc_powers_avx2 ENDP
 585_TEXT ENDS
 586_TEXT SEGMENT READONLY PARA
 587poly1305_setkey_avx2 PROC
 588        call	poly1305_setkey_avx
 589        vpxor	ymm0, ymm0, ymm0
 590        vmovdqu	YMMWORD PTR [rcx+64], ymm0
 591        vmovdqu	YMMWORD PTR [rcx+96], ymm0
 592        vmovdqu	YMMWORD PTR [rcx+128], ymm0
 593        vmovdqu	YMMWORD PTR [rcx+160], ymm0
 594        vmovdqu	YMMWORD PTR [rcx+192], ymm0
 595        mov	QWORD PTR [rcx+608], 0
 596        mov	WORD PTR [rcx+616], 0
 597        ret
 598poly1305_setkey_avx2 ENDP
 599_TEXT ENDS
 600_DATA SEGMENT
 601ALIGN 16
 602L_poly1305_avx2_blocks_mask QWORD \
 603     0000000003ffffffh,  0000000003ffffffh,
 604     0000000003ffffffh,  0000000003ffffffh
 605ptr_L_poly1305_avx2_blocks_mask QWORD L_poly1305_avx2_blocks_mask
 606_DATA ENDS
 607_DATA SEGMENT
 608ALIGN 16
 609L_poly1305_avx2_blocks_hibit QWORD \
 610     0000000001000000h,  0000000001000000h,
 611     0000000001000000h,  0000000001000000h
 612ptr_L_poly1305_avx2_blocks_hibit QWORD L_poly1305_avx2_blocks_hibit
 613_DATA ENDS
 614_TEXT SEGMENT READONLY PARA
 615poly1305_blocks_avx2 PROC
 616        push	r12
 617        push	rdi
 618        push	rsi
 619        push	rbx
 620        push	r13
 621        push	r14
 622        mov	rdi, rcx
 623        mov	rsi, rdx
 624        mov	rdx, r8
 625        sub	rsp, 480
 626        vmovdqu	OWORD PTR [rsp+320], xmm6
 627        vmovdqu	OWORD PTR [rsp+336], xmm7
 628        vmovdqu	OWORD PTR [rsp+352], xmm8
 629        vmovdqu	OWORD PTR [rsp+368], xmm9
 630        vmovdqu	OWORD PTR [rsp+384], xmm10
 631        vmovdqu	OWORD PTR [rsp+400], xmm11
 632        vmovdqu	OWORD PTR [rsp+416], xmm12
 633        vmovdqu	OWORD PTR [rsp+432], xmm13
 634        vmovdqu	OWORD PTR [rsp+448], xmm14
 635        vmovdqu	OWORD PTR [rsp+464], xmm15
 636        mov	r13, QWORD PTR [ptr_L_poly1305_avx2_blocks_mask]
 637        mov	r14, QWORD PTR [ptr_L_poly1305_avx2_blocks_hibit]
 638        mov	rcx, rsp
 639        and	rcx, -32
 640        add	rcx, 32
 641        vpxor	ymm15, ymm15, ymm15
 642        mov	rbx, rcx
 643        lea	rax, QWORD PTR [rdi+64]
 644        add	rbx, 160
 645        cmp	WORD PTR [rdi+616], 0
 646        jne	L_poly1305_avx2_blocks_begin_h
 647        ; Load the message data
 648        vmovdqu	ymm0, YMMWORD PTR [rsi]
 649        vmovdqu	ymm1, YMMWORD PTR [rsi+32]
 650        vperm2i128	ymm2, ymm0, ymm1, 32
 651        vperm2i128	ymm0, ymm0, ymm1, 49
 652        vpunpckldq	ymm1, ymm2, ymm0
 653        vpunpckhdq	ymm3, ymm2, ymm0
 654        vpunpckldq	ymm0, ymm1, ymm15
 655        vpunpckhdq	ymm1, ymm1, ymm15
 656        vpunpckldq	ymm2, ymm3, ymm15
 657        vpunpckhdq	ymm3, ymm3, ymm15
 658        vmovdqu	ymm4, YMMWORD PTR [r14]
 659        vpsllq	ymm1, ymm1, 6
 660        vpsllq	ymm2, ymm2, 12
 661        vpsllq	ymm3, ymm3, 18
 662        vmovdqu	ymm14, YMMWORD PTR [r13]
 663        ; Reduce, in place, the message data
 664        vpsrlq	ymm10, ymm0, 26
 665        vpsrlq	ymm11, ymm3, 26
 666        vpand	ymm0, ymm0, ymm14
 667        vpand	ymm3, ymm3, ymm14
 668        vpaddq	ymm1, ymm10, ymm1
 669        vpaddq	ymm4, ymm11, ymm4
 670        vpsrlq	ymm10, ymm1, 26
 671        vpsrlq	ymm11, ymm4, 26
 672        vpand	ymm1, ymm1, ymm14
 673        vpand	ymm4, ymm4, ymm14
 674        vpaddq	ymm2, ymm10, ymm2
 675        vpslld	ymm12, ymm11, 2
 676        vpaddd	ymm12, ymm11, ymm12
 677        vpsrlq	ymm10, ymm2, 26
 678        vpaddq	ymm0, ymm12, ymm0
 679        vpsrlq	ymm11, ymm0, 26
 680        vpand	ymm2, ymm2, ymm14
 681        vpand	ymm0, ymm0, ymm14
 682        vpaddq	ymm3, ymm10, ymm3
 683        vpaddq	ymm1, ymm11, ymm1
 684        vpsrlq	ymm10, ymm3, 26
 685        vpand	ymm3, ymm3, ymm14
 686        vpaddq	ymm4, ymm10, ymm4
 687        add	rsi, 64
 688        sub	rdx, 64
 689        jz	L_poly1305_avx2_blocks_store
 690        jmp	L_poly1305_avx2_blocks_load_r4
 691L_poly1305_avx2_blocks_begin_h:
 692        ; Load the H values.
 693        vmovdqu	ymm0, YMMWORD PTR [rax]
 694        vmovdqu	ymm1, YMMWORD PTR [rax+32]
 695        vmovdqu	ymm2, YMMWORD PTR [rax+64]
 696        vmovdqu	ymm3, YMMWORD PTR [rax+96]
 697        vmovdqu	ymm4, YMMWORD PTR [rax+128]
 698        ; Check if there is a power of r to load - otherwise use r^4.
 699        cmp	BYTE PTR [rdi+616], 0
 700        je	L_poly1305_avx2_blocks_load_r4
 701        ; Load the 4 powers of r - r^4, r^3, r^2, r^1.
 702        vmovdqu	ymm8, YMMWORD PTR [rdi+224]
 703        vmovdqu	ymm7, YMMWORD PTR [rdi+256]
 704        vmovdqu	ymm6, YMMWORD PTR [rdi+288]
 705        vmovdqu	ymm5, YMMWORD PTR [rdi+320]
 706        vpermq	ymm5, ymm5, 216
 707        vpermq	ymm6, ymm6, 216
 708        vpermq	ymm7, ymm7, 216
 709        vpermq	ymm8, ymm8, 216
 710        vpunpcklqdq	ymm10, ymm5, ymm6
 711        vpunpckhqdq	ymm11, ymm5, ymm6
 712        vpunpcklqdq	ymm12, ymm7, ymm8
 713        vpunpckhqdq	ymm13, ymm7, ymm8
 714        vperm2i128	ymm5, ymm10, ymm12, 32
 715        vperm2i128	ymm7, ymm10, ymm12, 49
 716        vperm2i128	ymm9, ymm11, ymm13, 32
 717        vpsrlq	ymm6, ymm5, 32
 718        vpsrlq	ymm8, ymm7, 32
 719        jmp	L_poly1305_avx2_blocks_mul_5
 720L_poly1305_avx2_blocks_load_r4:
 721        ; Load r^4 into all four positions.
 722        vmovdqu	ymm13, YMMWORD PTR [rdi+320]
 723        vpermq	ymm5, ymm13, 0
 724        vpsrlq	ymm14, ymm13, 32
 725        vpermq	ymm7, ymm13, 85
 726        vpermq	ymm9, ymm13, 170
 727        vpermq	ymm6, ymm14, 0
 728        vpermq	ymm8, ymm14, 85
 729L_poly1305_avx2_blocks_mul_5:
 730        ; Multiply top 4 26-bit values of all four H by 5
 731        vpslld	ymm10, ymm6, 2
 732        vpslld	ymm11, ymm7, 2
 733        vpslld	ymm12, ymm8, 2
 734        vpslld	ymm13, ymm9, 2
 735        vpaddq	ymm10, ymm6, ymm10
 736        vpaddq	ymm11, ymm7, ymm11
 737        vpaddq	ymm12, ymm8, ymm12
 738        vpaddq	ymm13, ymm9, ymm13
 739        ; Store powers of r and multiple of 5 for use in multiply.
 740        vmovdqa	YMMWORD PTR [rbx], ymm10
 741        vmovdqa	YMMWORD PTR [rbx+32], ymm11
 742        vmovdqa	YMMWORD PTR [rbx+64], ymm12
 743        vmovdqa	YMMWORD PTR [rbx+96], ymm13
 744        vmovdqa	YMMWORD PTR [rcx], ymm5
 745        vmovdqa	YMMWORD PTR [rcx+32], ymm6
 746        vmovdqa	YMMWORD PTR [rcx+64], ymm7
 747        vmovdqa	YMMWORD PTR [rcx+96], ymm8
 748        vmovdqa	YMMWORD PTR [rcx+128], ymm9
 749        vmovdqu	ymm14, YMMWORD PTR [r13]
 750        ; If not finished then loop over data
 751        cmp	BYTE PTR [rdi+616], 1
 752        jne	L_poly1305_avx2_blocks_start
 753        ; Do last multiply, reduce, add the four H together and move to
 754        ; 32-bit registers
 755        vpmuludq	ymm5, ymm4, [rbx]
 756        vpmuludq	ymm10, ymm3, [rbx+32]
 757        vpmuludq	ymm6, ymm4, [rbx+32]
 758        vpmuludq	ymm11, ymm3, [rbx+64]
 759        vpmuludq	ymm7, ymm4, [rbx+64]
 760        vpaddq	ymm5, ymm10, ymm5
 761        vpmuludq	ymm12, ymm2, [rbx+64]
 762        vpmuludq	ymm8, ymm4, [rbx+96]
 763        vpaddq	ymm6, ymm11, ymm6
 764        vpmuludq	ymm13, ymm1, [rbx+96]
 765        vpmuludq	ymm10, ymm2, [rbx+96]
 766        vpaddq	ymm5, ymm12, ymm5
 767        vpmuludq	ymm11, ymm3, [rbx+96]
 768        vpmuludq	ymm12, ymm3, [rcx]
 769        vpaddq	ymm5, ymm13, ymm5
 770        vpmuludq	ymm9, ymm4, [rcx]
 771        vpaddq	ymm6, ymm10, ymm6
 772        vpmuludq	ymm13, ymm0, [rcx]
 773        vpaddq	ymm7, ymm11, ymm7
 774        vpmuludq	ymm10, ymm1, [rcx]
 775        vpaddq	ymm8, ymm12, ymm8
 776        vpmuludq	ymm11, ymm2, [rcx]
 777        vpmuludq	ymm12, ymm2, [rcx+32]
 778        vpaddq	ymm5, ymm13, ymm5
 779        vpmuludq	ymm13, ymm3, [rcx+32]
 780        vpaddq	ymm6, ymm10, ymm6
 781        vpmuludq	ymm10, ymm0, [rcx+32]
 782        vpaddq	ymm7, ymm11, ymm7
 783        vpmuludq	ymm11, ymm1, [rcx+32]
 784        vpaddq	ymm8, ymm12, ymm8
 785        vpmuludq	ymm12, ymm1, [rcx+64]
 786        vpaddq	ymm9, ymm13, ymm9
 787        vpmuludq	ymm13, ymm2, [rcx+64]
 788        vpaddq	ymm6, ymm10, ymm6
 789        vpmuludq	ymm10, ymm0, [rcx+64]
 790        vpaddq	ymm7, ymm11, ymm7
 791        vpmuludq	ymm11, ymm0, [rcx+96]
 792        vpaddq	ymm8, ymm12, ymm8
 793        vpmuludq	ymm12, ymm1, [rcx+96]
 794        vpaddq	ymm9, ymm13, ymm9
 795        vpaddq	ymm7, ymm10, ymm7
 796        vpmuludq	ymm13, ymm0, [rcx+128]
 797        vpaddq	ymm8, ymm11, ymm8
 798        vpaddq	ymm9, ymm12, ymm9
 799        vpaddq	ymm9, ymm13, ymm9
 800        vpsrlq	ymm10, ymm5, 26
 801        vpsrlq	ymm11, ymm8, 26
 802        vpand	ymm5, ymm5, ymm14
 803        vpand	ymm8, ymm8, ymm14
 804        vpaddq	ymm6, ymm10, ymm6
 805        vpaddq	ymm9, ymm11, ymm9
 806        vpsrlq	ymm10, ymm6, 26
 807        vpsrlq	ymm11, ymm9, 26
 808        vpand	ymm1, ymm6, ymm14
 809        vpand	ymm4, ymm9, ymm14
 810        vpaddq	ymm7, ymm10, ymm7
 811        vpslld	ymm12, ymm11, 2
 812        vpaddd	ymm12, ymm11, ymm12
 813        vpsrlq	ymm10, ymm7, 26
 814        vpaddq	ymm5, ymm12, ymm5
 815        vpsrlq	ymm11, ymm5, 26
 816        vpand	ymm2, ymm7, ymm14
 817        vpand	ymm0, ymm5, ymm14
 818        vpaddq	ymm8, ymm10, ymm8
 819        vpaddq	ymm1, ymm11, ymm1
 820        vpsrlq	ymm10, ymm8, 26
 821        vpand	ymm3, ymm8, ymm14
 822        vpaddq	ymm4, ymm10, ymm4
 823        vpsrldq	ymm5, ymm0, 8
 824        vpsrldq	ymm6, ymm1, 8
 825        vpsrldq	ymm7, ymm2, 8
 826        vpsrldq	ymm8, ymm3, 8
 827        vpsrldq	ymm9, ymm4, 8
 828        vpaddq	ymm0, ymm5, ymm0
 829        vpaddq	ymm1, ymm6, ymm1
 830        vpaddq	ymm2, ymm7, ymm2
 831        vpaddq	ymm3, ymm8, ymm3
 832        vpaddq	ymm4, ymm9, ymm4
 833        vpermq	ymm5, ymm0, 2
 834        vpermq	ymm6, ymm1, 2
 835        vpermq	ymm7, ymm2, 2
 836        vpermq	ymm8, ymm3, 2
 837        vpermq	ymm9, ymm4, 2
 838        vpaddq	ymm0, ymm5, ymm0
 839        vpaddq	ymm1, ymm6, ymm1
 840        vpaddq	ymm2, ymm7, ymm2
 841        vpaddq	ymm3, ymm8, ymm3
 842        vpaddq	ymm4, ymm9, ymm4
 843        vmovd	r8d, xmm0
 844        vmovd	r9d, xmm1
 845        vmovd	r10d, xmm2
 846        vmovd	r11d, xmm3
 847        vmovd	r12d, xmm4
 848        jmp	L_poly1305_avx2_blocks_end_calc
 849L_poly1305_avx2_blocks_start:
 850        vmovdqu	ymm5, YMMWORD PTR [rsi]
 851        vmovdqu	ymm6, YMMWORD PTR [rsi+32]
 852        vperm2i128	ymm7, ymm5, ymm6, 32
 853        vperm2i128	ymm5, ymm5, ymm6, 49
 854        vpunpckldq	ymm6, ymm7, ymm5
 855        vpunpckhdq	ymm8, ymm7, ymm5
 856        vpunpckldq	ymm5, ymm6, ymm15
 857        vpunpckhdq	ymm6, ymm6, ymm15
 858        vpunpckldq	ymm7, ymm8, ymm15
 859        vpunpckhdq	ymm8, ymm8, ymm15
 860        vmovdqu	ymm9, YMMWORD PTR [r14]
 861        vpsllq	ymm6, ymm6, 6
 862        vpsllq	ymm7, ymm7, 12
 863        vpsllq	ymm8, ymm8, 18
 864        vpmuludq	ymm10, ymm4, [rbx]
 865        vpaddq	ymm5, ymm10, ymm5
 866        vpmuludq	ymm10, ymm3, [rbx+32]
 867        vpmuludq	ymm11, ymm4, [rbx+32]
 868        vpaddq	ymm6, ymm11, ymm6
 869        vpmuludq	ymm11, ymm3, [rbx+64]
 870        vpmuludq	ymm12, ymm4, [rbx+64]
 871        vpaddq	ymm7, ymm12, ymm7
 872        vpaddq	ymm5, ymm10, ymm5
 873        vpmuludq	ymm12, ymm2, [rbx+64]
 874        vpmuludq	ymm13, ymm4, [rbx+96]
 875        vpaddq	ymm8, ymm13, ymm8
 876        vpaddq	ymm6, ymm11, ymm6
 877        vpmuludq	ymm13, ymm1, [rbx+96]
 878        vpmuludq	ymm10, ymm2, [rbx+96]
 879        vpaddq	ymm5, ymm12, ymm5
 880        vpmuludq	ymm11, ymm3, [rbx+96]
 881        vpmuludq	ymm12, ymm3, [rcx]
 882        vpaddq	ymm5, ymm13, ymm5
 883        vpmuludq	ymm13, ymm4, [rcx]
 884        vpaddq	ymm9, ymm13, ymm9
 885        vpaddq	ymm6, ymm10, ymm6
 886        vpmuludq	ymm13, ymm0, [rcx]
 887        vpaddq	ymm7, ymm11, ymm7
 888        vpmuludq	ymm10, ymm1, [rcx]
 889        vpaddq	ymm8, ymm12, ymm8
 890        vpmuludq	ymm11, ymm2, [rcx]
 891        vpmuludq	ymm12, ymm2, [rcx+32]
 892        vpaddq	ymm5, ymm13, ymm5
 893        vpmuludq	ymm13, ymm3, [rcx+32]
 894        vpaddq	ymm6, ymm10, ymm6
 895        vpmuludq	ymm10, ymm0, [rcx+32]
 896        vpaddq	ymm7, ymm11, ymm7
 897        vpmuludq	ymm11, ymm1, [rcx+32]
 898        vpaddq	ymm8, ymm12, ymm8
 899        vpmuludq	ymm12, ymm1, [rcx+64]
 900        vpaddq	ymm9, ymm13, ymm9
 901        vpmuludq	ymm13, ymm2, [rcx+64]
 902        vpaddq	ymm6, ymm10, ymm6
 903        vpmuludq	ymm10, ymm0, [rcx+64]
 904        vpaddq	ymm7, ymm11, ymm7
 905        vpmuludq	ymm11, ymm0, [rcx+96]
 906        vpaddq	ymm8, ymm12, ymm8
 907        vpmuludq	ymm12, ymm1, [rcx+96]
 908        vpaddq	ymm9, ymm13, ymm9
 909        vpaddq	ymm7, ymm10, ymm7
 910        vpmuludq	ymm13, ymm0, [rcx+128]
 911        vpaddq	ymm8, ymm11, ymm8
 912        vpaddq	ymm9, ymm12, ymm9
 913        vpaddq	ymm9, ymm13, ymm9
 914        vpsrlq	ymm10, ymm5, 26
 915        vpsrlq	ymm11, ymm8, 26
 916        vpand	ymm5, ymm5, ymm14
 917        vpand	ymm8, ymm8, ymm14
 918        vpaddq	ymm6, ymm10, ymm6
 919        vpaddq	ymm9, ymm11, ymm9
 920        vpsrlq	ymm10, ymm6, 26
 921        vpsrlq	ymm11, ymm9, 26
 922        vpand	ymm1, ymm6, ymm14
 923        vpand	ymm4, ymm9, ymm14
 924        vpaddq	ymm7, ymm10, ymm7
 925        vpslld	ymm12, ymm11, 2
 926        vpaddd	ymm12, ymm11, ymm12
 927        vpsrlq	ymm10, ymm7, 26
 928        vpaddq	ymm5, ymm12, ymm5
 929        vpsrlq	ymm11, ymm5, 26
 930        vpand	ymm2, ymm7, ymm14
 931        vpand	ymm0, ymm5, ymm14
 932        vpaddq	ymm8, ymm10, ymm8
 933        vpaddq	ymm1, ymm11, ymm1
 934        vpsrlq	ymm10, ymm8, 26
 935        vpand	ymm3, ymm8, ymm14
 936        vpaddq	ymm4, ymm10, ymm4
 937        add	rsi, 64
 938        sub	rdx, 64
 939        jnz	L_poly1305_avx2_blocks_start
 940L_poly1305_avx2_blocks_store:
 941        ; Store four H values - state
 942        vmovdqu	YMMWORD PTR [rax], ymm0
 943        vmovdqu	YMMWORD PTR [rax+32], ymm1
 944        vmovdqu	YMMWORD PTR [rax+64], ymm2
 945        vmovdqu	YMMWORD PTR [rax+96], ymm3
 946        vmovdqu	YMMWORD PTR [rax+128], ymm4
 947L_poly1305_avx2_blocks_end_calc:
 948        cmp	BYTE PTR [rdi+616], 0
 949        je	L_poly1305_avx2_blocks_complete
 950        mov	rax, r8
 951        mov	rdx, r10
 952        mov	rcx, r12
 953        shr	rdx, 12
 954        shr	rcx, 24
 955        shl	r9, 26
 956        shl	r10, 52
 957        shl	r11, 14
 958        shl	r12, 40
 959        add	rax, r9
 960        adc	rax, r10
 961        adc	rdx, r11
 962        adc	rdx, r12
 963        adc	rcx, 0
 964        mov	r8, rcx
 965        and	rcx, 3
 966        shr	r8, 2
 967        lea	r8, QWORD PTR [r8+4*r8+0]
 968        add	rax, r8
 969        adc	rdx, 0
 970        adc	rcx, 0
 971        mov	QWORD PTR [rdi+24], rax
 972        mov	QWORD PTR [rdi+32], rdx
 973        mov	QWORD PTR [rdi+40], rcx
 974L_poly1305_avx2_blocks_complete:
 975        mov	BYTE PTR [rdi+617], 1
 976        vzeroupper
 977        vmovdqu	xmm6, OWORD PTR [rsp+320]
 978        vmovdqu	xmm7, OWORD PTR [rsp+336]
 979        vmovdqu	xmm8, OWORD PTR [rsp+352]
 980        vmovdqu	xmm9, OWORD PTR [rsp+368]
 981        vmovdqu	xmm10, OWORD PTR [rsp+384]
 982        vmovdqu	xmm11, OWORD PTR [rsp+400]
 983        vmovdqu	xmm12, OWORD PTR [rsp+416]
 984        vmovdqu	xmm13, OWORD PTR [rsp+432]
 985        vmovdqu	xmm14, OWORD PTR [rsp+448]
 986        vmovdqu	xmm15, OWORD PTR [rsp+464]
 987        add	rsp, 480
 988        pop	r14
 989        pop	r13
 990        pop	rbx
 991        pop	rsi
 992        pop	rdi
 993        pop	r12
 994        ret
 995poly1305_blocks_avx2 ENDP
 996_TEXT ENDS
 997_TEXT SEGMENT READONLY PARA
 998poly1305_final_avx2 PROC
 999        push	rdi
1000        push	rsi
1001        mov	rdi, rcx
1002        mov	rsi, rdx
1003        mov	BYTE PTR [rdi+616], 1
1004        mov	cl, BYTE PTR [rdi+617]
1005        cmp	cl, 0
1006        je	L_poly1305_avx2_final_done_blocks_X4
1007        push	rsi
1008        mov	r8, 64
1009        xor	rdx, rdx
1010        mov	rcx, rdi
1011        call	poly1305_blocks_avx2
1012        pop	rsi
1013L_poly1305_avx2_final_done_blocks_X4:
1014        mov	rax, QWORD PTR [rdi+608]
1015        mov	rcx, rax
1016        and	rcx, -16
1017        cmp	cl, 0
1018        je	L_poly1305_avx2_final_done_blocks
1019        push	rcx
1020        push	rax
1021        push	rsi
1022        mov	r8, rcx
1023        lea	rdx, QWORD PTR [rdi+480]
1024        mov	rcx, rdi
1025        call	poly1305_blocks_avx
1026        pop	rsi
1027        pop	rax
1028        pop	rcx
1029L_poly1305_avx2_final_done_blocks:
1030        sub	QWORD PTR [rdi+608], rcx
1031        xor	rdx, rdx
1032        jmp	L_poly1305_avx2_final_cmp_copy
1033L_poly1305_avx2_final_start_copy:
1034        mov	r8b, BYTE PTR [rdi+rcx+480]
1035        mov	BYTE PTR [rdi+rdx+480], r8b
1036        inc	cl
1037        inc	dl
1038L_poly1305_avx2_final_cmp_copy:
1039        cmp	al, cl
1040        jne	L_poly1305_avx2_final_start_copy
1041        mov	rcx, rdi
1042        mov	rdx, rsi
1043        call	poly1305_final_avx
1044        vpxor	ymm0, ymm0, ymm0
1045        vmovdqu	YMMWORD PTR [rdi+64], ymm0
1046        vmovdqu	YMMWORD PTR [rdi+96], ymm0
1047        vmovdqu	YMMWORD PTR [rdi+128], ymm0
1048        vmovdqu	YMMWORD PTR [rdi+160], ymm0
1049        vmovdqu	YMMWORD PTR [rdi+192], ymm0
1050        vmovdqu	YMMWORD PTR [rdi+224], ymm0
1051        vmovdqu	YMMWORD PTR [rdi+256], ymm0
1052        vmovdqu	YMMWORD PTR [rdi+288], ymm0
1053        vmovdqu	YMMWORD PTR [rdi+320], ymm0
1054        mov	QWORD PTR [rdi+608], 0
1055        mov	WORD PTR [rdi+616], 0
1056        vzeroupper
1057        pop	rsi
1058        pop	rdi
1059        ret
1060poly1305_final_avx2 ENDP
1061_TEXT ENDS
1062ENDIF
1063END