luna - wolfssl/wolfcrypt/src/port/arm/armv8-32-poly1305-asm.S

   1/* armv8-32-poly1305-asm
   2 *
   3 * Copyright (C) 2006-2026 wolfSSL Inc.
   4 *
   5 * This file is part of wolfSSL.
   6 *
   7 * wolfSSL is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License as published by
   9 * the Free Software Foundation; either version 3 of the License, or
  10 * (at your option) any later version.
  11 *
  12 * wolfSSL is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 * GNU General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU General Public License
  18 * along with this program; if not, write to the Free Software
  19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
  20 */
  21
  22/* Generated using (from wolfssl):
  23 *   cd ../scripts
  24 *   ruby ./poly1305/poly1305.rb arm32 \
  25 *       ../wolfssl/wolfcrypt/src/port/arm/armv8-32-poly1305-asm.S
  26 */
  27
  28#include <wolfssl/wolfcrypt/libwolfssl_sources_asm.h>
  29
  30#ifdef WOLFSSL_ARMASM
  31#if !defined(__aarch64__) && !defined(WOLFSSL_ARMASM_THUMB2)
  32#ifndef WOLFSSL_ARMASM_INLINE
  33#ifdef HAVE_POLY1305
  34#ifdef WOLFSSL_ARMASM_NO_NEON
  35	.text
  36	.align	4
  37	.globl	poly1305_arm32_blocks_16
  38	.type	poly1305_arm32_blocks_16, %function
  39poly1305_arm32_blocks_16:
  40	push	{r4, r5, r6, r7, r8, r9, r10, r11, lr}
  41	sub	sp, sp, #28
  42	cmp	r2, #0
  43	beq	L_poly1305_arm32_16_done
  44	add	lr, sp, #12
  45	stm	lr, {r0, r1, r2, r3}
  46	# Get h pointer
  47	add	lr, r0, #16
  48	ldm	lr, {r4, r5, r6, r7, r8}
  49L_poly1305_arm32_16_loop:
  50	# Add m to h
  51	ldr	r1, [sp, #16]
  52	ldr	r2, [r1]
  53	ldr	r3, [r1, #4]
  54	ldr	r9, [r1, #8]
  55	ldr	r10, [r1, #12]
  56	ldr	r11, [sp, #24]
  57	adds	r4, r4, r2
  58	adcs	r5, r5, r3
  59	adcs	r6, r6, r9
  60	adcs	r7, r7, r10
  61	add	r1, r1, #16
  62	adc	r8, r8, r11
  63#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6)
  64	stm	lr, {r4, r5, r6, r7, r8}
  65#else
  66	# h[0]-h[2] in r4-r6 for multiplication.
  67	str	r7, [lr, #12]
  68	str	r8, [lr, #16]
  69#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */
  70	str	r1, [sp, #16]
  71	ldr	r1, [sp, #12]
  72	# Multiply h by r
  73#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6)
  74	# r0 = #0, r1 = r, lr = h, r2 = h[j], r3 = r[i]
  75	ldr	r3, [r1]
  76	eor	r0, r0, r0
  77	# r[0] * h[0]
  78	# h[0] in r4
  79	umull	r4, r5, r3, r4
  80	# r[0] * h[2]
  81	# h[2] in r6
  82	umull	r6, r7, r3, r6
  83	# r[0] * h[4]
  84	# h[4] in r8
  85	mul	r8, r3, r8
  86	# r[0] * h[1]
  87	ldr	r2, [lr, #4]
  88	mov	r12, r0
  89	umlal	r5, r12, r3, r2
  90	# r[0] * h[3]
  91	ldr	r2, [lr, #12]
  92	adds	r6, r6, r12
  93	adc	r7, r7, r0
  94	umlal	r7, r8, r3, r2
  95	# r[1] * h[0]
  96	ldr	r3, [r1, #4]
  97	ldr	r2, [lr]
  98	mov	r12, r0
  99	umlal	r5, r12, r3, r2
 100	# r[1] * h[1]
 101	ldr	r2, [lr, #4]
 102	adds	r6, r6, r12
 103	adc	r12, r0, r0
 104	umlal	r6, r12, r3, r2
 105	# r[1] * h[2]
 106	ldr	r2, [lr, #8]
 107	adds	r7, r7, r12
 108	adc	r12, r0, r0
 109	umlal	r7, r12, r3, r2
 110	# r[1] * h[3]
 111	ldr	r2, [lr, #12]
 112	adds	r8, r8, r12
 113	adc	r9, r0, r0
 114	umlal	r8, r9, r3, r2
 115	# r[1] * h[4]
 116	ldr	r2, [lr, #16]
 117	mla	r9, r3, r2, r9
 118	# r[2] * h[0]
 119	ldr	r3, [r1, #8]
 120	ldr	r2, [lr]
 121	mov	r12, r0
 122	umlal	r6, r12, r3, r2
 123	# r[2] * h[1]
 124	ldr	r2, [lr, #4]
 125	adds	r7, r7, r12
 126	adc	r12, r0, r0
 127	umlal	r7, r12, r3, r2
 128	# r[2] * h[2]
 129	ldr	r2, [lr, #8]
 130	adds	r8, r8, r12
 131	adc	r12, r0, r0
 132	umlal	r8, r12, r3, r2
 133	# r[2] * h[3]
 134	ldr	r2, [lr, #12]
 135	adds	r9, r9, r12
 136	adc	r10, r0, r0
 137	umlal	r9, r10, r3, r2
 138	# r[2] * h[4]
 139	ldr	r2, [lr, #16]
 140	mla	r10, r3, r2, r10
 141	# r[3] * h[0]
 142	ldr	r3, [r1, #12]
 143	ldr	r2, [lr]
 144	mov	r12, r0
 145	umlal	r7, r12, r3, r2
 146	# r[3] * h[1]
 147	ldr	r2, [lr, #4]
 148	adds	r8, r8, r12
 149	adc	r12, r0, r0
 150	umlal	r8, r12, r3, r2
 151	# r[3] * h[2]
 152	ldr	r2, [lr, #8]
 153	adds	r9, r9, r12
 154	adc	r10, r10, r0
 155	umlal	r9, r10, r3, r2
 156	# r[3] * h[3]
 157	ldr	r2, [lr, #12]
 158	mov	r11, r0
 159	umlal	r10, r11, r3, r2
 160	# r[3] * h[4]
 161	ldr	r2, [lr, #16]
 162	mov	r12, r0
 163	mla	r11, r3, r2, r11
 164#else
 165	ldm	r1, {r0, r1, r2, r3}
 166	# r[0] * h[0]
 167	umull	r10, r11, r0, r4
 168	# r[1] * h[0]
 169	umull	r12, r7, r1, r4
 170	# r[0] * h[1]
 171	umaal	r11, r12, r0, r5
 172	# r[2] * h[0]
 173	umull	r8, r9, r2, r4
 174	# r[1] * h[1]
 175	umaal	r12, r8, r1, r5
 176	# r[0] * h[2]
 177	umaal	r12, r7, r0, r6
 178	# r[3] * h[0]
 179	umaal	r8, r9, r3, r4
 180	stm	sp, {r10, r11, r12}
 181	# r[2] * h[1]
 182	umaal	r7, r8, r2, r5
 183	# Replace h[0] with h[3]
 184	ldr	r4, [lr, #12]
 185	# r[1] * h[2]
 186	umull	r10, r11, r1, r6
 187	# r[2] * h[2]
 188	umaal	r8, r9, r2, r6
 189	# r[0] * h[3]
 190	umaal	r7, r10, r0, r4
 191	# r[3] * h[1]
 192	umaal	r8, r11, r3, r5
 193	# r[1] * h[3]
 194	umaal	r8, r10, r1, r4
 195	# r[3] * h[2]
 196	umaal	r9, r11, r3, r6
 197	# r[2] * h[3]
 198	umaal	r9, r10, r2, r4
 199	# Replace h[1] with h[4]
 200	ldr	r5, [lr, #16]
 201	# r[3] * h[3]
 202	umaal	r10, r11, r3, r4
 203	mov	r12, #0
 204	# r[0] * h[4]
 205	umaal	r8, r12, r0, r5
 206	# r[1] * h[4]
 207	umaal	r9, r12, r1, r5
 208	# r[2] * h[4]
 209	umaal	r10, r12, r2, r5
 210	# r[3] * h[4]
 211	umaal	r11, r12, r3, r5
 212	# DONE
 213	ldm	sp, {r4, r5, r6}
 214#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */
 215	# r12 will be zero because r is masked.
 216	# Load length
 217	ldr	r2, [sp, #20]
 218	# Reduce mod 2^130 - 5
 219	bic	r3, r8, #0x3
 220	and	r8, r8, #3
 221	adds	r4, r4, r3
 222	lsr	r3, r3, #2
 223	adcs	r5, r5, r9
 224	orr	r3, r3, r9, lsl #30
 225	adcs	r6, r6, r10
 226	lsr	r9, r9, #2
 227	adcs	r7, r7, r11
 228	orr	r9, r9, r10, lsl #30
 229	adc	r8, r8, r12
 230	lsr	r10, r10, #2
 231	adds	r4, r4, r3
 232	orr	r10, r10, r11, lsl #30
 233	adcs	r5, r5, r9
 234	lsr	r11, r11, #2
 235	adcs	r6, r6, r10
 236	adcs	r7, r7, r11
 237	adc	r8, r8, r12
 238	# Sub 16 from length.
 239	subs	r2, r2, #16
 240	# Store length.
 241	str	r2, [sp, #20]
 242	# Loop again if more message to do.
 243	bgt	L_poly1305_arm32_16_loop
 244	stm	lr, {r4, r5, r6, r7, r8}
 245L_poly1305_arm32_16_done:
 246	add	sp, sp, #28
 247	pop	{r4, r5, r6, r7, r8, r9, r10, r11, pc}
 248	.size	poly1305_arm32_blocks_16,.-poly1305_arm32_blocks_16
 249#ifndef __APPLE__
 250	.text
 251	.type	L_poly1305_arm32_clamp, %object
 252	.size	L_poly1305_arm32_clamp, 16
 253#else
 254	.section	__DATA,__data
 255#endif /* __APPLE__ */
 256	# 8-byte aligned, 64-bit aligned
 257#ifndef __APPLE__
 258	.align	3
 259#else
 260	.p2align	3
 261#endif /* __APPLE__ */
 262L_poly1305_arm32_clamp:
 263	.long	0x0fffffff,0x0ffffffc,0x0ffffffc,0x0ffffffc
 264	.text
 265	.align	4
 266	.globl	poly1305_set_key
 267	.type	poly1305_set_key, %function
 268poly1305_set_key:
 269	push	{r4, r5, r6, r7, r8, lr}
 270	# Load mask.
 271	adr	lr, L_poly1305_arm32_clamp
 272	ldm	lr, {r6, r7, r8, r12}
 273	# Load and cache padding.
 274	ldr	r2, [r1, #16]
 275	ldr	r3, [r1, #20]
 276	ldr	r4, [r1, #24]
 277	ldr	r5, [r1, #28]
 278	add	lr, r0, #36
 279	stm	lr, {r2, r3, r4, r5}
 280	# Load, mask and store r.
 281	ldr	r2, [r1]
 282	ldr	r3, [r1, #4]
 283	ldr	r4, [r1, #8]
 284	ldr	r5, [r1, #12]
 285	and	r2, r2, r6
 286	and	r3, r3, r7
 287	and	r4, r4, r8
 288	and	r5, r5, r12
 289	add	lr, r0, #0
 290	stm	lr, {r2, r3, r4, r5}
 291	# h (accumulator) = 0
 292	eor	r6, r6, r6
 293	eor	r7, r7, r7
 294	eor	r8, r8, r8
 295	eor	r12, r12, r12
 296	add	lr, r0, #16
 297	eor	r5, r5, r5
 298	stm	lr, {r5, r6, r7, r8, r12}
 299	# Zero leftover
 300	str	r5, [r0, #52]
 301	pop	{r4, r5, r6, r7, r8, pc}
 302	.size	poly1305_set_key,.-poly1305_set_key
 303	.text
 304	.align	4
 305	.globl	poly1305_final
 306	.type	poly1305_final, %function
 307poly1305_final:
 308	push	{r4, r5, r6, r7, r8, r9, lr}
 309	add	r9, r0, #16
 310	ldm	r9, {r4, r5, r6, r7, r8}
 311	# Add 5 and check for h larger than p.
 312	adds	r2, r4, #5
 313	adcs	r2, r5, #0
 314	adcs	r2, r6, #0
 315	adcs	r2, r7, #0
 316	adc	r2, r8, #0
 317	sub	r2, r2, #4
 318	lsr	r2, r2, #31
 319	sub	r2, r2, #1
 320	and	r2, r2, #5
 321	# Add 0/5 to h.
 322	adds	r4, r4, r2
 323	adcs	r5, r5, #0
 324	adcs	r6, r6, #0
 325	adc	r7, r7, #0
 326	# Add padding
 327	add	r9, r0, #36
 328	ldm	r9, {r2, r3, r12, lr}
 329	adds	r4, r4, r2
 330	adcs	r5, r5, r3
 331	adcs	r6, r6, r12
 332	adc	r7, r7, lr
 333	# Store MAC
 334	str	r4, [r1]
 335	str	r5, [r1, #4]
 336	str	r6, [r1, #8]
 337	str	r7, [r1, #12]
 338	# Zero out h.
 339	eor	r4, r4, r4
 340	eor	r5, r5, r5
 341	eor	r6, r6, r6
 342	eor	r7, r7, r7
 343	eor	r8, r8, r8
 344	add	r9, r0, #16
 345	stm	r9, {r4, r5, r6, r7, r8}
 346	# Zero out r.
 347	add	r9, r0, #0
 348	stm	r9, {r4, r5, r6, r7}
 349	# Zero out padding.
 350	add	r9, r0, #36
 351	stm	r9, {r4, r5, r6, r7}
 352	pop	{r4, r5, r6, r7, r8, r9, pc}
 353	.size	poly1305_final,.-poly1305_final
 354#else
 355	.text
 356	.align	4
 357	.globl	poly1305_arm32_blocks_16
 358	.type	poly1305_arm32_blocks_16, %function
 359poly1305_arm32_blocks_16:
 360	push	{r4, r5, r6, r7, r8, r9, r10, r11, lr}
 361	sub	sp, sp, #28
 362	cmp	r2, #0
 363	beq	L_poly1305_arm32_16_done
 364	add	lr, sp, #12
 365	stm	lr, {r0, r1, r2, r3}
 366	# Get h pointer
 367	add	lr, r0, #16
 368	ldm	lr, {r4, r5, r6, r7, r8}
 369L_poly1305_arm32_16_loop:
 370	# Add m to h
 371	ldr	r1, [sp, #16]
 372	ldr	r2, [r1]
 373	ldr	r3, [r1, #4]
 374	ldr	r9, [r1, #8]
 375	ldr	r10, [r1, #12]
 376	ldr	r11, [sp, #24]
 377	adds	r4, r4, r2
 378	adcs	r5, r5, r3
 379	adcs	r6, r6, r9
 380	adcs	r7, r7, r10
 381	add	r1, r1, #16
 382	adc	r8, r8, r11
 383#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6)
 384	stm	lr, {r4, r5, r6, r7, r8}
 385#else
 386	# h[0]-h[2] in r4-r6 for multiplication.
 387	str	r7, [lr, #12]
 388	str	r8, [lr, #16]
 389#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */
 390	str	r1, [sp, #16]
 391	ldr	r1, [sp, #12]
 392	# Multiply h by r
 393#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6)
 394	# r0 = #0, r1 = r, lr = h, r2 = h[j], r3 = r[i]
 395	ldr	r3, [r1]
 396	eor	r0, r0, r0
 397	# r[0] * h[0]
 398	# h[0] in r4
 399	umull	r4, r5, r3, r4
 400	# r[0] * h[2]
 401	# h[2] in r6
 402	umull	r6, r7, r3, r6
 403	# r[0] * h[4]
 404	# h[4] in r8
 405	mul	r8, r3, r8
 406	# r[0] * h[1]
 407	ldr	r2, [lr, #4]
 408	mov	r12, r0
 409	umlal	r5, r12, r3, r2
 410	# r[0] * h[3]
 411	ldr	r2, [lr, #12]
 412	adds	r6, r6, r12
 413	adc	r7, r7, r0
 414	umlal	r7, r8, r3, r2
 415	# r[1] * h[0]
 416	ldr	r3, [r1, #4]
 417	ldr	r2, [lr]
 418	mov	r12, r0
 419	umlal	r5, r12, r3, r2
 420	# r[1] * h[1]
 421	ldr	r2, [lr, #4]
 422	adds	r6, r6, r12
 423	adc	r12, r0, r0
 424	umlal	r6, r12, r3, r2
 425	# r[1] * h[2]
 426	ldr	r2, [lr, #8]
 427	adds	r7, r7, r12
 428	adc	r12, r0, r0
 429	umlal	r7, r12, r3, r2
 430	# r[1] * h[3]
 431	ldr	r2, [lr, #12]
 432	adds	r8, r8, r12
 433	adc	r9, r0, r0
 434	umlal	r8, r9, r3, r2
 435	# r[1] * h[4]
 436	ldr	r2, [lr, #16]
 437	mla	r9, r3, r2, r9
 438	# r[2] * h[0]
 439	ldr	r3, [r1, #8]
 440	ldr	r2, [lr]
 441	mov	r12, r0
 442	umlal	r6, r12, r3, r2
 443	# r[2] * h[1]
 444	ldr	r2, [lr, #4]
 445	adds	r7, r7, r12
 446	adc	r12, r0, r0
 447	umlal	r7, r12, r3, r2
 448	# r[2] * h[2]
 449	ldr	r2, [lr, #8]
 450	adds	r8, r8, r12
 451	adc	r12, r0, r0
 452	umlal	r8, r12, r3, r2
 453	# r[2] * h[3]
 454	ldr	r2, [lr, #12]
 455	adds	r9, r9, r12
 456	adc	r10, r0, r0
 457	umlal	r9, r10, r3, r2
 458	# r[2] * h[4]
 459	ldr	r2, [lr, #16]
 460	mla	r10, r3, r2, r10
 461	# r[3] * h[0]
 462	ldr	r3, [r1, #12]
 463	ldr	r2, [lr]
 464	mov	r12, r0
 465	umlal	r7, r12, r3, r2
 466	# r[3] * h[1]
 467	ldr	r2, [lr, #4]
 468	adds	r8, r8, r12
 469	adc	r12, r0, r0
 470	umlal	r8, r12, r3, r2
 471	# r[3] * h[2]
 472	ldr	r2, [lr, #8]
 473	adds	r9, r9, r12
 474	adc	r10, r10, r0
 475	umlal	r9, r10, r3, r2
 476	# r[3] * h[3]
 477	ldr	r2, [lr, #12]
 478	mov	r11, r0
 479	umlal	r10, r11, r3, r2
 480	# r[3] * h[4]
 481	ldr	r2, [lr, #16]
 482	mov	r12, r0
 483	mla	r11, r3, r2, r11
 484#else
 485	ldm	r1, {r0, r1, r2, r3}
 486	# r[0] * h[0]
 487	umull	r10, r11, r0, r4
 488	# r[1] * h[0]
 489	umull	r12, r7, r1, r4
 490	# r[0] * h[1]
 491	umaal	r11, r12, r0, r5
 492	# r[2] * h[0]
 493	umull	r8, r9, r2, r4
 494	# r[1] * h[1]
 495	umaal	r12, r8, r1, r5
 496	# r[0] * h[2]
 497	umaal	r12, r7, r0, r6
 498	# r[3] * h[0]
 499	umaal	r8, r9, r3, r4
 500	stm	sp, {r10, r11, r12}
 501	# r[2] * h[1]
 502	umaal	r7, r8, r2, r5
 503	# Replace h[0] with h[3]
 504	ldr	r4, [lr, #12]
 505	# r[1] * h[2]
 506	umull	r10, r11, r1, r6
 507	# r[2] * h[2]
 508	umaal	r8, r9, r2, r6
 509	# r[0] * h[3]
 510	umaal	r7, r10, r0, r4
 511	# r[3] * h[1]
 512	umaal	r8, r11, r3, r5
 513	# r[1] * h[3]
 514	umaal	r8, r10, r1, r4
 515	# r[3] * h[2]
 516	umaal	r9, r11, r3, r6
 517	# r[2] * h[3]
 518	umaal	r9, r10, r2, r4
 519	# Replace h[1] with h[4]
 520	ldr	r5, [lr, #16]
 521	# r[3] * h[3]
 522	umaal	r10, r11, r3, r4
 523	mov	r12, #0
 524	# r[0] * h[4]
 525	umaal	r8, r12, r0, r5
 526	# r[1] * h[4]
 527	umaal	r9, r12, r1, r5
 528	# r[2] * h[4]
 529	umaal	r10, r12, r2, r5
 530	# r[3] * h[4]
 531	umaal	r11, r12, r3, r5
 532	# DONE
 533	ldm	sp, {r4, r5, r6}
 534#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */
 535	# r12 will be zero because r is masked.
 536	# Load length
 537	ldr	r2, [sp, #20]
 538	# Reduce mod 2^130 - 5
 539	bic	r3, r8, #0x3
 540	and	r8, r8, #3
 541	adds	r4, r4, r3
 542	lsr	r3, r3, #2
 543	adcs	r5, r5, r9
 544	orr	r3, r3, r9, lsl #30
 545	adcs	r6, r6, r10
 546	lsr	r9, r9, #2
 547	adcs	r7, r7, r11
 548	orr	r9, r9, r10, lsl #30
 549	adc	r8, r8, r12
 550	lsr	r10, r10, #2
 551	adds	r4, r4, r3
 552	orr	r10, r10, r11, lsl #30
 553	adcs	r5, r5, r9
 554	lsr	r11, r11, #2
 555	adcs	r6, r6, r10
 556	adcs	r7, r7, r11
 557	adc	r8, r8, r12
 558	# Sub 16 from length.
 559	subs	r2, r2, #16
 560	# Store length.
 561	str	r2, [sp, #20]
 562	# Loop again if more message to do.
 563	bgt	L_poly1305_arm32_16_loop
 564	stm	lr, {r4, r5, r6, r7, r8}
 565L_poly1305_arm32_16_done:
 566	add	sp, sp, #28
 567	pop	{r4, r5, r6, r7, r8, r9, r10, r11, pc}
 568	.size	poly1305_arm32_blocks_16,.-poly1305_arm32_blocks_16
 569	.text
 570	.align	4
 571	.globl	poly1305_arm32_blocks
 572	.type	poly1305_arm32_blocks, %function
 573poly1305_arm32_blocks:
 574	push	{r4, r5, r6, r7, r8, r9, r10, r11, lr}
 575	vpush	{d8-d15}
 576	cmp	r2, #16
 577	add	r12, r0, #16
 578	bgt	L_poly1305_arm32_blocks_begin_neon
 579	ldm	r12, {r7, r8, r9, r10, r11}
 580	b	L_poly1305_arm32_blocks_start_1
 581L_poly1305_arm32_blocks_begin_neon:
 582	vmov.i16	q15, #0xffff
 583	vshr.u64	q15, q15, #38
 584	vld1.64	{d0-d2}, [r12]
 585	vshl.u64	d4, d2, #24
 586	vsri.u64	d4, d1, #40
 587	vshr.u64	d3, d1, #14
 588	vshl.u64	d2, d1, #12
 589	vsri.u64	d1, d0, #26
 590	vsri.u64	d2, d0, #52
 591	vand.u64	d0, d0, d31
 592	vand.u64	d3, d3, d31
 593	vand.u64	d2, d2, d31
 594	vand.u64	d1, d1, d31
 595	add	r3, r0, #0x7c
 596	vldm.32	r3, {d20-d24}
 597	cmp	r2, #0x40
 598	bge	L_poly1305_arm32_blocks_begin_4
 599	vshl.u32	d6, d21, #2
 600	vshl.u32	d7, d22, #2
 601	vshl.u32	d8, d23, #2
 602	vshl.u32	d9, d24, #2
 603	vadd.u32	d6, d6, d21
 604	vadd.u32	d7, d7, d22
 605	vadd.u32	d8, d8, d23
 606	vadd.u32	d9, d9, d24
 607	b	L_poly1305_arm32_blocks_start_2
 608L_poly1305_arm32_blocks_begin_4:
 609	add	r3, r0, #0xa4
 610	vldm.32	r3, {d26-d30}
 611L_poly1305_arm32_blocks_start_4:
 612	sub	r2, #0x40
 613	vld4.32	{d10-d13}, [r1]!
 614	vshl.u32	d6, d27, #2
 615	vshl.u32	d7, d28, #2
 616	vshl.u32	d8, d29, #2
 617	vshl.u32	d9, d30, #2
 618	vadd.u32	d6, d6, d27
 619	vadd.u32	d7, d7, d28
 620	vadd.u32	d8, d8, d29
 621	vadd.u32	d9, d9, d30
 622	vshr.u32	d14, d13, #8
 623	vshl.u32	d13, d13, #18
 624	vorr.i32	d14, d14, #0x1000000
 625	vsri.u32	d13, d12, #14
 626	vshl.u32	d12, d12, #12
 627	vand.i32	d13, d13, #0x3ffffff
 628	vsri.u32	d12, d11, #20
 629	vshl.u32	d11, d11, #6
 630	vand.i32	d12, d12, #0x3ffffff
 631	vsri.u32	d11, d10, #26
 632	vand.i32	d10, d10, #0x3ffffff
 633	vand.i32	d11, d11, #0x3ffffff
 634	vadd.u32	d4, d4, d14
 635	vadd.u32	q1, q1, q6
 636	vadd.u32	q0, q0, q5
 637	vmull.u32	q5, d0, d26
 638	vmull.u32	q6, d0, d27
 639	vmull.u32	q7, d0, d28
 640	vmull.u32	q8, d0, d29
 641	vmull.u32	q9, d0, d30
 642	vmlal.u32	q5, d1, d9
 643	vmlal.u32	q6, d1, d26
 644	vmlal.u32	q7, d1, d27
 645	vmlal.u32	q8, d1, d28
 646	vmlal.u32	q9, d1, d29
 647	vmlal.u32	q5, d2, d8
 648	vmlal.u32	q6, d2, d9
 649	vmlal.u32	q7, d2, d26
 650	vmlal.u32	q8, d2, d27
 651	vmlal.u32	q9, d2, d28
 652	vmlal.u32	q5, d3, d7
 653	vmlal.u32	q6, d3, d8
 654	vmlal.u32	q7, d3, d9
 655	vmlal.u32	q8, d3, d26
 656	vmlal.u32	q9, d3, d27
 657	vmlal.u32	q5, d4, d6
 658	vmlal.u32	q6, d4, d7
 659	vmlal.u32	q7, d4, d8
 660	vmlal.u32	q8, d4, d9
 661	vmlal.u32	q9, d4, d26
 662	vld4.32	{d0-d3}, [r1]!
 663	vshl.u32	d6, d21, #2
 664	vshl.u32	d7, d22, #2
 665	vshl.u32	d8, d23, #2
 666	vshl.u32	d9, d24, #2
 667	vadd.u32	d6, d6, d21
 668	vadd.u32	d7, d7, d22
 669	vadd.u32	d8, d8, d23
 670	vadd.u32	d9, d9, d24
 671	vshr.u32	d4, d3, #8
 672	vshl.u32	d3, d3, #18
 673	vorr.i32	d4, d4, #0x1000000
 674	vsri.u32	d3, d2, #14
 675	vshl.u32	d2, d2, #12
 676	vand.i32	d3, d3, #0x3ffffff
 677	vsri.u32	d2, d1, #20
 678	vshl.u32	d1, d1, #6
 679	vand.i32	d2, d2, #0x3ffffff
 680	vsri.u32	d1, d0, #26
 681	vand.i32	d0, d0, #0x3ffffff
 682	vand.i32	d1, d1, #0x3ffffff
 683	vmlal.u32	q5, d0, d20
 684	vmlal.u32	q6, d0, d21
 685	vmlal.u32	q7, d0, d22
 686	vmlal.u32	q8, d0, d23
 687	vmlal.u32	q9, d0, d24
 688	vmlal.u32	q5, d1, d9
 689	vmlal.u32	q6, d1, d20
 690	vmlal.u32	q7, d1, d21
 691	vmlal.u32	q8, d1, d22
 692	vmlal.u32	q9, d1, d23
 693	vmlal.u32	q5, d2, d8
 694	vmlal.u32	q6, d2, d9
 695	vmlal.u32	q7, d2, d20
 696	vmlal.u32	q8, d2, d21
 697	vmlal.u32	q9, d2, d22
 698	vmlal.u32	q5, d3, d7
 699	vmlal.u32	q6, d3, d8
 700	vmlal.u32	q7, d3, d9
 701	vmlal.u32	q8, d3, d20
 702	vmlal.u32	q9, d3, d21
 703	vmlal.u32	q5, d4, d6
 704	vmlal.u32	q6, d4, d7
 705	vmlal.u32	q7, d4, d8
 706	vmlal.u32	q8, d4, d9
 707	vmlal.u32	q9, d4, d20
 708	vadd.u64	d0, d10, d11
 709	vadd.u64	d1, d12, d13
 710	vadd.u64	d2, d14, d15
 711	vadd.u64	d3, d16, d17
 712	vadd.u64	d4, d18, d19
 713	vsra.u64	d1, d0, #26
 714	vand.u64	d0, d0, d31
 715	vsra.u64	d2, d1, #26
 716	vand.u64	d1, d1, d31
 717	vsra.u64	d3, d2, #26
 718	vand.u64	d2, d2, d31
 719	vsra.u64	d4, d3, #26
 720	vand.u64	d3, d3, d31
 721	vshr.u64	d15, d4, #26
 722	vand.u64	d4, d4, d31
 723	vadd.u64	d0, d0, d15
 724	vshl.u64	d15, d15, #2
 725	vadd.u64	d0, d0, d15
 726	vsra.u64	d1, d0, #26
 727	vand.u64	d0, d0, d31
 728	cmp	r2, #0x40
 729	bge	L_poly1305_arm32_blocks_start_4
 730	cmp	r2, #32
 731	blt	L_poly1305_arm32_blocks_done_neon
 732L_poly1305_arm32_blocks_start_2:
 733	sub	r2, #32
 734	vld4.32	{d10-d13}, [r1]!
 735	vshr.u32	d14, d13, #8
 736	vshl.u32	d13, d13, #18
 737	vorr.i32	d14, d14, #0x1000000
 738	vsri.u32	d13, d12, #14
 739	vshl.u32	d12, d12, #12
 740	vand.i32	d13, d13, #0x3ffffff
 741	vsri.u32	d12, d11, #20
 742	vshl.u32	d11, d11, #6
 743	vand.i32	d12, d12, #0x3ffffff
 744	vsri.u32	d11, d10, #26
 745	vand.i32	d10, d10, #0x3ffffff
 746	vand.i32	d11, d11, #0x3ffffff
 747	vadd.u32	d4, d4, d14
 748	vadd.u32	q1, q1, q6
 749	vadd.u32	q0, q0, q5
 750	vmull.u32	q5, d0, d20
 751	vmull.u32	q6, d0, d21
 752	vmull.u32	q7, d0, d22
 753	vmull.u32	q8, d0, d23
 754	vmull.u32	q9, d0, d24
 755	vmlal.u32	q5, d1, d9
 756	vmlal.u32	q6, d1, d20
 757	vmlal.u32	q7, d1, d21
 758	vmlal.u32	q8, d1, d22
 759	vmlal.u32	q9, d1, d23
 760	vmlal.u32	q5, d2, d8
 761	vmlal.u32	q6, d2, d9
 762	vmlal.u32	q7, d2, d20
 763	vmlal.u32	q8, d2, d21
 764	vmlal.u32	q9, d2, d22
 765	vmlal.u32	q5, d3, d7
 766	vmlal.u32	q6, d3, d8
 767	vmlal.u32	q7, d3, d9
 768	vmlal.u32	q8, d3, d20
 769	vmlal.u32	q9, d3, d21
 770	vmlal.u32	q5, d4, d6
 771	vmlal.u32	q6, d4, d7
 772	vmlal.u32	q7, d4, d8
 773	vmlal.u32	q8, d4, d9
 774	vmlal.u32	q9, d4, d20
 775	vadd.u64	d0, d10, d11
 776	vadd.u64	d1, d12, d13
 777	vadd.u64	d2, d14, d15
 778	vadd.u64	d3, d16, d17
 779	vadd.u64	d4, d18, d19
 780	vsra.u64	d1, d0, #26
 781	vand.u64	d0, d0, d31
 782	vsra.u64	d2, d1, #26
 783	vand.u64	d1, d1, d31
 784	vsra.u64	d3, d2, #26
 785	vand.u64	d2, d2, d31
 786	vsra.u64	d4, d3, #26
 787	vand.u64	d3, d3, d31
 788	vshr.u64	d5, d4, #26
 789	vand.u64	d4, d4, d31
 790	vadd.u64	d0, d0, d5
 791	vshl.u64	d5, d5, #2
 792	vadd.u64	d0, d0, d5
 793	vsra.u64	d1, d0, #26
 794	vand.u64	d0, d0, d31
 795L_poly1305_arm32_blocks_done_neon:
 796	cmp	r2, #16
 797	beq	L_poly1305_arm32_blocks_begin_1
 798	add	r12, r0, #16
 799	vsli.u64	d0, d1, #26
 800	vsli.u64	d0, d2, #52
 801	vshr.u64	d1, d2, #12
 802	vsli.u64	d1, d3, #14
 803	vsli.u64	d1, d4, #40
 804	vshr.u64	d2, d4, #24
 805	vst1.64	{d0-d2}, [r12]
 806	b	L_poly1305_arm32_blocks_done
 807L_poly1305_arm32_blocks_begin_1:
 808	vsli.u64	d0, d1, #26
 809	vsli.u64	d0, d2, #52
 810	vshr.u64	d1, d2, #12
 811	vsli.u64	d1, d3, #14
 812	vsli.u64	d1, d4, #40
 813	vshr.u64	d2, d4, #24
 814	vmov	r7, r8, d0
 815	vmov	r9, r10, d1
 816	vmov	r11, d2[0]
 817L_poly1305_arm32_blocks_start_1:
 818	mov	r12, #1
 819	push	{r2}
 820	# Load message
 821	ldr	r2, [r1]
 822	ldr	r3, [r1, #4]
 823	ldr	r4, [r1, #8]
 824	ldr	r5, [r1, #12]
 825	# Add message
 826	adds	r7, r7, r2
 827	adcs	r8, r8, r3
 828	adcs	r9, r9, r4
 829	adcs	r10, r10, r5
 830	adc	r11, r11, r12
 831	push	{r0-r1}
 832	add	r1, r0, #0
 833	add	lr, r0, #16
 834#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6)
 835	stm	lr, {r7, r8, r9, r10, r11}
 836#else
 837	# h[0]-h[2] in r4-r6 for multiplication.
 838	str	r10, [lr, #12]
 839	str	r11, [lr, #16]
 840#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */
 841#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6)
 842	# r0 = #0, r1 = r, lr = h, r2 = h[j], r3 = r[i]
 843	ldr	r3, [r1]
 844	eor	r0, r0, r0
 845	# r[0] * h[0]
 846	# h[0] in r4
 847	umull	r7, r8, r3, r7
 848	# r[0] * h[2]
 849	# h[2] in r6
 850	umull	r9, r10, r3, r9
 851	# r[0] * h[4]
 852	# h[4] in r8
 853	mul	r11, r3, r11
 854	# r[0] * h[1]
 855	ldr	r2, [lr, #4]
 856	mov	r12, r0
 857	umlal	r8, r12, r3, r2
 858	# r[0] * h[3]
 859	ldr	r2, [lr, #12]
 860	adds	r9, r9, r12
 861	adc	r10, r10, r0
 862	umlal	r10, r11, r3, r2
 863	# r[1] * h[0]
 864	ldr	r3, [r1, #4]
 865	ldr	r2, [lr]
 866	mov	r12, r0
 867	umlal	r8, r12, r3, r2
 868	# r[1] * h[1]
 869	ldr	r2, [lr, #4]
 870	adds	r9, r9, r12
 871	adc	r12, r0, r0
 872	umlal	r9, r12, r3, r2
 873	# r[1] * h[2]
 874	ldr	r2, [lr, #8]
 875	adds	r10, r10, r12
 876	adc	r12, r0, r0
 877	umlal	r10, r12, r3, r2
 878	# r[1] * h[3]
 879	ldr	r2, [lr, #12]
 880	adds	r11, r11, r12
 881	adc	r4, r0, r0
 882	umlal	r11, r4, r3, r2
 883	# r[1] * h[4]
 884	ldr	r2, [lr, #16]
 885	mla	r4, r3, r2, r4
 886	# r[2] * h[0]
 887	ldr	r3, [r1, #8]
 888	ldr	r2, [lr]
 889	mov	r12, r0
 890	umlal	r9, r12, r3, r2
 891	# r[2] * h[1]
 892	ldr	r2, [lr, #4]
 893	adds	r10, r10, r12
 894	adc	r12, r0, r0
 895	umlal	r10, r12, r3, r2
 896	# r[2] * h[2]
 897	ldr	r2, [lr, #8]
 898	adds	r11, r11, r12
 899	adc	r12, r0, r0
 900	umlal	r11, r12, r3, r2
 901	# r[2] * h[3]
 902	ldr	r2, [lr, #12]
 903	adds	r4, r4, r12
 904	adc	r5, r0, r0
 905	umlal	r4, r5, r3, r2
 906	# r[2] * h[4]
 907	ldr	r2, [lr, #16]
 908	mla	r5, r3, r2, r5
 909	# r[3] * h[0]
 910	ldr	r3, [r1, #12]
 911	ldr	r2, [lr]
 912	mov	r12, r0
 913	umlal	r10, r12, r3, r2
 914	# r[3] * h[1]
 915	ldr	r2, [lr, #4]
 916	adds	r11, r11, r12
 917	adc	r12, r0, r0
 918	umlal	r11, r12, r3, r2
 919	# r[3] * h[2]
 920	ldr	r2, [lr, #8]
 921	adds	r4, r4, r12
 922	adc	r5, r5, r0
 923	umlal	r4, r5, r3, r2
 924	# r[3] * h[3]
 925	ldr	r2, [lr, #12]
 926	mov	r6, r0
 927	umlal	r5, r6, r3, r2
 928	# r[3] * h[4]
 929	ldr	r2, [lr, #16]
 930	mov	r12, r0
 931	mla	r6, r3, r2, r6
 932#else
 933	sub	sp, sp, #12
 934	ldm	r1, {r0, r1, r2, r3}
 935	# r[0] * h[0]
 936	umull	r5, r6, r0, r7
 937	# r[1] * h[0]
 938	umull	r12, r10, r1, r7
 939	# r[0] * h[1]
 940	umaal	r6, r12, r0, r8
 941	# r[2] * h[0]
 942	umull	r11, r4, r2, r7
 943	# r[1] * h[1]
 944	umaal	r12, r11, r1, r8
 945	# r[0] * h[2]
 946	umaal	r12, r10, r0, r9
 947	# r[3] * h[0]
 948	umaal	r11, r4, r3, r7
 949	stm	sp, {r5, r6, r12}
 950	# r[2] * h[1]
 951	umaal	r10, r11, r2, r8
 952	# Replace h[0] with h[3]
 953	ldr	r7, [lr, #12]
 954	# r[1] * h[2]
 955	umull	r5, r6, r1, r9
 956	# r[2] * h[2]
 957	umaal	r11, r4, r2, r9
 958	# r[0] * h[3]
 959	umaal	r10, r5, r0, r7
 960	# r[3] * h[1]
 961	umaal	r11, r6, r3, r8
 962	# r[1] * h[3]
 963	umaal	r11, r5, r1, r7
 964	# r[3] * h[2]
 965	umaal	r4, r6, r3, r9
 966	# r[2] * h[3]
 967	umaal	r4, r5, r2, r7
 968	# Replace h[1] with h[4]
 969	ldr	r8, [lr, #16]
 970	# r[3] * h[3]
 971	umaal	r5, r6, r3, r7
 972	mov	r12, #0
 973	# r[0] * h[4]
 974	umaal	r11, r12, r0, r8
 975	# r[1] * h[4]
 976	umaal	r4, r12, r1, r8
 977	# r[2] * h[4]
 978	umaal	r5, r12, r2, r8
 979	# r[3] * h[4]
 980	umaal	r6, r12, r3, r8
 981	# DONE
 982	ldm	sp, {r7, r8, r9}
 983	add	sp, sp, #12
 984#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */
 985	# Reduce mod 2^130 - 5
 986	bic	r3, r11, #0x3
 987	and	r11, r11, #3
 988	adds	r7, r7, r3
 989	lsr	r3, r3, #2
 990	adcs	r8, r8, r4
 991	orr	r3, r3, r4, lsl #30
 992	adcs	r9, r9, r5
 993	lsr	r4, r4, #2
 994	adcs	r10, r10, r6
 995	orr	r4, r4, r5, lsl #30
 996	adc	r11, r11, r12
 997	lsr	r5, r5, #2
 998	adds	r7, r7, r3
 999	orr	r5, r5, r6, lsl #30
1000	adcs	r8, r8, r4
1001	lsr	r6, r6, #2
1002	adcs	r9, r9, r5
1003	adcs	r10, r10, r6
1004	adc	r11, r11, r12
1005	pop	{r0-r1}
1006	pop	{r2}
1007	add	r12, r0, #16
1008	stm	r12, {r7, r8, r9, r10, r11}
1009L_poly1305_arm32_blocks_done:
1010	vpop	{d8-d15}
1011	pop	{r4, r5, r6, r7, r8, r9, r10, r11, pc}
1012	.size	poly1305_arm32_blocks,.-poly1305_arm32_blocks
1013#ifndef __APPLE__
1014	.text
1015	.type	L_poly1305_arm32_clamp, %object
1016	.size	L_poly1305_arm32_clamp, 16
1017#else
1018	.section	__DATA,__data
1019#endif /* __APPLE__ */
1020	# 8-byte aligned, 64-bit aligned
1021#ifndef __APPLE__
1022	.align	3
1023#else
1024	.p2align	3
1025#endif /* __APPLE__ */
1026L_poly1305_arm32_clamp:
1027	.long	0x0fffffff,0x0ffffffc,0x0ffffffc,0x0ffffffc
1028	.text
1029	.align	4
1030	.globl	poly1305_set_key
1031	.type	poly1305_set_key, %function
1032poly1305_set_key:
1033	push	{r4, r5, r6, r7, r8, r9, r10, r11, lr}
1034	vpush	{d8-d15}
1035	# Load mask.
1036	adr	lr, L_poly1305_arm32_clamp
1037	ldm	lr, {r6, r7, r8, r9}
1038	# Load and cache padding.
1039	ldr	r2, [r1, #16]
1040	ldr	r3, [r1, #20]
1041	ldr	r4, [r1, #24]
1042	ldr	r5, [r1, #28]
1043	add	lr, r0, #40
1044	stm	lr, {r2, r3, r4, r5}
1045	# Load, mask and store r.
1046	ldr	r2, [r1]
1047	ldr	r3, [r1, #4]
1048	ldr	r4, [r1, #8]
1049	ldr	r5, [r1, #12]
1050	and	r2, r2, r6
1051	and	r3, r3, r7
1052	and	r4, r4, r8
1053	and	r5, r5, r9
1054	add	lr, r0, #0
1055	stm	lr, {r2, r3, r4, r5}
1056	vmov.i16	q10, #0xffff
1057	vshr.u64	q10, q10, #38
1058	lsr	r8, r2, #26
1059	lsr	r9, r3, #20
1060	lsr	r10, r4, #14
1061	lsr	r11, r5, #8
1062	eor	r8, r8, r3, lsl #6
1063	eor	r9, r9, r4, lsl #12
1064	eor	r10, r10, r5, lsl #18
1065	and	r7, r2, #0x3ffffff
1066	and	r8, r8, #0x3ffffff
1067	and	r9, r9, #0x3ffffff
1068	and	r10, r10, #0x3ffffff
1069	vmov.i32	s1, r7
1070	vmov.i32	s3, r8
1071	vmov.i32	s5, r9
1072	vmov.i32	s7, r10
1073	vmov.i32	s9, r11
1074	push	{r0-r1}
1075#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6)
1076	# Square r
1077	umull	r1, r6, r2, r3
1078	mov	r12, #0
1079	umull	r7, r8, r2, r5
1080	mov	lr, r12
1081	umlal	r6, lr, r2, r4
1082	adds	r7, r7, lr
1083	adc	lr, r12, r12
1084	umlal	r7, lr, r3, r4
1085	mov	r9, r12
1086	umlal	lr, r9, r3, r5
1087	adds	r8, r8, lr
1088	adcs	r9, r9, r12
1089	adc	r10, r12, r12
1090	umlal	r9, r10, r4, r5
1091	adds	r1, r1, r1
1092	adcs	r6, r6, r6
1093	adcs	r7, r7, r7
1094	adcs	r8, r8, r8
1095	adcs	r9, r9, r9
1096	adcs	r10, r10, r10
1097	adc	r11, r12, r12
1098	umull	r0, lr, r2, r2
1099	adds	r1, r1, lr
1100	adcs	r6, r6, r12
1101	adc	lr, r12, r12
1102	umlal	r6, lr, r3, r3
1103	adds	r7, r7, lr
1104	adcs	r8, r8, r12
1105	adc	lr, r12, r12
1106	umlal	r8, lr, r4, r4
1107	adds	r9, r9, lr
1108	adcs	r10, r10, r12
1109	adc	r11, r11, r12
1110	umlal	r10, r11, r5, r5
1111#else
1112	umull	r0, r1, r2, r2
1113	umull	r6, r7, r2, r3
1114	adds	r6, r6, r6
1115	mov	r12, #0
1116	umaal	r1, r6, r12, r12
1117	mov	r8, r12
1118	umaal	r8, r7, r2, r4
1119	adcs	r8, r8, r8
1120	umaal	r6, r8, r3, r3
1121	umull	r9, r10, r2, r5
1122	umaal	r7, r9, r3, r4
1123	adcs	r7, r7, r7
1124	umaal	r7, r8, r12, r12
1125	umaal	r10, r9, r3, r5
1126	adcs	r10, r10, r10
1127	umaal	r8, r10, r4, r4
1128	mov	r11, r12
1129	umaal	r9, r11, r4, r5
1130	adcs	r9, r9, r9
1131	umaal	r9, r10, r12, r12
1132	adcs	r11, r11, r11
1133	umaal	r10, r11, r5, r5
1134	adc	r11, r11, r12
1135#endif /* defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) */
1136	# Reduce mod 2^130 - 5
1137	bic	r2, r8, #0x3
1138	and	r8, r8, #3
1139	adds	r0, r0, r2
1140	lsr	r2, r2, #2
1141	adcs	r1, r1, r9
1142	orr	r2, r2, r9, lsl #30
1143	adcs	r6, r6, r10
1144	lsr	r9, r9, #2
1145	adcs	r7, r7, r11
1146	orr	r9, r9, r10, lsl #30
1147	adc	r8, r8, r12
1148	lsr	r10, r10, #2
1149	adds	r0, r0, r2
1150	orr	r10, r10, r11, lsl #30
1151	adcs	r1, r1, r9
1152	lsr	r11, r11, #2
1153	adcs	r6, r6, r10
1154	adcs	r7, r7, r11
1155	adc	r8, r8, r12
1156	lsr	r3, r0, #26
1157	lsr	r4, r1, #20
1158	lsr	r5, r6, #14
1159	lsr	r10, r7, #8
1160	eor	r3, r3, r1, lsl #6
1161	eor	r4, r4, r6, lsl #12
1162	eor	r5, r5, r7, lsl #18
1163	eor	r10, r10, r8, lsl #24
1164	and	r2, r0, #0x3ffffff
1165	and	r3, r3, #0x3ffffff
1166	and	r4, r4, #0x3ffffff
1167	and	r5, r5, #0x3ffffff
1168	vmov.i32	s0, r2
1169	vmov.i32	s2, r3
1170	vmov.i32	s4, r4
1171	vmov.i32	s6, r5
1172	vmov.i32	s8, r10
1173	pop	{r0-r1}
1174	add	lr, r0, #0x7c
1175	vstm.32	lr, {d0-d4}
1176	# Multiply r^2, r by r^2
1177	vshl.u32	d6, d1, #2
1178	vshl.u32	d7, d2, #2
1179	vshl.u32	d8, d3, #2
1180	vshl.u32	d9, d4, #2
1181	vadd.u32	d6, d6, d1
1182	vadd.u32	d7, d7, d2
1183	vadd.u32	d8, d8, d3
1184	vadd.u32	d9, d9, d4
1185	vmull.u32	q5, d0, d0[0]
1186	vmull.u32	q6, d0, d1[0]
1187	vmull.u32	q7, d0, d2[0]
1188	vmull.u32	q8, d0, d3[0]
1189	vmull.u32	q9, d0, d4[0]
1190	vmlal.u32	q5, d1, d9[0]
1191	vmlal.u32	q6, d1, d0[0]
1192	vmlal.u32	q7, d1, d1[0]
1193	vmlal.u32	q8, d1, d2[0]
1194	vmlal.u32	q9, d1, d3[0]
1195	vmlal.u32	q5, d2, d8[0]
1196	vmlal.u32	q6, d2, d9[0]
1197	vmlal.u32	q7, d2, d0[0]
1198	vmlal.u32	q8, d2, d1[0]
1199	vmlal.u32	q9, d2, d2[0]
1200	vmlal.u32	q5, d3, d7[0]
1201	vmlal.u32	q6, d3, d8[0]
1202	vmlal.u32	q7, d3, d9[0]
1203	vmlal.u32	q8, d3, d0[0]
1204	vmlal.u32	q9, d3, d1[0]
1205	vmlal.u32	q5, d4, d6[0]
1206	vmlal.u32	q6, d4, d7[0]
1207	vmlal.u32	q7, d4, d8[0]
1208	vmlal.u32	q8, d4, d9[0]
1209	vmlal.u32	q9, d4, d0[0]
1210	vsra.u64	q6, q5, #26
1211	vand.u64	q5, q5, q10
1212	vsra.u64	q7, q6, #26
1213	vand.u64	q6, q6, q10
1214	vsra.u64	q8, q7, #26
1215	vand.u64	q7, q7, q10
1216	vsra.u64	q9, q8, #26
1217	vand.u64	q8, q8, q10
1218	vshr.u64	q3, q9, #26
1219	vand.u64	q9, q9, q10
1220	vadd.u64	q5, q5, q3
1221	vshl.u64	q3, q3, #2
1222	vadd.u64	q5, q5, q3
1223	vsra.u64	q6, q5, #26
1224	vand.u64	q5, q5, q10
1225	vmovn.i64	d10, q5
1226	vmovn.i64	d11, q6
1227	vmovn.i64	d12, q7
1228	vmovn.i64	d13, q8
1229	vmovn.i64	d14, q9
1230	add	lr, r0, #0xa4
1231	vstm.32	lr, {d10-d14}
1232	# h (accumulator) = 0
1233	eor	r6, r6, r6
1234	eor	r7, r7, r7
1235	eor	r8, r8, r8
1236	eor	r9, r9, r9
1237	add	lr, r0, #16
1238	eor	r4, r4, r4
1239	eor	r5, r5, r5
1240	stm	lr, {r4, r5, r6, r7, r8, r9}
1241	# Zero leftover
1242	str	r5, [r0, #56]
1243	vpop	{d8-d15}
1244	pop	{r4, r5, r6, r7, r8, r9, r10, r11, pc}
1245	.size	poly1305_set_key,.-poly1305_set_key
1246	.text
1247	.align	4
1248	.globl	poly1305_final
1249	.type	poly1305_final, %function
1250poly1305_final:
1251	push	{r4, r5, r6, r7, r8, r9, lr}
1252	add	r9, r0, #16
1253	ldm	r9, {r4, r5, r6, r7, r8}
1254	# Add 5 and check for h larger than p.
1255	adds	r2, r4, #5
1256	adcs	r2, r5, #0
1257	adcs	r2, r6, #0
1258	adcs	r2, r7, #0
1259	adc	r2, r8, #0
1260	sub	r2, r2, #4
1261	lsr	r2, r2, #31
1262	sub	r2, r2, #1
1263	and	r2, r2, #5
1264	# Add 0/5 to h.
1265	adds	r4, r4, r2
1266	adcs	r5, r5, #0
1267	adcs	r6, r6, #0
1268	adc	r7, r7, #0
1269	# Add padding
1270	add	r9, r0, #40
1271	ldm	r9, {r2, r3, r12, lr}
1272	adds	r4, r4, r2
1273	adcs	r5, r5, r3
1274	adcs	r6, r6, r12
1275	adc	r7, r7, lr
1276	# Store MAC
1277	str	r4, [r1]
1278	str	r5, [r1, #4]
1279	str	r6, [r1, #8]
1280	str	r7, [r1, #12]
1281	# Zero out h.
1282	eor	r4, r4, r4
1283	eor	r5, r5, r5
1284	eor	r6, r6, r6
1285	eor	r7, r7, r7
1286	eor	r8, r8, r8
1287	add	r9, r0, #16
1288	stm	r9, {r4, r5, r6, r7, r8}
1289	# Zero out r.
1290	add	r9, r0, #0
1291	stm	r9, {r4, r5, r6, r7}
1292	# Zero out padding.
1293	add	r9, r0, #40
1294	stm	r9, {r4, r5, r6, r7}
1295	pop	{r4, r5, r6, r7, r8, r9, pc}
1296	.size	poly1305_final,.-poly1305_final
1297#endif /* WOLFSSL_ARMASM_NO_NEON */
1298#endif /* HAVE_POLY1305 */
1299#endif /* !__aarch64__ && !WOLFSSL_ARMASM_THUMB2 */
1300#endif /* WOLFSSL_ARMASM */
1301
1302#if defined(__linux__) && defined(__ELF__)
1303.section	.note.GNU-stack,"",%progbits
1304#endif
1305#endif /* !WOLFSSL_ARMASM_INLINE */