luna - wolfssl/wolfcrypt/src/port/arm/armv8-sha512-asm.S

   1/* armv8-sha512-asm
   2 *
   3 * Copyright (C) 2006-2026 wolfSSL Inc.
   4 *
   5 * This file is part of wolfSSL.
   6 *
   7 * wolfSSL is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License as published by
   9 * the Free Software Foundation; either version 3 of the License, or
  10 * (at your option) any later version.
  11 *
  12 * wolfSSL is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 * GNU General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU General Public License
  18 * along with this program; if not, write to the Free Software
  19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
  20 */
  21
  22#include <wolfssl/wolfcrypt/libwolfssl_sources_asm.h>
  23
  24/* Generated using (from wolfssl):
  25 *   cd ../scripts
  26 *   ruby ./sha2/sha512.rb arm64 \
  27 *       ../wolfssl/wolfcrypt/src/port/arm/armv8-sha512-asm.S
  28 */
  29#ifdef WOLFSSL_ARMASM
  30#ifdef __aarch64__
  31#ifndef WOLFSSL_ARMASM_INLINE
  32#if defined(WOLFSSL_SHA512) || defined(WOLFSSL_SHA384)
  33#ifndef __APPLE__
  34	.text
  35	.section	.rodata
  36	.type	L_SHA512_transform_neon_len_k, %object
  37	.size	L_SHA512_transform_neon_len_k, 640
  38#else
  39	.section	__DATA,__data
  40#endif /* __APPLE__ */
  41	# 16-byte aligned, 128-bit aligned
  42#ifndef __APPLE__
  43	.align	4
  44#else
  45	.p2align	4
  46#endif /* __APPLE__ */
  47L_SHA512_transform_neon_len_k:
  48	.quad	0x428a2f98d728ae22,0x7137449123ef65cd
  49	.quad	0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
  50	.quad	0x3956c25bf348b538,0x59f111f1b605d019
  51	.quad	0x923f82a4af194f9b,0xab1c5ed5da6d8118
  52	.quad	0xd807aa98a3030242,0x12835b0145706fbe
  53	.quad	0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
  54	.quad	0x72be5d74f27b896f,0x80deb1fe3b1696b1
  55	.quad	0x9bdc06a725c71235,0xc19bf174cf692694
  56	.quad	0xe49b69c19ef14ad2,0xefbe4786384f25e3
  57	.quad	0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
  58	.quad	0x2de92c6f592b0275,0x4a7484aa6ea6e483
  59	.quad	0x5cb0a9dcbd41fbd4,0x76f988da831153b5
  60	.quad	0x983e5152ee66dfab,0xa831c66d2db43210
  61	.quad	0xb00327c898fb213f,0xbf597fc7beef0ee4
  62	.quad	0xc6e00bf33da88fc2,0xd5a79147930aa725
  63	.quad	0x06ca6351e003826f,0x142929670a0e6e70
  64	.quad	0x27b70a8546d22ffc,0x2e1b21385c26c926
  65	.quad	0x4d2c6dfc5ac42aed,0x53380d139d95b3df
  66	.quad	0x650a73548baf63de,0x766a0abb3c77b2a8
  67	.quad	0x81c2c92e47edaee6,0x92722c851482353b
  68	.quad	0xa2bfe8a14cf10364,0xa81a664bbc423001
  69	.quad	0xc24b8b70d0f89791,0xc76c51a30654be30
  70	.quad	0xd192e819d6ef5218,0xd69906245565a910
  71	.quad	0xf40e35855771202a,0x106aa07032bbd1b8
  72	.quad	0x19a4c116b8d2d0c8,0x1e376c085141ab53
  73	.quad	0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
  74	.quad	0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
  75	.quad	0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
  76	.quad	0x748f82ee5defb2fc,0x78a5636f43172f60
  77	.quad	0x84c87814a1f0ab72,0x8cc702081a6439ec
  78	.quad	0x90befffa23631e28,0xa4506cebde82bde9
  79	.quad	0xbef9a3f7b2c67915,0xc67178f2e372532b
  80	.quad	0xca273eceea26619c,0xd186b8c721c0c207
  81	.quad	0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
  82	.quad	0x06f067aa72176fba,0x0a637dc5a2c898a6
  83	.quad	0x113f9804bef90dae,0x1b710b35131c471b
  84	.quad	0x28db77f523047d84,0x32caab7b40c72493
  85	.quad	0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
  86	.quad	0x4cc5d4becb3e42b6,0x597f299cfc657e2a
  87	.quad	0x5fcb6fab3ad6faec,0x6c44198c4a475817
  88#ifndef __APPLE__
  89	.text
  90	.section	.rodata
  91	.type	L_SHA512_transform_neon_len_r8, %object
  92	.size	L_SHA512_transform_neon_len_r8, 16
  93#else
  94	.section	__DATA,__data
  95#endif /* __APPLE__ */
  96	# 16-byte aligned, 128-bit aligned
  97#ifndef __APPLE__
  98	.align	4
  99#else
 100	.p2align	4
 101#endif /* __APPLE__ */
 102L_SHA512_transform_neon_len_r8:
 103	.quad	0x0007060504030201,0x080f0e0d0c0b0a09
 104#ifndef __APPLE__
 105.text
 106.globl	Transform_Sha512_Len_neon
 107.type	Transform_Sha512_Len_neon,@function
 108.align	2
 109Transform_Sha512_Len_neon:
 110#else
 111.section	__TEXT,__text
 112.globl	_Transform_Sha512_Len_neon
 113.p2align	2
 114_Transform_Sha512_Len_neon:
 115#endif /* __APPLE__ */
 116	stp	x29, x30, [sp, #-128]!
 117	add	x29, sp, #0
 118	stp	x17, x19, [x29, #16]
 119	stp	x20, x21, [x29, #32]
 120	stp	x22, x23, [x29, #48]
 121	stp	x24, x25, [x29, #64]
 122	stp	x26, x27, [x29, #80]
 123	stp	d8, d9, [x29, #96]
 124	stp	d10, d11, [x29, #112]
 125#ifndef __APPLE__
 126	adrp x3, L_SHA512_transform_neon_len_k
 127	add  x3, x3, :lo12:L_SHA512_transform_neon_len_k
 128#else
 129	adrp x3, L_SHA512_transform_neon_len_k@PAGE
 130	add  x3, x3, L_SHA512_transform_neon_len_k@PAGEOFF
 131#endif /* __APPLE__ */
 132#ifndef __APPLE__
 133	adrp x27, L_SHA512_transform_neon_len_r8
 134	add  x27, x27, :lo12:L_SHA512_transform_neon_len_r8
 135#else
 136	adrp x27, L_SHA512_transform_neon_len_r8@PAGE
 137	add  x27, x27, L_SHA512_transform_neon_len_r8@PAGEOFF
 138#endif /* __APPLE__ */
 139	ld1	{v11.16b}, [x27]
 140	# Load digest into working vars
 141	ldp	x4, x5, [x0]
 142	ldp	x6, x7, [x0, #16]
 143	ldp	x8, x9, [x0, #32]
 144	ldp	x10, x11, [x0, #48]
 145	# Start of loop processing a block
 146L_sha512_len_neon_begin:
 147	# Load W
 148	# Copy digest to add in at end
 149	ld1	{v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40
 150	mov	x19, x4
 151	ld1	{v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40
 152	mov	x20, x5
 153	rev64	v0.16b, v0.16b
 154	mov	x21, x6
 155	rev64	v1.16b, v1.16b
 156	mov	x22, x7
 157	rev64	v2.16b, v2.16b
 158	mov	x23, x8
 159	rev64	v3.16b, v3.16b
 160	mov	x24, x9
 161	rev64	v4.16b, v4.16b
 162	mov	x25, x10
 163	rev64	v5.16b, v5.16b
 164	mov	x26, x11
 165	rev64	v6.16b, v6.16b
 166	rev64	v7.16b, v7.16b
 167	# Pre-calc: b ^ c
 168	eor	x16, x5, x6
 169	mov	x27, #4
 170	# Start of 16 rounds
 171L_sha512_len_neon_start:
 172	# Round 0
 173	mov	x13, v0.d[0]
 174	ldr	x15, [x3], #8
 175	ror	x12, x8, #14
 176	ror	x14, x4, #28
 177	eor	x12, x12, x8, ror 18
 178	eor	x14, x14, x4, ror 34
 179	eor	x12, x12, x8, ror 41
 180	eor	x14, x14, x4, ror 39
 181	add	x11, x11, x12
 182	eor	x17, x4, x5
 183	eor	x12, x9, x10
 184	and	x16, x17, x16
 185	and	x12, x12, x8
 186	add	x11, x11, x13
 187	eor	x12, x12, x10
 188	add	x11, x11, x15
 189	eor	x16, x16, x5
 190	add	x11, x11, x12
 191	add	x14, x14, x16
 192	add	x7, x7, x11
 193	add	x11, x11, x14
 194	# Round 1
 195	mov	x13, v0.d[1]
 196	ldr	x15, [x3], #8
 197	ext	v10.16b, v0.16b, v1.16b, #8
 198	ror	x12, x7, #14
 199	shl	v8.2d, v7.2d, #45
 200	ror	x14, x11, #28
 201	sri	v8.2d, v7.2d, #19
 202	eor	x12, x12, x7, ror 18
 203	shl	v9.2d, v7.2d, #3
 204	eor	x14, x14, x11, ror 34
 205	sri	v9.2d, v7.2d, #61
 206	eor	x12, x12, x7, ror 41
 207	eor	v9.16b, v9.16b, v8.16b
 208	eor	x14, x14, x11, ror 39
 209	ushr	v8.2d, v7.2d, #6
 210	add	x10, x10, x12
 211	eor	v9.16b, v9.16b, v8.16b
 212	eor	x16, x11, x4
 213	add	v0.2d, v0.2d, v9.2d
 214	eor	x12, x8, x9
 215	ext	v9.16b, v4.16b, v5.16b, #8
 216	and	x17, x16, x17
 217	add	v0.2d, v0.2d, v9.2d
 218	and	x12, x12, x7
 219	shl	v8.2d, v10.2d, #63
 220	add	x10, x10, x13
 221	sri	v8.2d, v10.2d, #1
 222	eor	x12, x12, x9
 223	tbl	v9.16b, {v10.16b}, v11.16b
 224	add	x10, x10, x15
 225	eor	v9.16b, v9.16b, v8.16b
 226	eor	x17, x17, x4
 227	ushr	v10.2d, v10.2d, #7
 228	add	x10, x10, x12
 229	eor	v9.16b, v9.16b, v10.16b
 230	add	x14, x14, x17
 231	add	v0.2d, v0.2d, v9.2d
 232	add	x6, x6, x10
 233	add	x10, x10, x14
 234	# Round 2
 235	mov	x13, v1.d[0]
 236	ldr	x15, [x3], #8
 237	ror	x12, x6, #14
 238	ror	x14, x10, #28
 239	eor	x12, x12, x6, ror 18
 240	eor	x14, x14, x10, ror 34
 241	eor	x12, x12, x6, ror 41
 242	eor	x14, x14, x10, ror 39
 243	add	x9, x9, x12
 244	eor	x17, x10, x11
 245	eor	x12, x7, x8
 246	and	x16, x17, x16
 247	and	x12, x12, x6
 248	add	x9, x9, x13
 249	eor	x12, x12, x8
 250	add	x9, x9, x15
 251	eor	x16, x16, x11
 252	add	x9, x9, x12
 253	add	x14, x14, x16
 254	add	x5, x5, x9
 255	add	x9, x9, x14
 256	# Round 3
 257	mov	x13, v1.d[1]
 258	ldr	x15, [x3], #8
 259	ext	v10.16b, v1.16b, v2.16b, #8
 260	ror	x12, x5, #14
 261	shl	v8.2d, v0.2d, #45
 262	ror	x14, x9, #28
 263	sri	v8.2d, v0.2d, #19
 264	eor	x12, x12, x5, ror 18
 265	shl	v9.2d, v0.2d, #3
 266	eor	x14, x14, x9, ror 34
 267	sri	v9.2d, v0.2d, #61
 268	eor	x12, x12, x5, ror 41
 269	eor	v9.16b, v9.16b, v8.16b
 270	eor	x14, x14, x9, ror 39
 271	ushr	v8.2d, v0.2d, #6
 272	add	x8, x8, x12
 273	eor	v9.16b, v9.16b, v8.16b
 274	eor	x16, x9, x10
 275	add	v1.2d, v1.2d, v9.2d
 276	eor	x12, x6, x7
 277	ext	v9.16b, v5.16b, v6.16b, #8
 278	and	x17, x16, x17
 279	add	v1.2d, v1.2d, v9.2d
 280	and	x12, x12, x5
 281	shl	v8.2d, v10.2d, #63
 282	add	x8, x8, x13
 283	sri	v8.2d, v10.2d, #1
 284	eor	x12, x12, x7
 285	tbl	v9.16b, {v10.16b}, v11.16b
 286	add	x8, x8, x15
 287	eor	v9.16b, v9.16b, v8.16b
 288	eor	x17, x17, x10
 289	ushr	v10.2d, v10.2d, #7
 290	add	x8, x8, x12
 291	eor	v9.16b, v9.16b, v10.16b
 292	add	x14, x14, x17
 293	add	v1.2d, v1.2d, v9.2d
 294	add	x4, x4, x8
 295	add	x8, x8, x14
 296	# Round 4
 297	mov	x13, v2.d[0]
 298	ldr	x15, [x3], #8
 299	ror	x12, x4, #14
 300	ror	x14, x8, #28
 301	eor	x12, x12, x4, ror 18
 302	eor	x14, x14, x8, ror 34
 303	eor	x12, x12, x4, ror 41
 304	eor	x14, x14, x8, ror 39
 305	add	x7, x7, x12
 306	eor	x17, x8, x9
 307	eor	x12, x5, x6
 308	and	x16, x17, x16
 309	and	x12, x12, x4
 310	add	x7, x7, x13
 311	eor	x12, x12, x6
 312	add	x7, x7, x15
 313	eor	x16, x16, x9
 314	add	x7, x7, x12
 315	add	x14, x14, x16
 316	add	x11, x11, x7
 317	add	x7, x7, x14
 318	# Round 5
 319	mov	x13, v2.d[1]
 320	ldr	x15, [x3], #8
 321	ext	v10.16b, v2.16b, v3.16b, #8
 322	ror	x12, x11, #14
 323	shl	v8.2d, v1.2d, #45
 324	ror	x14, x7, #28
 325	sri	v8.2d, v1.2d, #19
 326	eor	x12, x12, x11, ror 18
 327	shl	v9.2d, v1.2d, #3
 328	eor	x14, x14, x7, ror 34
 329	sri	v9.2d, v1.2d, #61
 330	eor	x12, x12, x11, ror 41
 331	eor	v9.16b, v9.16b, v8.16b
 332	eor	x14, x14, x7, ror 39
 333	ushr	v8.2d, v1.2d, #6
 334	add	x6, x6, x12
 335	eor	v9.16b, v9.16b, v8.16b
 336	eor	x16, x7, x8
 337	add	v2.2d, v2.2d, v9.2d
 338	eor	x12, x4, x5
 339	ext	v9.16b, v6.16b, v7.16b, #8
 340	and	x17, x16, x17
 341	add	v2.2d, v2.2d, v9.2d
 342	and	x12, x12, x11
 343	shl	v8.2d, v10.2d, #63
 344	add	x6, x6, x13
 345	sri	v8.2d, v10.2d, #1
 346	eor	x12, x12, x5
 347	tbl	v9.16b, {v10.16b}, v11.16b
 348	add	x6, x6, x15
 349	eor	v9.16b, v9.16b, v8.16b
 350	eor	x17, x17, x8
 351	ushr	v10.2d, v10.2d, #7
 352	add	x6, x6, x12
 353	eor	v9.16b, v9.16b, v10.16b
 354	add	x14, x14, x17
 355	add	v2.2d, v2.2d, v9.2d
 356	add	x10, x10, x6
 357	add	x6, x6, x14
 358	# Round 6
 359	mov	x13, v3.d[0]
 360	ldr	x15, [x3], #8
 361	ror	x12, x10, #14
 362	ror	x14, x6, #28
 363	eor	x12, x12, x10, ror 18
 364	eor	x14, x14, x6, ror 34
 365	eor	x12, x12, x10, ror 41
 366	eor	x14, x14, x6, ror 39
 367	add	x5, x5, x12
 368	eor	x17, x6, x7
 369	eor	x12, x11, x4
 370	and	x16, x17, x16
 371	and	x12, x12, x10
 372	add	x5, x5, x13
 373	eor	x12, x12, x4
 374	add	x5, x5, x15
 375	eor	x16, x16, x7
 376	add	x5, x5, x12
 377	add	x14, x14, x16
 378	add	x9, x9, x5
 379	add	x5, x5, x14
 380	# Round 7
 381	mov	x13, v3.d[1]
 382	ldr	x15, [x3], #8
 383	ext	v10.16b, v3.16b, v4.16b, #8
 384	ror	x12, x9, #14
 385	shl	v8.2d, v2.2d, #45
 386	ror	x14, x5, #28
 387	sri	v8.2d, v2.2d, #19
 388	eor	x12, x12, x9, ror 18
 389	shl	v9.2d, v2.2d, #3
 390	eor	x14, x14, x5, ror 34
 391	sri	v9.2d, v2.2d, #61
 392	eor	x12, x12, x9, ror 41
 393	eor	v9.16b, v9.16b, v8.16b
 394	eor	x14, x14, x5, ror 39
 395	ushr	v8.2d, v2.2d, #6
 396	add	x4, x4, x12
 397	eor	v9.16b, v9.16b, v8.16b
 398	eor	x16, x5, x6
 399	add	v3.2d, v3.2d, v9.2d
 400	eor	x12, x10, x11
 401	ext	v9.16b, v7.16b, v0.16b, #8
 402	and	x17, x16, x17
 403	add	v3.2d, v3.2d, v9.2d
 404	and	x12, x12, x9
 405	shl	v8.2d, v10.2d, #63
 406	add	x4, x4, x13
 407	sri	v8.2d, v10.2d, #1
 408	eor	x12, x12, x11
 409	tbl	v9.16b, {v10.16b}, v11.16b
 410	add	x4, x4, x15
 411	eor	v9.16b, v9.16b, v8.16b
 412	eor	x17, x17, x6
 413	ushr	v10.2d, v10.2d, #7
 414	add	x4, x4, x12
 415	eor	v9.16b, v9.16b, v10.16b
 416	add	x14, x14, x17
 417	add	v3.2d, v3.2d, v9.2d
 418	add	x8, x8, x4
 419	add	x4, x4, x14
 420	# Round 8
 421	mov	x13, v4.d[0]
 422	ldr	x15, [x3], #8
 423	ror	x12, x8, #14
 424	ror	x14, x4, #28
 425	eor	x12, x12, x8, ror 18
 426	eor	x14, x14, x4, ror 34
 427	eor	x12, x12, x8, ror 41
 428	eor	x14, x14, x4, ror 39
 429	add	x11, x11, x12
 430	eor	x17, x4, x5
 431	eor	x12, x9, x10
 432	and	x16, x17, x16
 433	and	x12, x12, x8
 434	add	x11, x11, x13
 435	eor	x12, x12, x10
 436	add	x11, x11, x15
 437	eor	x16, x16, x5
 438	add	x11, x11, x12
 439	add	x14, x14, x16
 440	add	x7, x7, x11
 441	add	x11, x11, x14
 442	# Round 9
 443	mov	x13, v4.d[1]
 444	ldr	x15, [x3], #8
 445	ext	v10.16b, v4.16b, v5.16b, #8
 446	ror	x12, x7, #14
 447	shl	v8.2d, v3.2d, #45
 448	ror	x14, x11, #28
 449	sri	v8.2d, v3.2d, #19
 450	eor	x12, x12, x7, ror 18
 451	shl	v9.2d, v3.2d, #3
 452	eor	x14, x14, x11, ror 34
 453	sri	v9.2d, v3.2d, #61
 454	eor	x12, x12, x7, ror 41
 455	eor	v9.16b, v9.16b, v8.16b
 456	eor	x14, x14, x11, ror 39
 457	ushr	v8.2d, v3.2d, #6
 458	add	x10, x10, x12
 459	eor	v9.16b, v9.16b, v8.16b
 460	eor	x16, x11, x4
 461	add	v4.2d, v4.2d, v9.2d
 462	eor	x12, x8, x9
 463	ext	v9.16b, v0.16b, v1.16b, #8
 464	and	x17, x16, x17
 465	add	v4.2d, v4.2d, v9.2d
 466	and	x12, x12, x7
 467	shl	v8.2d, v10.2d, #63
 468	add	x10, x10, x13
 469	sri	v8.2d, v10.2d, #1
 470	eor	x12, x12, x9
 471	tbl	v9.16b, {v10.16b}, v11.16b
 472	add	x10, x10, x15
 473	eor	v9.16b, v9.16b, v8.16b
 474	eor	x17, x17, x4
 475	ushr	v10.2d, v10.2d, #7
 476	add	x10, x10, x12
 477	eor	v9.16b, v9.16b, v10.16b
 478	add	x14, x14, x17
 479	add	v4.2d, v4.2d, v9.2d
 480	add	x6, x6, x10
 481	add	x10, x10, x14
 482	# Round 10
 483	mov	x13, v5.d[0]
 484	ldr	x15, [x3], #8
 485	ror	x12, x6, #14
 486	ror	x14, x10, #28
 487	eor	x12, x12, x6, ror 18
 488	eor	x14, x14, x10, ror 34
 489	eor	x12, x12, x6, ror 41
 490	eor	x14, x14, x10, ror 39
 491	add	x9, x9, x12
 492	eor	x17, x10, x11
 493	eor	x12, x7, x8
 494	and	x16, x17, x16
 495	and	x12, x12, x6
 496	add	x9, x9, x13
 497	eor	x12, x12, x8
 498	add	x9, x9, x15
 499	eor	x16, x16, x11
 500	add	x9, x9, x12
 501	add	x14, x14, x16
 502	add	x5, x5, x9
 503	add	x9, x9, x14
 504	# Round 11
 505	mov	x13, v5.d[1]
 506	ldr	x15, [x3], #8
 507	ext	v10.16b, v5.16b, v6.16b, #8
 508	ror	x12, x5, #14
 509	shl	v8.2d, v4.2d, #45
 510	ror	x14, x9, #28
 511	sri	v8.2d, v4.2d, #19
 512	eor	x12, x12, x5, ror 18
 513	shl	v9.2d, v4.2d, #3
 514	eor	x14, x14, x9, ror 34
 515	sri	v9.2d, v4.2d, #61
 516	eor	x12, x12, x5, ror 41
 517	eor	v9.16b, v9.16b, v8.16b
 518	eor	x14, x14, x9, ror 39
 519	ushr	v8.2d, v4.2d, #6
 520	add	x8, x8, x12
 521	eor	v9.16b, v9.16b, v8.16b
 522	eor	x16, x9, x10
 523	add	v5.2d, v5.2d, v9.2d
 524	eor	x12, x6, x7
 525	ext	v9.16b, v1.16b, v2.16b, #8
 526	and	x17, x16, x17
 527	add	v5.2d, v5.2d, v9.2d
 528	and	x12, x12, x5
 529	shl	v8.2d, v10.2d, #63
 530	add	x8, x8, x13
 531	sri	v8.2d, v10.2d, #1
 532	eor	x12, x12, x7
 533	tbl	v9.16b, {v10.16b}, v11.16b
 534	add	x8, x8, x15
 535	eor	v9.16b, v9.16b, v8.16b
 536	eor	x17, x17, x10
 537	ushr	v10.2d, v10.2d, #7
 538	add	x8, x8, x12
 539	eor	v9.16b, v9.16b, v10.16b
 540	add	x14, x14, x17
 541	add	v5.2d, v5.2d, v9.2d
 542	add	x4, x4, x8
 543	add	x8, x8, x14
 544	# Round 12
 545	mov	x13, v6.d[0]
 546	ldr	x15, [x3], #8
 547	ror	x12, x4, #14
 548	ror	x14, x8, #28
 549	eor	x12, x12, x4, ror 18
 550	eor	x14, x14, x8, ror 34
 551	eor	x12, x12, x4, ror 41
 552	eor	x14, x14, x8, ror 39
 553	add	x7, x7, x12
 554	eor	x17, x8, x9
 555	eor	x12, x5, x6
 556	and	x16, x17, x16
 557	and	x12, x12, x4
 558	add	x7, x7, x13
 559	eor	x12, x12, x6
 560	add	x7, x7, x15
 561	eor	x16, x16, x9
 562	add	x7, x7, x12
 563	add	x14, x14, x16
 564	add	x11, x11, x7
 565	add	x7, x7, x14
 566	# Round 13
 567	mov	x13, v6.d[1]
 568	ldr	x15, [x3], #8
 569	ext	v10.16b, v6.16b, v7.16b, #8
 570	ror	x12, x11, #14
 571	shl	v8.2d, v5.2d, #45
 572	ror	x14, x7, #28
 573	sri	v8.2d, v5.2d, #19
 574	eor	x12, x12, x11, ror 18
 575	shl	v9.2d, v5.2d, #3
 576	eor	x14, x14, x7, ror 34
 577	sri	v9.2d, v5.2d, #61
 578	eor	x12, x12, x11, ror 41
 579	eor	v9.16b, v9.16b, v8.16b
 580	eor	x14, x14, x7, ror 39
 581	ushr	v8.2d, v5.2d, #6
 582	add	x6, x6, x12
 583	eor	v9.16b, v9.16b, v8.16b
 584	eor	x16, x7, x8
 585	add	v6.2d, v6.2d, v9.2d
 586	eor	x12, x4, x5
 587	ext	v9.16b, v2.16b, v3.16b, #8
 588	and	x17, x16, x17
 589	add	v6.2d, v6.2d, v9.2d
 590	and	x12, x12, x11
 591	shl	v8.2d, v10.2d, #63
 592	add	x6, x6, x13
 593	sri	v8.2d, v10.2d, #1
 594	eor	x12, x12, x5
 595	tbl	v9.16b, {v10.16b}, v11.16b
 596	add	x6, x6, x15
 597	eor	v9.16b, v9.16b, v8.16b
 598	eor	x17, x17, x8
 599	ushr	v10.2d, v10.2d, #7
 600	add	x6, x6, x12
 601	eor	v9.16b, v9.16b, v10.16b
 602	add	x14, x14, x17
 603	add	v6.2d, v6.2d, v9.2d
 604	add	x10, x10, x6
 605	add	x6, x6, x14
 606	# Round 14
 607	mov	x13, v7.d[0]
 608	ldr	x15, [x3], #8
 609	ror	x12, x10, #14
 610	ror	x14, x6, #28
 611	eor	x12, x12, x10, ror 18
 612	eor	x14, x14, x6, ror 34
 613	eor	x12, x12, x10, ror 41
 614	eor	x14, x14, x6, ror 39
 615	add	x5, x5, x12
 616	eor	x17, x6, x7
 617	eor	x12, x11, x4
 618	and	x16, x17, x16
 619	and	x12, x12, x10
 620	add	x5, x5, x13
 621	eor	x12, x12, x4
 622	add	x5, x5, x15
 623	eor	x16, x16, x7
 624	add	x5, x5, x12
 625	add	x14, x14, x16
 626	add	x9, x9, x5
 627	add	x5, x5, x14
 628	# Round 15
 629	mov	x13, v7.d[1]
 630	ldr	x15, [x3], #8
 631	ext	v10.16b, v7.16b, v0.16b, #8
 632	ror	x12, x9, #14
 633	shl	v8.2d, v6.2d, #45
 634	ror	x14, x5, #28
 635	sri	v8.2d, v6.2d, #19
 636	eor	x12, x12, x9, ror 18
 637	shl	v9.2d, v6.2d, #3
 638	eor	x14, x14, x5, ror 34
 639	sri	v9.2d, v6.2d, #61
 640	eor	x12, x12, x9, ror 41
 641	eor	v9.16b, v9.16b, v8.16b
 642	eor	x14, x14, x5, ror 39
 643	ushr	v8.2d, v6.2d, #6
 644	add	x4, x4, x12
 645	eor	v9.16b, v9.16b, v8.16b
 646	eor	x16, x5, x6
 647	add	v7.2d, v7.2d, v9.2d
 648	eor	x12, x10, x11
 649	ext	v9.16b, v3.16b, v4.16b, #8
 650	and	x17, x16, x17
 651	add	v7.2d, v7.2d, v9.2d
 652	and	x12, x12, x9
 653	shl	v8.2d, v10.2d, #63
 654	add	x4, x4, x13
 655	sri	v8.2d, v10.2d, #1
 656	eor	x12, x12, x11
 657	tbl	v9.16b, {v10.16b}, v11.16b
 658	add	x4, x4, x15
 659	eor	v9.16b, v9.16b, v8.16b
 660	eor	x17, x17, x6
 661	ushr	v10.2d, v10.2d, #7
 662	add	x4, x4, x12
 663	eor	v9.16b, v9.16b, v10.16b
 664	add	x14, x14, x17
 665	add	v7.2d, v7.2d, v9.2d
 666	add	x8, x8, x4
 667	add	x4, x4, x14
 668	subs	x27, x27, #1
 669	bne	L_sha512_len_neon_start
 670	# Round 0
 671	mov	x13, v0.d[0]
 672	ldr	x15, [x3], #8
 673	ror	x12, x8, #14
 674	ror	x14, x4, #28
 675	eor	x12, x12, x8, ror 18
 676	eor	x14, x14, x4, ror 34
 677	eor	x12, x12, x8, ror 41
 678	eor	x14, x14, x4, ror 39
 679	add	x11, x11, x12
 680	eor	x17, x4, x5
 681	eor	x12, x9, x10
 682	and	x16, x17, x16
 683	and	x12, x12, x8
 684	add	x11, x11, x13
 685	eor	x12, x12, x10
 686	add	x11, x11, x15
 687	eor	x16, x16, x5
 688	add	x11, x11, x12
 689	add	x14, x14, x16
 690	add	x7, x7, x11
 691	add	x11, x11, x14
 692	# Round 1
 693	mov	x13, v0.d[1]
 694	ldr	x15, [x3], #8
 695	ror	x12, x7, #14
 696	ror	x14, x11, #28
 697	eor	x12, x12, x7, ror 18
 698	eor	x14, x14, x11, ror 34
 699	eor	x12, x12, x7, ror 41
 700	eor	x14, x14, x11, ror 39
 701	add	x10, x10, x12
 702	eor	x16, x11, x4
 703	eor	x12, x8, x9
 704	and	x17, x16, x17
 705	and	x12, x12, x7
 706	add	x10, x10, x13
 707	eor	x12, x12, x9
 708	add	x10, x10, x15
 709	eor	x17, x17, x4
 710	add	x10, x10, x12
 711	add	x14, x14, x17
 712	add	x6, x6, x10
 713	add	x10, x10, x14
 714	# Round 2
 715	mov	x13, v1.d[0]
 716	ldr	x15, [x3], #8
 717	ror	x12, x6, #14
 718	ror	x14, x10, #28
 719	eor	x12, x12, x6, ror 18
 720	eor	x14, x14, x10, ror 34
 721	eor	x12, x12, x6, ror 41
 722	eor	x14, x14, x10, ror 39
 723	add	x9, x9, x12
 724	eor	x17, x10, x11
 725	eor	x12, x7, x8
 726	and	x16, x17, x16
 727	and	x12, x12, x6
 728	add	x9, x9, x13
 729	eor	x12, x12, x8
 730	add	x9, x9, x15
 731	eor	x16, x16, x11
 732	add	x9, x9, x12
 733	add	x14, x14, x16
 734	add	x5, x5, x9
 735	add	x9, x9, x14
 736	# Round 3
 737	mov	x13, v1.d[1]
 738	ldr	x15, [x3], #8
 739	ror	x12, x5, #14
 740	ror	x14, x9, #28
 741	eor	x12, x12, x5, ror 18
 742	eor	x14, x14, x9, ror 34
 743	eor	x12, x12, x5, ror 41
 744	eor	x14, x14, x9, ror 39
 745	add	x8, x8, x12
 746	eor	x16, x9, x10
 747	eor	x12, x6, x7
 748	and	x17, x16, x17
 749	and	x12, x12, x5
 750	add	x8, x8, x13
 751	eor	x12, x12, x7
 752	add	x8, x8, x15
 753	eor	x17, x17, x10
 754	add	x8, x8, x12
 755	add	x14, x14, x17
 756	add	x4, x4, x8
 757	add	x8, x8, x14
 758	# Round 4
 759	mov	x13, v2.d[0]
 760	ldr	x15, [x3], #8
 761	ror	x12, x4, #14
 762	ror	x14, x8, #28
 763	eor	x12, x12, x4, ror 18
 764	eor	x14, x14, x8, ror 34
 765	eor	x12, x12, x4, ror 41
 766	eor	x14, x14, x8, ror 39
 767	add	x7, x7, x12
 768	eor	x17, x8, x9
 769	eor	x12, x5, x6
 770	and	x16, x17, x16
 771	and	x12, x12, x4
 772	add	x7, x7, x13
 773	eor	x12, x12, x6
 774	add	x7, x7, x15
 775	eor	x16, x16, x9
 776	add	x7, x7, x12
 777	add	x14, x14, x16
 778	add	x11, x11, x7
 779	add	x7, x7, x14
 780	# Round 5
 781	mov	x13, v2.d[1]
 782	ldr	x15, [x3], #8
 783	ror	x12, x11, #14
 784	ror	x14, x7, #28
 785	eor	x12, x12, x11, ror 18
 786	eor	x14, x14, x7, ror 34
 787	eor	x12, x12, x11, ror 41
 788	eor	x14, x14, x7, ror 39
 789	add	x6, x6, x12
 790	eor	x16, x7, x8
 791	eor	x12, x4, x5
 792	and	x17, x16, x17
 793	and	x12, x12, x11
 794	add	x6, x6, x13
 795	eor	x12, x12, x5
 796	add	x6, x6, x15
 797	eor	x17, x17, x8
 798	add	x6, x6, x12
 799	add	x14, x14, x17
 800	add	x10, x10, x6
 801	add	x6, x6, x14
 802	# Round 6
 803	mov	x13, v3.d[0]
 804	ldr	x15, [x3], #8
 805	ror	x12, x10, #14
 806	ror	x14, x6, #28
 807	eor	x12, x12, x10, ror 18
 808	eor	x14, x14, x6, ror 34
 809	eor	x12, x12, x10, ror 41
 810	eor	x14, x14, x6, ror 39
 811	add	x5, x5, x12
 812	eor	x17, x6, x7
 813	eor	x12, x11, x4
 814	and	x16, x17, x16
 815	and	x12, x12, x10
 816	add	x5, x5, x13
 817	eor	x12, x12, x4
 818	add	x5, x5, x15
 819	eor	x16, x16, x7
 820	add	x5, x5, x12
 821	add	x14, x14, x16
 822	add	x9, x9, x5
 823	add	x5, x5, x14
 824	# Round 7
 825	mov	x13, v3.d[1]
 826	ldr	x15, [x3], #8
 827	ror	x12, x9, #14
 828	ror	x14, x5, #28
 829	eor	x12, x12, x9, ror 18
 830	eor	x14, x14, x5, ror 34
 831	eor	x12, x12, x9, ror 41
 832	eor	x14, x14, x5, ror 39
 833	add	x4, x4, x12
 834	eor	x16, x5, x6
 835	eor	x12, x10, x11
 836	and	x17, x16, x17
 837	and	x12, x12, x9
 838	add	x4, x4, x13
 839	eor	x12, x12, x11
 840	add	x4, x4, x15
 841	eor	x17, x17, x6
 842	add	x4, x4, x12
 843	add	x14, x14, x17
 844	add	x8, x8, x4
 845	add	x4, x4, x14
 846	# Round 8
 847	mov	x13, v4.d[0]
 848	ldr	x15, [x3], #8
 849	ror	x12, x8, #14
 850	ror	x14, x4, #28
 851	eor	x12, x12, x8, ror 18
 852	eor	x14, x14, x4, ror 34
 853	eor	x12, x12, x8, ror 41
 854	eor	x14, x14, x4, ror 39
 855	add	x11, x11, x12
 856	eor	x17, x4, x5
 857	eor	x12, x9, x10
 858	and	x16, x17, x16
 859	and	x12, x12, x8
 860	add	x11, x11, x13
 861	eor	x12, x12, x10
 862	add	x11, x11, x15
 863	eor	x16, x16, x5
 864	add	x11, x11, x12
 865	add	x14, x14, x16
 866	add	x7, x7, x11
 867	add	x11, x11, x14
 868	# Round 9
 869	mov	x13, v4.d[1]
 870	ldr	x15, [x3], #8
 871	ror	x12, x7, #14
 872	ror	x14, x11, #28
 873	eor	x12, x12, x7, ror 18
 874	eor	x14, x14, x11, ror 34
 875	eor	x12, x12, x7, ror 41
 876	eor	x14, x14, x11, ror 39
 877	add	x10, x10, x12
 878	eor	x16, x11, x4
 879	eor	x12, x8, x9
 880	and	x17, x16, x17
 881	and	x12, x12, x7
 882	add	x10, x10, x13
 883	eor	x12, x12, x9
 884	add	x10, x10, x15
 885	eor	x17, x17, x4
 886	add	x10, x10, x12
 887	add	x14, x14, x17
 888	add	x6, x6, x10
 889	add	x10, x10, x14
 890	# Round 10
 891	mov	x13, v5.d[0]
 892	ldr	x15, [x3], #8
 893	ror	x12, x6, #14
 894	ror	x14, x10, #28
 895	eor	x12, x12, x6, ror 18
 896	eor	x14, x14, x10, ror 34
 897	eor	x12, x12, x6, ror 41
 898	eor	x14, x14, x10, ror 39
 899	add	x9, x9, x12
 900	eor	x17, x10, x11
 901	eor	x12, x7, x8
 902	and	x16, x17, x16
 903	and	x12, x12, x6
 904	add	x9, x9, x13
 905	eor	x12, x12, x8
 906	add	x9, x9, x15
 907	eor	x16, x16, x11
 908	add	x9, x9, x12
 909	add	x14, x14, x16
 910	add	x5, x5, x9
 911	add	x9, x9, x14
 912	# Round 11
 913	mov	x13, v5.d[1]
 914	ldr	x15, [x3], #8
 915	ror	x12, x5, #14
 916	ror	x14, x9, #28
 917	eor	x12, x12, x5, ror 18
 918	eor	x14, x14, x9, ror 34
 919	eor	x12, x12, x5, ror 41
 920	eor	x14, x14, x9, ror 39
 921	add	x8, x8, x12
 922	eor	x16, x9, x10
 923	eor	x12, x6, x7
 924	and	x17, x16, x17
 925	and	x12, x12, x5
 926	add	x8, x8, x13
 927	eor	x12, x12, x7
 928	add	x8, x8, x15
 929	eor	x17, x17, x10
 930	add	x8, x8, x12
 931	add	x14, x14, x17
 932	add	x4, x4, x8
 933	add	x8, x8, x14
 934	# Round 12
 935	mov	x13, v6.d[0]
 936	ldr	x15, [x3], #8
 937	ror	x12, x4, #14
 938	ror	x14, x8, #28
 939	eor	x12, x12, x4, ror 18
 940	eor	x14, x14, x8, ror 34
 941	eor	x12, x12, x4, ror 41
 942	eor	x14, x14, x8, ror 39
 943	add	x7, x7, x12
 944	eor	x17, x8, x9
 945	eor	x12, x5, x6
 946	and	x16, x17, x16
 947	and	x12, x12, x4
 948	add	x7, x7, x13
 949	eor	x12, x12, x6
 950	add	x7, x7, x15
 951	eor	x16, x16, x9
 952	add	x7, x7, x12
 953	add	x14, x14, x16
 954	add	x11, x11, x7
 955	add	x7, x7, x14
 956	# Round 13
 957	mov	x13, v6.d[1]
 958	ldr	x15, [x3], #8
 959	ror	x12, x11, #14
 960	ror	x14, x7, #28
 961	eor	x12, x12, x11, ror 18
 962	eor	x14, x14, x7, ror 34
 963	eor	x12, x12, x11, ror 41
 964	eor	x14, x14, x7, ror 39
 965	add	x6, x6, x12
 966	eor	x16, x7, x8
 967	eor	x12, x4, x5
 968	and	x17, x16, x17
 969	and	x12, x12, x11
 970	add	x6, x6, x13
 971	eor	x12, x12, x5
 972	add	x6, x6, x15
 973	eor	x17, x17, x8
 974	add	x6, x6, x12
 975	add	x14, x14, x17
 976	add	x10, x10, x6
 977	add	x6, x6, x14
 978	# Round 14
 979	mov	x13, v7.d[0]
 980	ldr	x15, [x3], #8
 981	ror	x12, x10, #14
 982	ror	x14, x6, #28
 983	eor	x12, x12, x10, ror 18
 984	eor	x14, x14, x6, ror 34
 985	eor	x12, x12, x10, ror 41
 986	eor	x14, x14, x6, ror 39
 987	add	x5, x5, x12
 988	eor	x17, x6, x7
 989	eor	x12, x11, x4
 990	and	x16, x17, x16
 991	and	x12, x12, x10
 992	add	x5, x5, x13
 993	eor	x12, x12, x4
 994	add	x5, x5, x15
 995	eor	x16, x16, x7
 996	add	x5, x5, x12
 997	add	x14, x14, x16
 998	add	x9, x9, x5
 999	add	x5, x5, x14
1000	# Round 15
1001	mov	x13, v7.d[1]
1002	ldr	x15, [x3], #8
1003	ror	x12, x9, #14
1004	ror	x14, x5, #28
1005	eor	x12, x12, x9, ror 18
1006	eor	x14, x14, x5, ror 34
1007	eor	x12, x12, x9, ror 41
1008	eor	x14, x14, x5, ror 39
1009	add	x4, x4, x12
1010	eor	x16, x5, x6
1011	eor	x12, x10, x11
1012	and	x17, x16, x17
1013	and	x12, x12, x9
1014	add	x4, x4, x13
1015	eor	x12, x12, x11
1016	add	x4, x4, x15
1017	eor	x17, x17, x6
1018	add	x4, x4, x12
1019	add	x14, x14, x17
1020	add	x8, x8, x4
1021	add	x4, x4, x14
1022	add	x11, x11, x26
1023	add	x10, x10, x25
1024	add	x9, x9, x24
1025	add	x8, x8, x23
1026	add	x7, x7, x22
1027	add	x6, x6, x21
1028	add	x5, x5, x20
1029	add	x4, x4, x19
1030	subs	w2, w2, #0x80
1031	sub	x3, x3, #0x280
1032	bne	L_sha512_len_neon_begin
1033	stp	x4, x5, [x0]
1034	stp	x6, x7, [x0, #16]
1035	stp	x8, x9, [x0, #32]
1036	stp	x10, x11, [x0, #48]
1037	ldp	x17, x19, [x29, #16]
1038	ldp	x20, x21, [x29, #32]
1039	ldp	x22, x23, [x29, #48]
1040	ldp	x24, x25, [x29, #64]
1041	ldp	x26, x27, [x29, #80]
1042	ldp	d8, d9, [x29, #96]
1043	ldp	d10, d11, [x29, #112]
1044	ldp	x29, x30, [sp], #0x80
1045	ret
1046#ifndef __APPLE__
1047	.size	Transform_Sha512_Len_neon,.-Transform_Sha512_Len_neon
1048#endif /* __APPLE__ */
1049#ifdef WOLFSSL_ARMASM_CRYPTO_SHA512
1050#ifndef __APPLE__
1051	.text
1052	.section	.rodata
1053	.type	L_SHA512_trans_crypto_len_k, %object
1054	.size	L_SHA512_trans_crypto_len_k, 640
1055#else
1056	.section	__DATA,__data
1057#endif /* __APPLE__ */
1058	# 16-byte aligned, 128-bit aligned
1059#ifndef __APPLE__
1060	.align	4
1061#else
1062	.p2align	4
1063#endif /* __APPLE__ */
1064L_SHA512_trans_crypto_len_k:
1065	.quad	0x428a2f98d728ae22,0x7137449123ef65cd
1066	.quad	0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
1067	.quad	0x3956c25bf348b538,0x59f111f1b605d019
1068	.quad	0x923f82a4af194f9b,0xab1c5ed5da6d8118
1069	.quad	0xd807aa98a3030242,0x12835b0145706fbe
1070	.quad	0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
1071	.quad	0x72be5d74f27b896f,0x80deb1fe3b1696b1
1072	.quad	0x9bdc06a725c71235,0xc19bf174cf692694
1073	.quad	0xe49b69c19ef14ad2,0xefbe4786384f25e3
1074	.quad	0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
1075	.quad	0x2de92c6f592b0275,0x4a7484aa6ea6e483
1076	.quad	0x5cb0a9dcbd41fbd4,0x76f988da831153b5
1077	.quad	0x983e5152ee66dfab,0xa831c66d2db43210
1078	.quad	0xb00327c898fb213f,0xbf597fc7beef0ee4
1079	.quad	0xc6e00bf33da88fc2,0xd5a79147930aa725
1080	.quad	0x06ca6351e003826f,0x142929670a0e6e70
1081	.quad	0x27b70a8546d22ffc,0x2e1b21385c26c926
1082	.quad	0x4d2c6dfc5ac42aed,0x53380d139d95b3df
1083	.quad	0x650a73548baf63de,0x766a0abb3c77b2a8
1084	.quad	0x81c2c92e47edaee6,0x92722c851482353b
1085	.quad	0xa2bfe8a14cf10364,0xa81a664bbc423001
1086	.quad	0xc24b8b70d0f89791,0xc76c51a30654be30
1087	.quad	0xd192e819d6ef5218,0xd69906245565a910
1088	.quad	0xf40e35855771202a,0x106aa07032bbd1b8
1089	.quad	0x19a4c116b8d2d0c8,0x1e376c085141ab53
1090	.quad	0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
1091	.quad	0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
1092	.quad	0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
1093	.quad	0x748f82ee5defb2fc,0x78a5636f43172f60
1094	.quad	0x84c87814a1f0ab72,0x8cc702081a6439ec
1095	.quad	0x90befffa23631e28,0xa4506cebde82bde9
1096	.quad	0xbef9a3f7b2c67915,0xc67178f2e372532b
1097	.quad	0xca273eceea26619c,0xd186b8c721c0c207
1098	.quad	0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
1099	.quad	0x06f067aa72176fba,0x0a637dc5a2c898a6
1100	.quad	0x113f9804bef90dae,0x1b710b35131c471b
1101	.quad	0x28db77f523047d84,0x32caab7b40c72493
1102	.quad	0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
1103	.quad	0x4cc5d4becb3e42b6,0x597f299cfc657e2a
1104	.quad	0x5fcb6fab3ad6faec,0x6c44198c4a475817
1105#ifndef __APPLE__
1106.text
1107.globl	Transform_Sha512_Len_crypto
1108.type	Transform_Sha512_Len_crypto,@function
1109.align	2
1110Transform_Sha512_Len_crypto:
1111#else
1112.section	__TEXT,__text
1113.globl	_Transform_Sha512_Len_crypto
1114.p2align	2
1115_Transform_Sha512_Len_crypto:
1116#endif /* __APPLE__ */
1117	stp	x29, x30, [sp, #-80]!
1118	add	x29, sp, #0
1119	stp	d8, d9, [x29, #16]
1120	stp	d10, d11, [x29, #32]
1121	stp	d12, d13, [x29, #48]
1122	stp	d14, d15, [x29, #64]
1123#ifndef __APPLE__
1124	adrp x4, L_SHA512_trans_crypto_len_k
1125	add  x4, x4, :lo12:L_SHA512_trans_crypto_len_k
1126#else
1127	adrp x4, L_SHA512_trans_crypto_len_k@PAGE
1128	add  x4, x4, L_SHA512_trans_crypto_len_k@PAGEOFF
1129#endif /* __APPLE__ */
1130#ifdef __APPLE__
1131.arch_extension sha3
1132#endif /* __APPLE__ */
1133	# Load K into vector registers
1134	ld1	{v8.2d, v9.2d, v10.2d, v11.2d}, [x4], #0x40
1135	ld1	{v12.2d, v13.2d, v14.2d, v15.2d}, [x4], #0x40
1136	# Load digest into working vars
1137	ld1	{v24.2d, v25.2d, v26.2d, v27.2d}, [x0]
1138	# Start of loop processing a block
1139L_sha512_len_crypto_begin:
1140	mov	x3, x4
1141	# Load W
1142	ld1	{v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40
1143	ld1	{v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40
1144	rev64	v0.16b, v0.16b
1145	rev64	v1.16b, v1.16b
1146	rev64	v2.16b, v2.16b
1147	rev64	v3.16b, v3.16b
1148	rev64	v4.16b, v4.16b
1149	rev64	v5.16b, v5.16b
1150	rev64	v6.16b, v6.16b
1151	rev64	v7.16b, v7.16b
1152	# Copy digest to add in at end
1153	mov	v28.16b, v24.16b
1154	mov	v29.16b, v25.16b
1155	mov	v30.16b, v26.16b
1156	mov	v31.16b, v27.16b
1157	# Start of 16 rounds
1158	# Round 0
1159	add	v20.2d, v0.2d, v8.2d
1160	ext	v20.16b, v20.16b, v20.16b, #8
1161	ext	v21.16b, v26.16b, v27.16b, #8
1162	ext	v22.16b, v25.16b, v26.16b, #8
1163	add	v27.2d, v27.2d, v20.2d
1164	sha512h	q27, q21, v22.2d
1165	add	v23.2d, v25.2d, v27.2d
1166	sha512h2	q27, q25, v24.2d
1167	# Round 1
1168	add	v20.2d, v1.2d, v9.2d
1169	ext	v20.16b, v20.16b, v20.16b, #8
1170	ext	v21.16b, v23.16b, v26.16b, #8
1171	ext	v22.16b, v24.16b, v23.16b, #8
1172	add	v26.2d, v26.2d, v20.2d
1173	sha512h	q26, q21, v22.2d
1174	add	v25.2d, v24.2d, v26.2d
1175	sha512h2	q26, q24, v27.2d
1176	# Round 2
1177	add	v20.2d, v2.2d, v10.2d
1178	ext	v20.16b, v20.16b, v20.16b, #8
1179	ext	v21.16b, v25.16b, v23.16b, #8
1180	ext	v22.16b, v27.16b, v25.16b, #8
1181	add	v23.2d, v23.2d, v20.2d
1182	sha512h	q23, q21, v22.2d
1183	add	v24.2d, v27.2d, v23.2d
1184	sha512h2	q23, q27, v26.2d
1185	# Round 3
1186	add	v20.2d, v3.2d, v11.2d
1187	ext	v20.16b, v20.16b, v20.16b, #8
1188	ext	v21.16b, v24.16b, v25.16b, #8
1189	ext	v22.16b, v26.16b, v24.16b, #8
1190	add	v25.2d, v25.2d, v20.2d
1191	sha512h	q25, q21, v22.2d
1192	add	v27.2d, v26.2d, v25.2d
1193	sha512h2	q25, q26, v23.2d
1194	# Round 4
1195	add	v20.2d, v4.2d, v12.2d
1196	ext	v20.16b, v20.16b, v20.16b, #8
1197	ext	v21.16b, v27.16b, v24.16b, #8
1198	ext	v22.16b, v23.16b, v27.16b, #8
1199	add	v24.2d, v24.2d, v20.2d
1200	sha512h	q24, q21, v22.2d
1201	add	v26.2d, v23.2d, v24.2d
1202	sha512h2	q24, q23, v25.2d
1203	# Round 5
1204	add	v20.2d, v5.2d, v13.2d
1205	ext	v20.16b, v20.16b, v20.16b, #8
1206	ext	v21.16b, v26.16b, v27.16b, #8
1207	ext	v22.16b, v25.16b, v26.16b, #8
1208	add	v27.2d, v27.2d, v20.2d
1209	sha512h	q27, q21, v22.2d
1210	add	v23.2d, v25.2d, v27.2d
1211	sha512h2	q27, q25, v24.2d
1212	# Round 6
1213	add	v20.2d, v6.2d, v14.2d
1214	ext	v20.16b, v20.16b, v20.16b, #8
1215	ext	v21.16b, v23.16b, v26.16b, #8
1216	ext	v22.16b, v24.16b, v23.16b, #8
1217	add	v26.2d, v26.2d, v20.2d
1218	sha512h	q26, q21, v22.2d
1219	add	v25.2d, v24.2d, v26.2d
1220	sha512h2	q26, q24, v27.2d
1221	# Round 7
1222	add	v20.2d, v7.2d, v15.2d
1223	ext	v20.16b, v20.16b, v20.16b, #8
1224	ext	v21.16b, v25.16b, v23.16b, #8
1225	ext	v22.16b, v27.16b, v25.16b, #8
1226	add	v23.2d, v23.2d, v20.2d
1227	sha512h	q23, q21, v22.2d
1228	add	v24.2d, v27.2d, v23.2d
1229	sha512h2	q23, q27, v26.2d
1230	# Load next 8 64-bit words of K
1231	ld1	{v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40
1232	# Round 8
1233	sha512su0	v0.2d, v1.2d
1234	ext	v21.16b, v4.16b, v5.16b, #8
1235	sha512su1	v0.2d, v7.2d, v21.2d
1236	add	v20.2d, v0.2d, v16.2d
1237	ext	v20.16b, v20.16b, v20.16b, #8
1238	ext	v21.16b, v24.16b, v25.16b, #8
1239	ext	v22.16b, v26.16b, v24.16b, #8
1240	add	v25.2d, v25.2d, v20.2d
1241	sha512h	q25, q21, v22.2d
1242	add	v27.2d, v26.2d, v25.2d
1243	sha512h2	q25, q26, v23.2d
1244	# Round 9
1245	sha512su0	v1.2d, v2.2d
1246	ext	v21.16b, v5.16b, v6.16b, #8
1247	sha512su1	v1.2d, v0.2d, v21.2d
1248	add	v20.2d, v1.2d, v17.2d
1249	ext	v20.16b, v20.16b, v20.16b, #8
1250	ext	v21.16b, v27.16b, v24.16b, #8
1251	ext	v22.16b, v23.16b, v27.16b, #8
1252	add	v24.2d, v24.2d, v20.2d
1253	sha512h	q24, q21, v22.2d
1254	add	v26.2d, v23.2d, v24.2d
1255	sha512h2	q24, q23, v25.2d
1256	# Round 10
1257	sha512su0	v2.2d, v3.2d
1258	ext	v21.16b, v6.16b, v7.16b, #8
1259	sha512su1	v2.2d, v1.2d, v21.2d
1260	add	v20.2d, v2.2d, v18.2d
1261	ext	v20.16b, v20.16b, v20.16b, #8
1262	ext	v21.16b, v26.16b, v27.16b, #8
1263	ext	v22.16b, v25.16b, v26.16b, #8
1264	add	v27.2d, v27.2d, v20.2d
1265	sha512h	q27, q21, v22.2d
1266	add	v23.2d, v25.2d, v27.2d
1267	sha512h2	q27, q25, v24.2d
1268	# Round 11
1269	sha512su0	v3.2d, v4.2d
1270	ext	v21.16b, v7.16b, v0.16b, #8
1271	sha512su1	v3.2d, v2.2d, v21.2d
1272	add	v20.2d, v3.2d, v19.2d
1273	ext	v20.16b, v20.16b, v20.16b, #8
1274	ext	v21.16b, v23.16b, v26.16b, #8
1275	ext	v22.16b, v24.16b, v23.16b, #8
1276	add	v26.2d, v26.2d, v20.2d
1277	sha512h	q26, q21, v22.2d
1278	add	v25.2d, v24.2d, v26.2d
1279	sha512h2	q26, q24, v27.2d
1280	# Load next 8 64-bit words of K
1281	ld1	{v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40
1282	# Round 12
1283	sha512su0	v4.2d, v5.2d
1284	ext	v21.16b, v0.16b, v1.16b, #8
1285	sha512su1	v4.2d, v3.2d, v21.2d
1286	add	v20.2d, v4.2d, v16.2d
1287	ext	v20.16b, v20.16b, v20.16b, #8
1288	ext	v21.16b, v25.16b, v23.16b, #8
1289	ext	v22.16b, v27.16b, v25.16b, #8
1290	add	v23.2d, v23.2d, v20.2d
1291	sha512h	q23, q21, v22.2d
1292	add	v24.2d, v27.2d, v23.2d
1293	sha512h2	q23, q27, v26.2d
1294	# Round 13
1295	sha512su0	v5.2d, v6.2d
1296	ext	v21.16b, v1.16b, v2.16b, #8
1297	sha512su1	v5.2d, v4.2d, v21.2d
1298	add	v20.2d, v5.2d, v17.2d
1299	ext	v20.16b, v20.16b, v20.16b, #8
1300	ext	v21.16b, v24.16b, v25.16b, #8
1301	ext	v22.16b, v26.16b, v24.16b, #8
1302	add	v25.2d, v25.2d, v20.2d
1303	sha512h	q25, q21, v22.2d
1304	add	v27.2d, v26.2d, v25.2d
1305	sha512h2	q25, q26, v23.2d
1306	# Round 14
1307	sha512su0	v6.2d, v7.2d
1308	ext	v21.16b, v2.16b, v3.16b, #8
1309	sha512su1	v6.2d, v5.2d, v21.2d
1310	add	v20.2d, v6.2d, v18.2d
1311	ext	v20.16b, v20.16b, v20.16b, #8
1312	ext	v21.16b, v27.16b, v24.16b, #8
1313	ext	v22.16b, v23.16b, v27.16b, #8
1314	add	v24.2d, v24.2d, v20.2d
1315	sha512h	q24, q21, v22.2d
1316	add	v26.2d, v23.2d, v24.2d
1317	sha512h2	q24, q23, v25.2d
1318	# Round 15
1319	sha512su0	v7.2d, v0.2d
1320	ext	v21.16b, v3.16b, v4.16b, #8
1321	sha512su1	v7.2d, v6.2d, v21.2d
1322	add	v20.2d, v7.2d, v19.2d
1323	ext	v20.16b, v20.16b, v20.16b, #8
1324	ext	v21.16b, v26.16b, v27.16b, #8
1325	ext	v22.16b, v25.16b, v26.16b, #8
1326	add	v27.2d, v27.2d, v20.2d
1327	sha512h	q27, q21, v22.2d
1328	add	v23.2d, v25.2d, v27.2d
1329	sha512h2	q27, q25, v24.2d
1330	# Load next 8 64-bit words of K
1331	ld1	{v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40
1332	# Round 16
1333	sha512su0	v0.2d, v1.2d
1334	ext	v21.16b, v4.16b, v5.16b, #8
1335	sha512su1	v0.2d, v7.2d, v21.2d
1336	add	v20.2d, v0.2d, v16.2d
1337	ext	v20.16b, v20.16b, v20.16b, #8
1338	ext	v21.16b, v23.16b, v26.16b, #8
1339	ext	v22.16b, v24.16b, v23.16b, #8
1340	add	v26.2d, v26.2d, v20.2d
1341	sha512h	q26, q21, v22.2d
1342	add	v25.2d, v24.2d, v26.2d
1343	sha512h2	q26, q24, v27.2d
1344	# Round 17
1345	sha512su0	v1.2d, v2.2d
1346	ext	v21.16b, v5.16b, v6.16b, #8
1347	sha512su1	v1.2d, v0.2d, v21.2d
1348	add	v20.2d, v1.2d, v17.2d
1349	ext	v20.16b, v20.16b, v20.16b, #8
1350	ext	v21.16b, v25.16b, v23.16b, #8
1351	ext	v22.16b, v27.16b, v25.16b, #8
1352	add	v23.2d, v23.2d, v20.2d
1353	sha512h	q23, q21, v22.2d
1354	add	v24.2d, v27.2d, v23.2d
1355	sha512h2	q23, q27, v26.2d
1356	# Round 18
1357	sha512su0	v2.2d, v3.2d
1358	ext	v21.16b, v6.16b, v7.16b, #8
1359	sha512su1	v2.2d, v1.2d, v21.2d
1360	add	v20.2d, v2.2d, v18.2d
1361	ext	v20.16b, v20.16b, v20.16b, #8
1362	ext	v21.16b, v24.16b, v25.16b, #8
1363	ext	v22.16b, v26.16b, v24.16b, #8
1364	add	v25.2d, v25.2d, v20.2d
1365	sha512h	q25, q21, v22.2d
1366	add	v27.2d, v26.2d, v25.2d
1367	sha512h2	q25, q26, v23.2d
1368	# Round 19
1369	sha512su0	v3.2d, v4.2d
1370	ext	v21.16b, v7.16b, v0.16b, #8
1371	sha512su1	v3.2d, v2.2d, v21.2d
1372	add	v20.2d, v3.2d, v19.2d
1373	ext	v20.16b, v20.16b, v20.16b, #8
1374	ext	v21.16b, v27.16b, v24.16b, #8
1375	ext	v22.16b, v23.16b, v27.16b, #8
1376	add	v24.2d, v24.2d, v20.2d
1377	sha512h	q24, q21, v22.2d
1378	add	v26.2d, v23.2d, v24.2d
1379	sha512h2	q24, q23, v25.2d
1380	# Load next 8 64-bit words of K
1381	ld1	{v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40
1382	# Round 20
1383	sha512su0	v4.2d, v5.2d
1384	ext	v21.16b, v0.16b, v1.16b, #8
1385	sha512su1	v4.2d, v3.2d, v21.2d
1386	add	v20.2d, v4.2d, v16.2d
1387	ext	v20.16b, v20.16b, v20.16b, #8
1388	ext	v21.16b, v26.16b, v27.16b, #8
1389	ext	v22.16b, v25.16b, v26.16b, #8
1390	add	v27.2d, v27.2d, v20.2d
1391	sha512h	q27, q21, v22.2d
1392	add	v23.2d, v25.2d, v27.2d
1393	sha512h2	q27, q25, v24.2d
1394	# Round 21
1395	sha512su0	v5.2d, v6.2d
1396	ext	v21.16b, v1.16b, v2.16b, #8
1397	sha512su1	v5.2d, v4.2d, v21.2d
1398	add	v20.2d, v5.2d, v17.2d
1399	ext	v20.16b, v20.16b, v20.16b, #8
1400	ext	v21.16b, v23.16b, v26.16b, #8
1401	ext	v22.16b, v24.16b, v23.16b, #8
1402	add	v26.2d, v26.2d, v20.2d
1403	sha512h	q26, q21, v22.2d
1404	add	v25.2d, v24.2d, v26.2d
1405	sha512h2	q26, q24, v27.2d
1406	# Round 22
1407	sha512su0	v6.2d, v7.2d
1408	ext	v21.16b, v2.16b, v3.16b, #8
1409	sha512su1	v6.2d, v5.2d, v21.2d
1410	add	v20.2d, v6.2d, v18.2d
1411	ext	v20.16b, v20.16b, v20.16b, #8
1412	ext	v21.16b, v25.16b, v23.16b, #8
1413	ext	v22.16b, v27.16b, v25.16b, #8
1414	add	v23.2d, v23.2d, v20.2d
1415	sha512h	q23, q21, v22.2d
1416	add	v24.2d, v27.2d, v23.2d
1417	sha512h2	q23, q27, v26.2d
1418	# Round 23
1419	sha512su0	v7.2d, v0.2d
1420	ext	v21.16b, v3.16b, v4.16b, #8
1421	sha512su1	v7.2d, v6.2d, v21.2d
1422	add	v20.2d, v7.2d, v19.2d
1423	ext	v20.16b, v20.16b, v20.16b, #8
1424	ext	v21.16b, v24.16b, v25.16b, #8
1425	ext	v22.16b, v26.16b, v24.16b, #8
1426	add	v25.2d, v25.2d, v20.2d
1427	sha512h	q25, q21, v22.2d
1428	add	v27.2d, v26.2d, v25.2d
1429	sha512h2	q25, q26, v23.2d
1430	# Load next 8 64-bit words of K
1431	ld1	{v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40
1432	# Round 24
1433	sha512su0	v0.2d, v1.2d
1434	ext	v21.16b, v4.16b, v5.16b, #8
1435	sha512su1	v0.2d, v7.2d, v21.2d
1436	add	v20.2d, v0.2d, v16.2d
1437	ext	v20.16b, v20.16b, v20.16b, #8
1438	ext	v21.16b, v27.16b, v24.16b, #8
1439	ext	v22.16b, v23.16b, v27.16b, #8
1440	add	v24.2d, v24.2d, v20.2d
1441	sha512h	q24, q21, v22.2d
1442	add	v26.2d, v23.2d, v24.2d
1443	sha512h2	q24, q23, v25.2d
1444	# Round 25
1445	sha512su0	v1.2d, v2.2d
1446	ext	v21.16b, v5.16b, v6.16b, #8
1447	sha512su1	v1.2d, v0.2d, v21.2d
1448	add	v20.2d, v1.2d, v17.2d
1449	ext	v20.16b, v20.16b, v20.16b, #8
1450	ext	v21.16b, v26.16b, v27.16b, #8
1451	ext	v22.16b, v25.16b, v26.16b, #8
1452	add	v27.2d, v27.2d, v20.2d
1453	sha512h	q27, q21, v22.2d
1454	add	v23.2d, v25.2d, v27.2d
1455	sha512h2	q27, q25, v24.2d
1456	# Round 26
1457	sha512su0	v2.2d, v3.2d
1458	ext	v21.16b, v6.16b, v7.16b, #8
1459	sha512su1	v2.2d, v1.2d, v21.2d
1460	add	v20.2d, v2.2d, v18.2d
1461	ext	v20.16b, v20.16b, v20.16b, #8
1462	ext	v21.16b, v23.16b, v26.16b, #8
1463	ext	v22.16b, v24.16b, v23.16b, #8
1464	add	v26.2d, v26.2d, v20.2d
1465	sha512h	q26, q21, v22.2d
1466	add	v25.2d, v24.2d, v26.2d
1467	sha512h2	q26, q24, v27.2d
1468	# Round 27
1469	sha512su0	v3.2d, v4.2d
1470	ext	v21.16b, v7.16b, v0.16b, #8
1471	sha512su1	v3.2d, v2.2d, v21.2d
1472	add	v20.2d, v3.2d, v19.2d
1473	ext	v20.16b, v20.16b, v20.16b, #8
1474	ext	v21.16b, v25.16b, v23.16b, #8
1475	ext	v22.16b, v27.16b, v25.16b, #8
1476	add	v23.2d, v23.2d, v20.2d
1477	sha512h	q23, q21, v22.2d
1478	add	v24.2d, v27.2d, v23.2d
1479	sha512h2	q23, q27, v26.2d
1480	# Load next 8 64-bit words of K
1481	ld1	{v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40
1482	# Round 28
1483	sha512su0	v4.2d, v5.2d
1484	ext	v21.16b, v0.16b, v1.16b, #8
1485	sha512su1	v4.2d, v3.2d, v21.2d
1486	add	v20.2d, v4.2d, v16.2d
1487	ext	v20.16b, v20.16b, v20.16b, #8
1488	ext	v21.16b, v24.16b, v25.16b, #8
1489	ext	v22.16b, v26.16b, v24.16b, #8
1490	add	v25.2d, v25.2d, v20.2d
1491	sha512h	q25, q21, v22.2d
1492	add	v27.2d, v26.2d, v25.2d
1493	sha512h2	q25, q26, v23.2d
1494	# Round 29
1495	sha512su0	v5.2d, v6.2d
1496	ext	v21.16b, v1.16b, v2.16b, #8
1497	sha512su1	v5.2d, v4.2d, v21.2d
1498	add	v20.2d, v5.2d, v17.2d
1499	ext	v20.16b, v20.16b, v20.16b, #8
1500	ext	v21.16b, v27.16b, v24.16b, #8
1501	ext	v22.16b, v23.16b, v27.16b, #8
1502	add	v24.2d, v24.2d, v20.2d
1503	sha512h	q24, q21, v22.2d
1504	add	v26.2d, v23.2d, v24.2d
1505	sha512h2	q24, q23, v25.2d
1506	# Round 30
1507	sha512su0	v6.2d, v7.2d
1508	ext	v21.16b, v2.16b, v3.16b, #8
1509	sha512su1	v6.2d, v5.2d, v21.2d
1510	add	v20.2d, v6.2d, v18.2d
1511	ext	v20.16b, v20.16b, v20.16b, #8
1512	ext	v21.16b, v26.16b, v27.16b, #8
1513	ext	v22.16b, v25.16b, v26.16b, #8
1514	add	v27.2d, v27.2d, v20.2d
1515	sha512h	q27, q21, v22.2d
1516	add	v23.2d, v25.2d, v27.2d
1517	sha512h2	q27, q25, v24.2d
1518	# Round 31
1519	sha512su0	v7.2d, v0.2d
1520	ext	v21.16b, v3.16b, v4.16b, #8
1521	sha512su1	v7.2d, v6.2d, v21.2d
1522	add	v20.2d, v7.2d, v19.2d
1523	ext	v20.16b, v20.16b, v20.16b, #8
1524	ext	v21.16b, v23.16b, v26.16b, #8
1525	ext	v22.16b, v24.16b, v23.16b, #8
1526	add	v26.2d, v26.2d, v20.2d
1527	sha512h	q26, q21, v22.2d
1528	add	v25.2d, v24.2d, v26.2d
1529	sha512h2	q26, q24, v27.2d
1530	# Load next 8 64-bit words of K
1531	ld1	{v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40
1532	# Round 32
1533	sha512su0	v0.2d, v1.2d
1534	ext	v21.16b, v4.16b, v5.16b, #8
1535	sha512su1	v0.2d, v7.2d, v21.2d
1536	add	v20.2d, v0.2d, v16.2d
1537	ext	v20.16b, v20.16b, v20.16b, #8
1538	ext	v21.16b, v25.16b, v23.16b, #8
1539	ext	v22.16b, v27.16b, v25.16b, #8
1540	add	v23.2d, v23.2d, v20.2d
1541	sha512h	q23, q21, v22.2d
1542	add	v24.2d, v27.2d, v23.2d
1543	sha512h2	q23, q27, v26.2d
1544	# Round 33
1545	sha512su0	v1.2d, v2.2d
1546	ext	v21.16b, v5.16b, v6.16b, #8
1547	sha512su1	v1.2d, v0.2d, v21.2d
1548	add	v20.2d, v1.2d, v17.2d
1549	ext	v20.16b, v20.16b, v20.16b, #8
1550	ext	v21.16b, v24.16b, v25.16b, #8
1551	ext	v22.16b, v26.16b, v24.16b, #8
1552	add	v25.2d, v25.2d, v20.2d
1553	sha512h	q25, q21, v22.2d
1554	add	v27.2d, v26.2d, v25.2d
1555	sha512h2	q25, q26, v23.2d
1556	# Round 34
1557	sha512su0	v2.2d, v3.2d
1558	ext	v21.16b, v6.16b, v7.16b, #8
1559	sha512su1	v2.2d, v1.2d, v21.2d
1560	add	v20.2d, v2.2d, v18.2d
1561	ext	v20.16b, v20.16b, v20.16b, #8
1562	ext	v21.16b, v27.16b, v24.16b, #8
1563	ext	v22.16b, v23.16b, v27.16b, #8
1564	add	v24.2d, v24.2d, v20.2d
1565	sha512h	q24, q21, v22.2d
1566	add	v26.2d, v23.2d, v24.2d
1567	sha512h2	q24, q23, v25.2d
1568	# Round 35
1569	sha512su0	v3.2d, v4.2d
1570	ext	v21.16b, v7.16b, v0.16b, #8
1571	sha512su1	v3.2d, v2.2d, v21.2d
1572	add	v20.2d, v3.2d, v19.2d
1573	ext	v20.16b, v20.16b, v20.16b, #8
1574	ext	v21.16b, v26.16b, v27.16b, #8
1575	ext	v22.16b, v25.16b, v26.16b, #8
1576	add	v27.2d, v27.2d, v20.2d
1577	sha512h	q27, q21, v22.2d
1578	add	v23.2d, v25.2d, v27.2d
1579	sha512h2	q27, q25, v24.2d
1580	# Load next 8 64-bit words of K
1581	ld1	{v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40
1582	# Round 36
1583	sha512su0	v4.2d, v5.2d
1584	ext	v21.16b, v0.16b, v1.16b, #8
1585	sha512su1	v4.2d, v3.2d, v21.2d
1586	add	v20.2d, v4.2d, v16.2d
1587	ext	v20.16b, v20.16b, v20.16b, #8
1588	ext	v21.16b, v23.16b, v26.16b, #8
1589	ext	v22.16b, v24.16b, v23.16b, #8
1590	add	v26.2d, v26.2d, v20.2d
1591	sha512h	q26, q21, v22.2d
1592	add	v25.2d, v24.2d, v26.2d
1593	sha512h2	q26, q24, v27.2d
1594	# Round 37
1595	sha512su0	v5.2d, v6.2d
1596	ext	v21.16b, v1.16b, v2.16b, #8
1597	sha512su1	v5.2d, v4.2d, v21.2d
1598	add	v20.2d, v5.2d, v17.2d
1599	ext	v20.16b, v20.16b, v20.16b, #8
1600	ext	v21.16b, v25.16b, v23.16b, #8
1601	ext	v22.16b, v27.16b, v25.16b, #8
1602	add	v23.2d, v23.2d, v20.2d
1603	sha512h	q23, q21, v22.2d
1604	add	v24.2d, v27.2d, v23.2d
1605	sha512h2	q23, q27, v26.2d
1606	# Round 38
1607	sha512su0	v6.2d, v7.2d
1608	ext	v21.16b, v2.16b, v3.16b, #8
1609	sha512su1	v6.2d, v5.2d, v21.2d
1610	add	v20.2d, v6.2d, v18.2d
1611	ext	v20.16b, v20.16b, v20.16b, #8
1612	ext	v21.16b, v24.16b, v25.16b, #8
1613	ext	v22.16b, v26.16b, v24.16b, #8
1614	add	v25.2d, v25.2d, v20.2d
1615	sha512h	q25, q21, v22.2d
1616	add	v27.2d, v26.2d, v25.2d
1617	sha512h2	q25, q26, v23.2d
1618	# Round 39
1619	sha512su0	v7.2d, v0.2d
1620	ext	v21.16b, v3.16b, v4.16b, #8
1621	sha512su1	v7.2d, v6.2d, v21.2d
1622	add	v20.2d, v7.2d, v19.2d
1623	ext	v20.16b, v20.16b, v20.16b, #8
1624	ext	v21.16b, v27.16b, v24.16b, #8
1625	ext	v22.16b, v23.16b, v27.16b, #8
1626	add	v24.2d, v24.2d, v20.2d
1627	sha512h	q24, q21, v22.2d
1628	add	v26.2d, v23.2d, v24.2d
1629	sha512h2	q24, q23, v25.2d
1630	add	v27.2d, v27.2d, v31.2d
1631	add	v26.2d, v26.2d, v30.2d
1632	add	v25.2d, v25.2d, v29.2d
1633	add	v24.2d, v24.2d, v28.2d
1634	subs	w2, w2, #0x80
1635	bne	L_sha512_len_crypto_begin
1636	# Store digest back
1637	st1	{v24.2d, v25.2d, v26.2d, v27.2d}, [x0]
1638	ldp	d8, d9, [x29, #16]
1639	ldp	d10, d11, [x29, #32]
1640	ldp	d12, d13, [x29, #48]
1641	ldp	d14, d15, [x29, #64]
1642	ldp	x29, x30, [sp], #0x50
1643	ret
1644#ifndef __APPLE__
1645	.size	Transform_Sha512_Len_crypto,.-Transform_Sha512_Len_crypto
1646#endif /* __APPLE__ */
1647#endif /* WOLFSSL_ARMASM_CRYPTO_SHA512 */
1648#endif /* WOLFSSL_SHA512 || WOLFSSL_SHA384 */
1649#endif /* __aarch64__ */
1650#endif /* WOLFSSL_ARMASM */
1651
1652#if defined(__linux__) && defined(__ELF__)
1653.section	.note.GNU-stack,"",%progbits
1654#endif
1655#endif /* !WOLFSSL_ARMASM_INLINE */