luna - wolfssl/wolfcrypt/src/port/arm/armv8-chacha-asm.S

   1/* armv8-chacha-asm
   2 *
   3 * Copyright (C) 2006-2026 wolfSSL Inc.
   4 *
   5 * This file is part of wolfSSL.
   6 *
   7 * wolfSSL is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License as published by
   9 * the Free Software Foundation; either version 3 of the License, or
  10 * (at your option) any later version.
  11 *
  12 * wolfSSL is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 * GNU General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU General Public License
  18 * along with this program; if not, write to the Free Software
  19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
  20 */
  21
  22#include <wolfssl/wolfcrypt/libwolfssl_sources_asm.h>
  23
  24/* Generated using (from wolfssl):
  25 *   cd ../scripts
  26 *   ruby ./chacha/chacha.rb arm64 \
  27 *       ../wolfssl/wolfcrypt/src/port/arm/armv8-chacha-asm.S
  28 */
  29#ifdef WOLFSSL_ARMASM
  30#ifdef __aarch64__
  31#ifndef WOLFSSL_ARMASM_INLINE
  32#ifdef HAVE_CHACHA
  33#ifndef __APPLE__
  34	.text
  35	.section	.rodata
  36	.type	L_chacha20_arm64_ctr, %object
  37	.size	L_chacha20_arm64_ctr, 16
  38#else
  39	.section	__DATA,__data
  40#endif /* __APPLE__ */
  41	# 8-byte aligned, 64-bit aligned
  42#ifndef __APPLE__
  43	.align	3
  44#else
  45	.p2align	3
  46#endif /* __APPLE__ */
  47L_chacha20_arm64_ctr:
  48	.long	0x00000000,0x00000001,0x00000002,0x00000003
  49#ifndef __APPLE__
  50	.text
  51	.section	.rodata
  52	.type	L_chacha20_arm64_rol8, %object
  53	.size	L_chacha20_arm64_rol8, 16
  54#else
  55	.section	__DATA,__data
  56#endif /* __APPLE__ */
  57	# 8-byte aligned, 64-bit aligned
  58#ifndef __APPLE__
  59	.align	3
  60#else
  61	.p2align	3
  62#endif /* __APPLE__ */
  63L_chacha20_arm64_rol8:
  64	.long	0x02010003,0x06050407,0x0a09080b,0x0e0d0c0f
  65#ifndef WOLFSSL_ARMASM_NO_NEON
  66#ifndef __APPLE__
  67.text
  68.globl	wc_chacha_crypt_bytes
  69.type	wc_chacha_crypt_bytes,@function
  70.align	2
  71wc_chacha_crypt_bytes:
  72#else
  73.section	__TEXT,__text
  74.globl	_wc_chacha_crypt_bytes
  75.p2align	2
  76_wc_chacha_crypt_bytes:
  77#endif /* __APPLE__ */
  78	stp	x29, x30, [sp, #-160]!
  79	add	x29, sp, #0
  80	stp	x17, x19, [x29, #24]
  81	stp	x20, x21, [x29, #40]
  82	stp	x22, x23, [x29, #56]
  83	stp	x24, x25, [x29, #72]
  84	str	x26, [x29, #88]
  85	stp	d8, d9, [x29, #96]
  86	stp	d10, d11, [x29, #112]
  87	stp	d12, d13, [x29, #128]
  88	stp	d14, d15, [x29, #144]
  89#ifndef __APPLE__
  90	adrp x5, L_chacha20_arm64_rol8
  91	add  x5, x5, :lo12:L_chacha20_arm64_rol8
  92#else
  93	adrp x5, L_chacha20_arm64_rol8@PAGE
  94	add  x5, x5, L_chacha20_arm64_rol8@PAGEOFF
  95#endif /* __APPLE__ */
  96#ifndef __APPLE__
  97	adrp x6, L_chacha20_arm64_ctr
  98	add  x6, x6, :lo12:L_chacha20_arm64_ctr
  99#else
 100	adrp x6, L_chacha20_arm64_ctr@PAGE
 101	add  x6, x6, L_chacha20_arm64_ctr@PAGEOFF
 102#endif /* __APPLE__ */
 103	eor	v29.16b, v29.16b, v29.16b
 104	mov	x26, #5
 105	eor	v31.16b, v31.16b, v31.16b
 106	mov	w7, #1
 107	ld1	{v30.16b}, [x5]
 108	ld1	{v28.4s}, [x6]
 109	add	x4, x0, #0x44
 110	mov	v29.s[0], w26
 111	mov	v31.s[0], w7
 112	# Load state to encrypt
 113	ld1	{v16.4s, v17.4s, v18.4s, v19.4s}, [x0]
 114	cmp	x3, #0x140
 115	blt	L_chacha_crypt_bytes_arm64_lt_320
 116	mov	w25, #4
 117L_chacha_crypt_bytes_arm64_loop_320:
 118	# Move state into regular register
 119	mov	x8, v16.d[0]
 120	mov	x10, v16.d[1]
 121	mov	x12, v17.d[0]
 122	mov	x14, v17.d[1]
 123	mov	x16, v18.d[0]
 124	mov	x19, v18.d[1]
 125	mov	x21, v19.d[0]
 126	mov	x23, v19.d[1]
 127	sub	x3, x3, #0x140
 128	# Move state into vector registers
 129	dup	v0.4s, v16.s[0]
 130	dup	v1.4s, v16.s[1]
 131	lsr	x9, x8, #32
 132	dup	v2.4s, v16.s[2]
 133	dup	v3.4s, v16.s[3]
 134	lsr	x11, x10, #32
 135	dup	v4.4s, v17.s[0]
 136	dup	v5.4s, v17.s[1]
 137	lsr	x13, x12, #32
 138	dup	v6.4s, v17.s[2]
 139	dup	v7.4s, v17.s[3]
 140	lsr	x15, x14, #32
 141	dup	v8.4s, v18.s[0]
 142	dup	v9.4s, v18.s[1]
 143	lsr	x17, x16, #32
 144	dup	v10.4s, v18.s[2]
 145	dup	v11.4s, v18.s[3]
 146	lsr	x20, x19, #32
 147	dup	v12.4s, v19.s[0]
 148	dup	v13.4s, v19.s[1]
 149	lsr	x22, x21, #32
 150	dup	v14.4s, v19.s[2]
 151	dup	v15.4s, v19.s[3]
 152	lsr	x24, x23, #32
 153	# Add to counter word
 154	add	v12.4s, v12.4s, v28.4s
 155	add	w21, w21, w25
 156	# Set number of odd+even rounds to perform
 157	mov	x26, #10
 158L_chacha_crypt_bytes_arm64_round_start_320:
 159	subs	x26, x26, #1
 160	# Round odd
 161	# a += b; d ^= a; d <<<= 16;
 162	add	v0.4s, v0.4s, v4.4s
 163	add	w8, w8, w12
 164	add	v1.4s, v1.4s, v5.4s
 165	add	w9, w9, w13
 166	add	v2.4s, v2.4s, v6.4s
 167	add	w10, w10, w14
 168	add	v3.4s, v3.4s, v7.4s
 169	add	w11, w11, w15
 170	eor	v12.16b, v12.16b, v0.16b
 171	eor	w21, w21, w8
 172	eor	v13.16b, v13.16b, v1.16b
 173	eor	w22, w22, w9
 174	eor	v14.16b, v14.16b, v2.16b
 175	eor	w23, w23, w10
 176	eor	v15.16b, v15.16b, v3.16b
 177	eor	w24, w24, w11
 178	rev32	v12.8h, v12.8h
 179	ror	w21, w21, #16
 180	rev32	v13.8h, v13.8h
 181	ror	w22, w22, #16
 182	rev32	v14.8h, v14.8h
 183	ror	w23, w23, #16
 184	rev32	v15.8h, v15.8h
 185	ror	w24, w24, #16
 186	# c += d; b ^= c; b <<<= 12;
 187	add	v8.4s, v8.4s, v12.4s
 188	add	w16, w16, w21
 189	add	v9.4s, v9.4s, v13.4s
 190	add	w17, w17, w22
 191	add	v10.4s, v10.4s, v14.4s
 192	add	w19, w19, w23
 193	add	v11.4s, v11.4s, v15.4s
 194	add	w20, w20, w24
 195	eor	v20.16b, v4.16b, v8.16b
 196	eor	w12, w12, w16
 197	eor	v21.16b, v5.16b, v9.16b
 198	eor	w13, w13, w17
 199	eor	v22.16b, v6.16b, v10.16b
 200	eor	w14, w14, w19
 201	eor	v23.16b, v7.16b, v11.16b
 202	eor	w15, w15, w20
 203	shl	v4.4s, v20.4s, #12
 204	ror	w12, w12, #20
 205	shl	v5.4s, v21.4s, #12
 206	ror	w13, w13, #20
 207	shl	v6.4s, v22.4s, #12
 208	ror	w14, w14, #20
 209	shl	v7.4s, v23.4s, #12
 210	ror	w15, w15, #20
 211	sri	v4.4s, v20.4s, #20
 212	sri	v5.4s, v21.4s, #20
 213	sri	v6.4s, v22.4s, #20
 214	sri	v7.4s, v23.4s, #20
 215	# a += b; d ^= a; d <<<= 8;
 216	add	v0.4s, v0.4s, v4.4s
 217	add	w8, w8, w12
 218	add	v1.4s, v1.4s, v5.4s
 219	add	w9, w9, w13
 220	add	v2.4s, v2.4s, v6.4s
 221	add	w10, w10, w14
 222	add	v3.4s, v3.4s, v7.4s
 223	add	w11, w11, w15
 224	eor	v12.16b, v12.16b, v0.16b
 225	eor	w21, w21, w8
 226	eor	v13.16b, v13.16b, v1.16b
 227	eor	w22, w22, w9
 228	eor	v14.16b, v14.16b, v2.16b
 229	eor	w23, w23, w10
 230	eor	v15.16b, v15.16b, v3.16b
 231	eor	w24, w24, w11
 232	tbl	v12.16b, {v12.16b}, v30.16b
 233	ror	w21, w21, #24
 234	tbl	v13.16b, {v13.16b}, v30.16b
 235	ror	w22, w22, #24
 236	tbl	v14.16b, {v14.16b}, v30.16b
 237	ror	w23, w23, #24
 238	tbl	v15.16b, {v15.16b}, v30.16b
 239	ror	w24, w24, #24
 240	# c += d; b ^= c; b <<<= 7;
 241	add	v8.4s, v8.4s, v12.4s
 242	add	w16, w16, w21
 243	add	v9.4s, v9.4s, v13.4s
 244	add	w17, w17, w22
 245	add	v10.4s, v10.4s, v14.4s
 246	add	w19, w19, w23
 247	add	v11.4s, v11.4s, v15.4s
 248	add	w20, w20, w24
 249	eor	v20.16b, v4.16b, v8.16b
 250	eor	w12, w12, w16
 251	eor	v21.16b, v5.16b, v9.16b
 252	eor	w13, w13, w17
 253	eor	v22.16b, v6.16b, v10.16b
 254	eor	w14, w14, w19
 255	eor	v23.16b, v7.16b, v11.16b
 256	eor	w15, w15, w20
 257	shl	v4.4s, v20.4s, #7
 258	ror	w12, w12, #25
 259	shl	v5.4s, v21.4s, #7
 260	ror	w13, w13, #25
 261	shl	v6.4s, v22.4s, #7
 262	ror	w14, w14, #25
 263	shl	v7.4s, v23.4s, #7
 264	ror	w15, w15, #25
 265	sri	v4.4s, v20.4s, #25
 266	sri	v5.4s, v21.4s, #25
 267	sri	v6.4s, v22.4s, #25
 268	sri	v7.4s, v23.4s, #25
 269	# Round even
 270	# a += b; d ^= a; d <<<= 16;
 271	add	v0.4s, v0.4s, v5.4s
 272	add	w8, w8, w13
 273	add	v1.4s, v1.4s, v6.4s
 274	add	w9, w9, w14
 275	add	v2.4s, v2.4s, v7.4s
 276	add	w10, w10, w15
 277	add	v3.4s, v3.4s, v4.4s
 278	add	w11, w11, w12
 279	eor	v15.16b, v15.16b, v0.16b
 280	eor	w24, w24, w8
 281	eor	v12.16b, v12.16b, v1.16b
 282	eor	w21, w21, w9
 283	eor	v13.16b, v13.16b, v2.16b
 284	eor	w22, w22, w10
 285	eor	v14.16b, v14.16b, v3.16b
 286	eor	w23, w23, w11
 287	rev32	v15.8h, v15.8h
 288	ror	w24, w24, #16
 289	rev32	v12.8h, v12.8h
 290	ror	w21, w21, #16
 291	rev32	v13.8h, v13.8h
 292	ror	w22, w22, #16
 293	rev32	v14.8h, v14.8h
 294	ror	w23, w23, #16
 295	# c += d; b ^= c; b <<<= 12;
 296	add	v10.4s, v10.4s, v15.4s
 297	add	w19, w19, w24
 298	add	v11.4s, v11.4s, v12.4s
 299	add	w20, w20, w21
 300	add	v8.4s, v8.4s, v13.4s
 301	add	w16, w16, w22
 302	add	v9.4s, v9.4s, v14.4s
 303	add	w17, w17, w23
 304	eor	v20.16b, v5.16b, v10.16b
 305	eor	w13, w13, w19
 306	eor	v21.16b, v6.16b, v11.16b
 307	eor	w14, w14, w20
 308	eor	v22.16b, v7.16b, v8.16b
 309	eor	w15, w15, w16
 310	eor	v23.16b, v4.16b, v9.16b
 311	eor	w12, w12, w17
 312	shl	v5.4s, v20.4s, #12
 313	ror	w13, w13, #20
 314	shl	v6.4s, v21.4s, #12
 315	ror	w14, w14, #20
 316	shl	v7.4s, v22.4s, #12
 317	ror	w15, w15, #20
 318	shl	v4.4s, v23.4s, #12
 319	ror	w12, w12, #20
 320	sri	v5.4s, v20.4s, #20
 321	sri	v6.4s, v21.4s, #20
 322	sri	v7.4s, v22.4s, #20
 323	sri	v4.4s, v23.4s, #20
 324	# a += b; d ^= a; d <<<= 8;
 325	add	v0.4s, v0.4s, v5.4s
 326	add	w8, w8, w13
 327	add	v1.4s, v1.4s, v6.4s
 328	add	w9, w9, w14
 329	add	v2.4s, v2.4s, v7.4s
 330	add	w10, w10, w15
 331	add	v3.4s, v3.4s, v4.4s
 332	add	w11, w11, w12
 333	eor	v15.16b, v15.16b, v0.16b
 334	eor	w24, w24, w8
 335	eor	v12.16b, v12.16b, v1.16b
 336	eor	w21, w21, w9
 337	eor	v13.16b, v13.16b, v2.16b
 338	eor	w22, w22, w10
 339	eor	v14.16b, v14.16b, v3.16b
 340	eor	w23, w23, w11
 341	tbl	v15.16b, {v15.16b}, v30.16b
 342	ror	w24, w24, #24
 343	tbl	v12.16b, {v12.16b}, v30.16b
 344	ror	w21, w21, #24
 345	tbl	v13.16b, {v13.16b}, v30.16b
 346	ror	w22, w22, #24
 347	tbl	v14.16b, {v14.16b}, v30.16b
 348	ror	w23, w23, #24
 349	# c += d; b ^= c; b <<<= 7;
 350	add	v10.4s, v10.4s, v15.4s
 351	add	w19, w19, w24
 352	add	v11.4s, v11.4s, v12.4s
 353	add	w20, w20, w21
 354	add	v8.4s, v8.4s, v13.4s
 355	add	w16, w16, w22
 356	add	v9.4s, v9.4s, v14.4s
 357	add	w17, w17, w23
 358	eor	v20.16b, v5.16b, v10.16b
 359	eor	w13, w13, w19
 360	eor	v21.16b, v6.16b, v11.16b
 361	eor	w14, w14, w20
 362	eor	v22.16b, v7.16b, v8.16b
 363	eor	w15, w15, w16
 364	eor	v23.16b, v4.16b, v9.16b
 365	eor	w12, w12, w17
 366	shl	v5.4s, v20.4s, #7
 367	ror	w13, w13, #25
 368	shl	v6.4s, v21.4s, #7
 369	ror	w14, w14, #25
 370	shl	v7.4s, v22.4s, #7
 371	ror	w15, w15, #25
 372	shl	v4.4s, v23.4s, #7
 373	ror	w12, w12, #25
 374	sri	v5.4s, v20.4s, #25
 375	sri	v6.4s, v21.4s, #25
 376	sri	v7.4s, v22.4s, #25
 377	sri	v4.4s, v23.4s, #25
 378	bne	L_chacha_crypt_bytes_arm64_round_start_320
 379	# Add counter now rather than after transposed
 380	add	v12.4s, v12.4s, v28.4s
 381	add	w21, w21, w25
 382	# Load message
 383	ld1	{v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40
 384	# Transpose vectors
 385	trn1	v20.4s, v0.4s, v1.4s
 386	trn1	v22.4s, v2.4s, v3.4s
 387	orr	x8, x8, x9, lsl 32
 388	trn2	v21.4s, v0.4s, v1.4s
 389	trn2	v23.4s, v2.4s, v3.4s
 390	trn1	v0.2d, v20.2d, v22.2d
 391	trn1	v1.2d, v21.2d, v23.2d
 392	orr	x10, x10, x11, lsl 32
 393	trn2	v2.2d, v20.2d, v22.2d
 394	trn2	v3.2d, v21.2d, v23.2d
 395	trn1	v20.4s, v4.4s, v5.4s
 396	trn1	v22.4s, v6.4s, v7.4s
 397	orr	x12, x12, x13, lsl 32
 398	trn2	v21.4s, v4.4s, v5.4s
 399	trn2	v23.4s, v6.4s, v7.4s
 400	trn1	v4.2d, v20.2d, v22.2d
 401	trn1	v5.2d, v21.2d, v23.2d
 402	orr	x14, x14, x15, lsl 32
 403	trn2	v6.2d, v20.2d, v22.2d
 404	trn2	v7.2d, v21.2d, v23.2d
 405	trn1	v20.4s, v8.4s, v9.4s
 406	trn1	v22.4s, v10.4s, v11.4s
 407	orr	x16, x16, x17, lsl 32
 408	trn2	v21.4s, v8.4s, v9.4s
 409	trn2	v23.4s, v10.4s, v11.4s
 410	trn1	v8.2d, v20.2d, v22.2d
 411	trn1	v9.2d, v21.2d, v23.2d
 412	orr	x19, x19, x20, lsl 32
 413	trn2	v10.2d, v20.2d, v22.2d
 414	trn2	v11.2d, v21.2d, v23.2d
 415	trn1	v20.4s, v12.4s, v13.4s
 416	trn1	v22.4s, v14.4s, v15.4s
 417	orr	x21, x21, x22, lsl 32
 418	trn2	v21.4s, v12.4s, v13.4s
 419	trn2	v23.4s, v14.4s, v15.4s
 420	trn1	v12.2d, v20.2d, v22.2d
 421	trn1	v13.2d, v21.2d, v23.2d
 422	orr	x23, x23, x24, lsl 32
 423	trn2	v14.2d, v20.2d, v22.2d
 424	trn2	v15.2d, v21.2d, v23.2d
 425	# Add back state, XOR in message and store (load next block)
 426	add	v20.4s, v0.4s, v16.4s
 427	add	v21.4s, v4.4s, v17.4s
 428	add	v22.4s, v8.4s, v18.4s
 429	add	v23.4s, v12.4s, v19.4s
 430	eor	v20.16b, v20.16b, v24.16b
 431	eor	v21.16b, v21.16b, v25.16b
 432	eor	v22.16b, v22.16b, v26.16b
 433	eor	v23.16b, v23.16b, v27.16b
 434	ld1	{v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40
 435	st1	{v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40
 436	add	v20.4s, v1.4s, v16.4s
 437	add	v21.4s, v5.4s, v17.4s
 438	add	v22.4s, v9.4s, v18.4s
 439	add	v23.4s, v13.4s, v19.4s
 440	eor	v20.16b, v20.16b, v24.16b
 441	eor	v21.16b, v21.16b, v25.16b
 442	eor	v22.16b, v22.16b, v26.16b
 443	eor	v23.16b, v23.16b, v27.16b
 444	ld1	{v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40
 445	st1	{v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40
 446	add	v20.4s, v2.4s, v16.4s
 447	add	v21.4s, v6.4s, v17.4s
 448	add	v22.4s, v10.4s, v18.4s
 449	add	v23.4s, v14.4s, v19.4s
 450	eor	v20.16b, v20.16b, v24.16b
 451	eor	v21.16b, v21.16b, v25.16b
 452	eor	v22.16b, v22.16b, v26.16b
 453	eor	v23.16b, v23.16b, v27.16b
 454	ld1	{v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40
 455	st1	{v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40
 456	add	v20.4s, v3.4s, v16.4s
 457	add	v21.4s, v7.4s, v17.4s
 458	add	v22.4s, v11.4s, v18.4s
 459	add	v23.4s, v15.4s, v19.4s
 460	eor	v20.16b, v20.16b, v24.16b
 461	eor	v21.16b, v21.16b, v25.16b
 462	eor	v22.16b, v22.16b, v26.16b
 463	eor	v23.16b, v23.16b, v27.16b
 464	ld1	{v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40
 465	st1	{v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40
 466	# Move regular registers into vector registers for adding and xor
 467	mov	v0.d[0], x8
 468	mov	v0.d[1], x10
 469	mov	v1.d[0], x12
 470	mov	v1.d[1], x14
 471	mov	v2.d[0], x16
 472	mov	v2.d[1], x19
 473	mov	v3.d[0], x21
 474	mov	v3.d[1], x23
 475	# Add back state, XOR in message and store
 476	add	v0.4s, v0.4s, v16.4s
 477	add	v1.4s, v1.4s, v17.4s
 478	add	v2.4s, v2.4s, v18.4s
 479	add	v3.4s, v3.4s, v19.4s
 480	eor	v0.16b, v0.16b, v24.16b
 481	eor	v1.16b, v1.16b, v25.16b
 482	eor	v2.16b, v2.16b, v26.16b
 483	eor	v3.16b, v3.16b, v27.16b
 484	st1	{v0.4s, v1.4s, v2.4s, v3.4s}, [x1], #0x40
 485	cmp	x3, #0x140
 486	add	v19.4s, v19.4s, v29.4s
 487	bge	L_chacha_crypt_bytes_arm64_loop_320
 488	# Done doing 320 bytes at a time
 489L_chacha_crypt_bytes_arm64_lt_320:
 490	cmp	x3, #0x100
 491	blt	L_chacha_crypt_bytes_arm64_lt_256
 492	# Move state into vector registers
 493	dup	v0.4s, v16.s[0]
 494	dup	v1.4s, v16.s[1]
 495	dup	v2.4s, v16.s[2]
 496	dup	v3.4s, v16.s[3]
 497	dup	v4.4s, v17.s[0]
 498	dup	v5.4s, v17.s[1]
 499	dup	v6.4s, v17.s[2]
 500	dup	v7.4s, v17.s[3]
 501	dup	v8.4s, v18.s[0]
 502	dup	v9.4s, v18.s[1]
 503	dup	v10.4s, v18.s[2]
 504	dup	v11.4s, v18.s[3]
 505	dup	v12.4s, v19.s[0]
 506	dup	v13.4s, v19.s[1]
 507	dup	v14.4s, v19.s[2]
 508	dup	v15.4s, v19.s[3]
 509	# Add to counter word
 510	add	v12.4s, v12.4s, v28.4s
 511	# Set number of odd+even rounds to perform
 512	mov	x26, #10
 513L_chacha_crypt_bytes_arm64_round_start_256:
 514	subs	x26, x26, #1
 515	# Round odd
 516	# a += b; d ^= a; d <<<= 16;
 517	add	v0.4s, v0.4s, v4.4s
 518	add	v1.4s, v1.4s, v5.4s
 519	add	v2.4s, v2.4s, v6.4s
 520	add	v3.4s, v3.4s, v7.4s
 521	eor	v12.16b, v12.16b, v0.16b
 522	eor	v13.16b, v13.16b, v1.16b
 523	eor	v14.16b, v14.16b, v2.16b
 524	eor	v15.16b, v15.16b, v3.16b
 525	rev32	v12.8h, v12.8h
 526	rev32	v13.8h, v13.8h
 527	rev32	v14.8h, v14.8h
 528	rev32	v15.8h, v15.8h
 529	# c += d; b ^= c; b <<<= 12;
 530	add	v8.4s, v8.4s, v12.4s
 531	add	v9.4s, v9.4s, v13.4s
 532	add	v10.4s, v10.4s, v14.4s
 533	add	v11.4s, v11.4s, v15.4s
 534	eor	v20.16b, v4.16b, v8.16b
 535	eor	v21.16b, v5.16b, v9.16b
 536	eor	v22.16b, v6.16b, v10.16b
 537	eor	v23.16b, v7.16b, v11.16b
 538	shl	v4.4s, v20.4s, #12
 539	shl	v5.4s, v21.4s, #12
 540	shl	v6.4s, v22.4s, #12
 541	shl	v7.4s, v23.4s, #12
 542	sri	v4.4s, v20.4s, #20
 543	sri	v5.4s, v21.4s, #20
 544	sri	v6.4s, v22.4s, #20
 545	sri	v7.4s, v23.4s, #20
 546	# a += b; d ^= a; d <<<= 8;
 547	add	v0.4s, v0.4s, v4.4s
 548	add	v1.4s, v1.4s, v5.4s
 549	add	v2.4s, v2.4s, v6.4s
 550	add	v3.4s, v3.4s, v7.4s
 551	eor	v12.16b, v12.16b, v0.16b
 552	eor	v13.16b, v13.16b, v1.16b
 553	eor	v14.16b, v14.16b, v2.16b
 554	eor	v15.16b, v15.16b, v3.16b
 555	tbl	v12.16b, {v12.16b}, v30.16b
 556	tbl	v13.16b, {v13.16b}, v30.16b
 557	tbl	v14.16b, {v14.16b}, v30.16b
 558	tbl	v15.16b, {v15.16b}, v30.16b
 559	# c += d; b ^= c; b <<<= 7;
 560	add	v8.4s, v8.4s, v12.4s
 561	add	v9.4s, v9.4s, v13.4s
 562	add	v10.4s, v10.4s, v14.4s
 563	add	v11.4s, v11.4s, v15.4s
 564	eor	v20.16b, v4.16b, v8.16b
 565	eor	v21.16b, v5.16b, v9.16b
 566	eor	v22.16b, v6.16b, v10.16b
 567	eor	v23.16b, v7.16b, v11.16b
 568	shl	v4.4s, v20.4s, #7
 569	shl	v5.4s, v21.4s, #7
 570	shl	v6.4s, v22.4s, #7
 571	shl	v7.4s, v23.4s, #7
 572	sri	v4.4s, v20.4s, #25
 573	sri	v5.4s, v21.4s, #25
 574	sri	v6.4s, v22.4s, #25
 575	sri	v7.4s, v23.4s, #25
 576	# Round even
 577	# a += b; d ^= a; d <<<= 16;
 578	add	v0.4s, v0.4s, v5.4s
 579	add	v1.4s, v1.4s, v6.4s
 580	add	v2.4s, v2.4s, v7.4s
 581	add	v3.4s, v3.4s, v4.4s
 582	eor	v15.16b, v15.16b, v0.16b
 583	eor	v12.16b, v12.16b, v1.16b
 584	eor	v13.16b, v13.16b, v2.16b
 585	eor	v14.16b, v14.16b, v3.16b
 586	rev32	v15.8h, v15.8h
 587	rev32	v12.8h, v12.8h
 588	rev32	v13.8h, v13.8h
 589	rev32	v14.8h, v14.8h
 590	# c += d; b ^= c; b <<<= 12;
 591	add	v10.4s, v10.4s, v15.4s
 592	add	v11.4s, v11.4s, v12.4s
 593	add	v8.4s, v8.4s, v13.4s
 594	add	v9.4s, v9.4s, v14.4s
 595	eor	v20.16b, v5.16b, v10.16b
 596	eor	v21.16b, v6.16b, v11.16b
 597	eor	v22.16b, v7.16b, v8.16b
 598	eor	v23.16b, v4.16b, v9.16b
 599	shl	v5.4s, v20.4s, #12
 600	shl	v6.4s, v21.4s, #12
 601	shl	v7.4s, v22.4s, #12
 602	shl	v4.4s, v23.4s, #12
 603	sri	v5.4s, v20.4s, #20
 604	sri	v6.4s, v21.4s, #20
 605	sri	v7.4s, v22.4s, #20
 606	sri	v4.4s, v23.4s, #20
 607	# a += b; d ^= a; d <<<= 8;
 608	add	v0.4s, v0.4s, v5.4s
 609	add	v1.4s, v1.4s, v6.4s
 610	add	v2.4s, v2.4s, v7.4s
 611	add	v3.4s, v3.4s, v4.4s
 612	eor	v15.16b, v15.16b, v0.16b
 613	eor	v12.16b, v12.16b, v1.16b
 614	eor	v13.16b, v13.16b, v2.16b
 615	eor	v14.16b, v14.16b, v3.16b
 616	tbl	v15.16b, {v15.16b}, v30.16b
 617	tbl	v12.16b, {v12.16b}, v30.16b
 618	tbl	v13.16b, {v13.16b}, v30.16b
 619	tbl	v14.16b, {v14.16b}, v30.16b
 620	# c += d; b ^= c; b <<<= 7;
 621	add	v10.4s, v10.4s, v15.4s
 622	add	v11.4s, v11.4s, v12.4s
 623	add	v8.4s, v8.4s, v13.4s
 624	add	v9.4s, v9.4s, v14.4s
 625	eor	v20.16b, v5.16b, v10.16b
 626	eor	v21.16b, v6.16b, v11.16b
 627	eor	v22.16b, v7.16b, v8.16b
 628	eor	v23.16b, v4.16b, v9.16b
 629	shl	v5.4s, v20.4s, #7
 630	shl	v6.4s, v21.4s, #7
 631	shl	v7.4s, v22.4s, #7
 632	shl	v4.4s, v23.4s, #7
 633	sri	v5.4s, v20.4s, #25
 634	sri	v6.4s, v21.4s, #25
 635	sri	v7.4s, v22.4s, #25
 636	sri	v4.4s, v23.4s, #25
 637	bne	L_chacha_crypt_bytes_arm64_round_start_256
 638	mov	x26, #4
 639	# Add counter now rather than after transposed
 640	add	v12.4s, v12.4s, v28.4s
 641	# Load message
 642	ld1	{v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40
 643	# Transpose vectors
 644	trn1	v20.4s, v0.4s, v1.4s
 645	trn1	v22.4s, v2.4s, v3.4s
 646	trn2	v21.4s, v0.4s, v1.4s
 647	trn2	v23.4s, v2.4s, v3.4s
 648	trn1	v0.2d, v20.2d, v22.2d
 649	trn1	v1.2d, v21.2d, v23.2d
 650	trn2	v2.2d, v20.2d, v22.2d
 651	trn2	v3.2d, v21.2d, v23.2d
 652	trn1	v20.4s, v4.4s, v5.4s
 653	trn1	v22.4s, v6.4s, v7.4s
 654	trn2	v21.4s, v4.4s, v5.4s
 655	trn2	v23.4s, v6.4s, v7.4s
 656	trn1	v4.2d, v20.2d, v22.2d
 657	trn1	v5.2d, v21.2d, v23.2d
 658	trn2	v6.2d, v20.2d, v22.2d
 659	trn2	v7.2d, v21.2d, v23.2d
 660	trn1	v20.4s, v8.4s, v9.4s
 661	trn1	v22.4s, v10.4s, v11.4s
 662	trn2	v21.4s, v8.4s, v9.4s
 663	trn2	v23.4s, v10.4s, v11.4s
 664	trn1	v8.2d, v20.2d, v22.2d
 665	trn1	v9.2d, v21.2d, v23.2d
 666	trn2	v10.2d, v20.2d, v22.2d
 667	trn2	v11.2d, v21.2d, v23.2d
 668	trn1	v20.4s, v12.4s, v13.4s
 669	trn1	v22.4s, v14.4s, v15.4s
 670	trn2	v21.4s, v12.4s, v13.4s
 671	trn2	v23.4s, v14.4s, v15.4s
 672	trn1	v12.2d, v20.2d, v22.2d
 673	trn1	v13.2d, v21.2d, v23.2d
 674	trn2	v14.2d, v20.2d, v22.2d
 675	trn2	v15.2d, v21.2d, v23.2d
 676	# Add back state, XOR in message and store (load next block)
 677	add	v20.4s, v0.4s, v16.4s
 678	add	v21.4s, v4.4s, v17.4s
 679	add	v22.4s, v8.4s, v18.4s
 680	add	v23.4s, v12.4s, v19.4s
 681	eor	v20.16b, v20.16b, v24.16b
 682	eor	v21.16b, v21.16b, v25.16b
 683	eor	v22.16b, v22.16b, v26.16b
 684	eor	v23.16b, v23.16b, v27.16b
 685	ld1	{v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40
 686	st1	{v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40
 687	add	v20.4s, v1.4s, v16.4s
 688	add	v21.4s, v5.4s, v17.4s
 689	add	v22.4s, v9.4s, v18.4s
 690	add	v23.4s, v13.4s, v19.4s
 691	eor	v20.16b, v20.16b, v24.16b
 692	eor	v21.16b, v21.16b, v25.16b
 693	eor	v22.16b, v22.16b, v26.16b
 694	eor	v23.16b, v23.16b, v27.16b
 695	ld1	{v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40
 696	st1	{v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40
 697	add	v20.4s, v2.4s, v16.4s
 698	add	v21.4s, v6.4s, v17.4s
 699	add	v22.4s, v10.4s, v18.4s
 700	add	v23.4s, v14.4s, v19.4s
 701	eor	v20.16b, v20.16b, v24.16b
 702	eor	v21.16b, v21.16b, v25.16b
 703	eor	v22.16b, v22.16b, v26.16b
 704	eor	v23.16b, v23.16b, v27.16b
 705	ld1	{v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40
 706	st1	{v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40
 707	add	v20.4s, v3.4s, v16.4s
 708	add	v21.4s, v7.4s, v17.4s
 709	add	v22.4s, v11.4s, v18.4s
 710	add	v23.4s, v15.4s, v19.4s
 711	eor	v20.16b, v20.16b, v24.16b
 712	eor	v21.16b, v21.16b, v25.16b
 713	eor	v22.16b, v22.16b, v26.16b
 714	eor	v23.16b, v23.16b, v27.16b
 715	st1	{v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40
 716	mov	v29.s[0], w26
 717	sub	x3, x3, #0x100
 718	add	v19.4s, v19.4s, v29.4s
 719	# Done 256-byte block
 720L_chacha_crypt_bytes_arm64_lt_256:
 721	cmp	x3, #0x80
 722	blt	L_chacha_crypt_bytes_arm64_lt_128
 723	ld1	{v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40
 724	# Move state into vector registers
 725	mov	v4.16b, v16.16b
 726	mov	v5.16b, v17.16b
 727	mov	v6.16b, v18.16b
 728	mov	v7.16b, v19.16b
 729	mov	v0.16b, v16.16b
 730	mov	v1.16b, v17.16b
 731	mov	v2.16b, v18.16b
 732	mov	v3.16b, v19.16b
 733	# Add counter word
 734	add	v7.4s, v7.4s, v31.4s
 735	# Set number of odd+even rounds to perform
 736	mov	x26, #10
 737L_chacha_crypt_bytes_arm64_round_start_128:
 738	subs	x26, x26, #1
 739	# Round odd
 740	# a += b; d ^= a; d <<<= 16;
 741	add	v0.4s, v0.4s, v1.4s
 742	add	v4.4s, v4.4s, v5.4s
 743	eor	v3.16b, v3.16b, v0.16b
 744	eor	v7.16b, v7.16b, v4.16b
 745	rev32	v3.8h, v3.8h
 746	rev32	v7.8h, v7.8h
 747	# c += d; b ^= c; b <<<= 12;
 748	add	v2.4s, v2.4s, v3.4s
 749	add	v6.4s, v6.4s, v7.4s
 750	eor	v20.16b, v1.16b, v2.16b
 751	eor	v21.16b, v5.16b, v6.16b
 752	shl	v1.4s, v20.4s, #12
 753	shl	v5.4s, v21.4s, #12
 754	sri	v1.4s, v20.4s, #20
 755	sri	v5.4s, v21.4s, #20
 756	# a += b; d ^= a; d <<<= 8;
 757	add	v0.4s, v0.4s, v1.4s
 758	add	v4.4s, v4.4s, v5.4s
 759	eor	v3.16b, v3.16b, v0.16b
 760	eor	v7.16b, v7.16b, v4.16b
 761	tbl	v3.16b, {v3.16b}, v30.16b
 762	tbl	v7.16b, {v7.16b}, v30.16b
 763	# c += d; b ^= c; b <<<= 7;
 764	add	v2.4s, v2.4s, v3.4s
 765	add	v6.4s, v6.4s, v7.4s
 766	eor	v20.16b, v1.16b, v2.16b
 767	eor	v21.16b, v5.16b, v6.16b
 768	shl	v1.4s, v20.4s, #7
 769	shl	v5.4s, v21.4s, #7
 770	sri	v1.4s, v20.4s, #25
 771	sri	v5.4s, v21.4s, #25
 772	ext	v3.16b, v3.16b, v3.16b, #12
 773	ext	v7.16b, v7.16b, v7.16b, #12
 774	ext	v1.16b, v1.16b, v1.16b, #4
 775	ext	v5.16b, v5.16b, v5.16b, #4
 776	ext	v2.16b, v2.16b, v2.16b, #8
 777	ext	v6.16b, v6.16b, v6.16b, #8
 778	# Round even
 779	# a += b; d ^= a; d <<<= 16;
 780	add	v0.4s, v0.4s, v1.4s
 781	add	v4.4s, v4.4s, v5.4s
 782	eor	v3.16b, v3.16b, v0.16b
 783	eor	v7.16b, v7.16b, v4.16b
 784	rev32	v3.8h, v3.8h
 785	rev32	v7.8h, v7.8h
 786	# c += d; b ^= c; b <<<= 12;
 787	add	v2.4s, v2.4s, v3.4s
 788	add	v6.4s, v6.4s, v7.4s
 789	eor	v20.16b, v1.16b, v2.16b
 790	eor	v21.16b, v5.16b, v6.16b
 791	shl	v1.4s, v20.4s, #12
 792	shl	v5.4s, v21.4s, #12
 793	sri	v1.4s, v20.4s, #20
 794	sri	v5.4s, v21.4s, #20
 795	# a += b; d ^= a; d <<<= 8;
 796	add	v0.4s, v0.4s, v1.4s
 797	add	v4.4s, v4.4s, v5.4s
 798	eor	v3.16b, v3.16b, v0.16b
 799	eor	v7.16b, v7.16b, v4.16b
 800	tbl	v3.16b, {v3.16b}, v30.16b
 801	tbl	v7.16b, {v7.16b}, v30.16b
 802	# c += d; b ^= c; b <<<= 7;
 803	add	v2.4s, v2.4s, v3.4s
 804	add	v6.4s, v6.4s, v7.4s
 805	eor	v20.16b, v1.16b, v2.16b
 806	eor	v21.16b, v5.16b, v6.16b
 807	shl	v1.4s, v20.4s, #7
 808	shl	v5.4s, v21.4s, #7
 809	sri	v1.4s, v20.4s, #25
 810	sri	v5.4s, v21.4s, #25
 811	ext	v3.16b, v3.16b, v3.16b, #4
 812	ext	v7.16b, v7.16b, v7.16b, #4
 813	ext	v1.16b, v1.16b, v1.16b, #12
 814	ext	v5.16b, v5.16b, v5.16b, #12
 815	ext	v2.16b, v2.16b, v2.16b, #8
 816	ext	v6.16b, v6.16b, v6.16b, #8
 817	bne	L_chacha_crypt_bytes_arm64_round_start_128
 818	# Add back state, XOR in message and store (load next block)
 819	add	v0.4s, v0.4s, v16.4s
 820	add	v1.4s, v1.4s, v17.4s
 821	add	v2.4s, v2.4s, v18.4s
 822	add	v3.4s, v3.4s, v19.4s
 823	eor	v24.16b, v24.16b, v0.16b
 824	eor	v25.16b, v25.16b, v1.16b
 825	eor	v26.16b, v26.16b, v2.16b
 826	eor	v27.16b, v27.16b, v3.16b
 827	ld1	{v20.16b, v21.16b, v22.16b, v23.16b}, [x2], #0x40
 828	st1	{v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40
 829	add	v19.4s, v19.4s, v31.4s
 830	add	v4.4s, v4.4s, v16.4s
 831	add	v5.4s, v5.4s, v17.4s
 832	add	v6.4s, v6.4s, v18.4s
 833	add	v7.4s, v7.4s, v19.4s
 834	eor	v20.16b, v20.16b, v4.16b
 835	eor	v21.16b, v21.16b, v5.16b
 836	eor	v22.16b, v22.16b, v6.16b
 837	eor	v23.16b, v23.16b, v7.16b
 838	st1	{v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40
 839	add	v19.4s, v19.4s, v31.4s
 840	sub	x3, x3, #0x80
 841	# Done 128-byte block
 842L_chacha_crypt_bytes_arm64_lt_128:
 843	cmp	x3, #0
 844	beq	L_chacha_crypt_bytes_arm64_done_all
 845	mov	w5, #0x40
 846L_chacha_crypt_bytes_arm64_loop_64:
 847	# Move state into vector registers
 848	mov	v0.16b, v16.16b
 849	mov	v1.16b, v17.16b
 850	mov	v2.16b, v18.16b
 851	mov	v3.16b, v19.16b
 852	# Set number of odd+even rounds to perform
 853	mov	x26, #10
 854L_chacha_crypt_bytes_arm64_round_64:
 855	subs	x26, x26, #1
 856	# Round odd
 857	# a += b; d ^= a; d <<<= 16;
 858	add	v0.4s, v0.4s, v1.4s
 859	eor	v3.16b, v3.16b, v0.16b
 860	rev32	v3.8h, v3.8h
 861	# c += d; b ^= c; b <<<= 12;
 862	add	v2.4s, v2.4s, v3.4s
 863	eor	v20.16b, v1.16b, v2.16b
 864	shl	v1.4s, v20.4s, #12
 865	sri	v1.4s, v20.4s, #20
 866	# a += b; d ^= a; d <<<= 8;
 867	add	v0.4s, v0.4s, v1.4s
 868	eor	v3.16b, v3.16b, v0.16b
 869	tbl	v3.16b, {v3.16b}, v30.16b
 870	# c += d; b ^= c; b <<<= 7;
 871	add	v2.4s, v2.4s, v3.4s
 872	eor	v20.16b, v1.16b, v2.16b
 873	shl	v1.4s, v20.4s, #7
 874	sri	v1.4s, v20.4s, #25
 875	ext	v3.16b, v3.16b, v3.16b, #12
 876	ext	v1.16b, v1.16b, v1.16b, #4
 877	ext	v2.16b, v2.16b, v2.16b, #8
 878	# Round even
 879	# a += b; d ^= a; d <<<= 16;
 880	add	v0.4s, v0.4s, v1.4s
 881	eor	v3.16b, v3.16b, v0.16b
 882	rev32	v3.8h, v3.8h
 883	# c += d; b ^= c; b <<<= 12;
 884	add	v2.4s, v2.4s, v3.4s
 885	eor	v20.16b, v1.16b, v2.16b
 886	shl	v1.4s, v20.4s, #12
 887	sri	v1.4s, v20.4s, #20
 888	# a += b; d ^= a; d <<<= 8;
 889	add	v0.4s, v0.4s, v1.4s
 890	eor	v3.16b, v3.16b, v0.16b
 891	tbl	v3.16b, {v3.16b}, v30.16b
 892	# c += d; b ^= c; b <<<= 7;
 893	add	v2.4s, v2.4s, v3.4s
 894	eor	v20.16b, v1.16b, v2.16b
 895	shl	v1.4s, v20.4s, #7
 896	sri	v1.4s, v20.4s, #25
 897	ext	v3.16b, v3.16b, v3.16b, #4
 898	ext	v1.16b, v1.16b, v1.16b, #12
 899	ext	v2.16b, v2.16b, v2.16b, #8
 900	bne	L_chacha_crypt_bytes_arm64_round_64
 901	# Add back state
 902	add	v0.4s, v0.4s, v16.4s
 903	add	v1.4s, v1.4s, v17.4s
 904	add	v2.4s, v2.4s, v18.4s
 905	add	v3.4s, v3.4s, v19.4s
 906	# Check if data is less than 64 bytes - store in over
 907	cmp	x3, #0x40
 908	add	v19.4s, v19.4s, v31.4s
 909	blt	L_chacha_crypt_bytes_arm64_lt_64
 910	# Encipher 64 bytes
 911	ld1	{v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40
 912	eor	v24.16b, v24.16b, v0.16b
 913	eor	v25.16b, v25.16b, v1.16b
 914	eor	v26.16b, v26.16b, v2.16b
 915	eor	v27.16b, v27.16b, v3.16b
 916	st1	{v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40
 917	# Check for more bytes to be enciphered
 918	subs	x3, x3, #0x40
 919	bne	L_chacha_crypt_bytes_arm64_loop_64
 920	b	L_chacha_crypt_bytes_arm64_done
 921L_chacha_crypt_bytes_arm64_lt_64:
 922	# Calculate bytes left in block not used
 923	sub	w5, w5, w3
 924	# Store encipher block in over for further operations and left
 925	st1	{v0.4s, v1.4s, v2.4s, v3.4s}, [x4]
 926	str	w5, [x0, #64]
 927	# Encipher 32 bytes
 928	cmp	x3, #32
 929	blt	L_chacha_crypt_bytes_arm64_lt_32
 930	ld1	{v24.16b, v25.16b}, [x2], #32
 931	eor	v24.16b, v24.16b, v0.16b
 932	eor	v25.16b, v25.16b, v1.16b
 933	st1	{v24.16b, v25.16b}, [x1], #32
 934	subs	x3, x3, #32
 935	mov	v0.16b, v2.16b
 936	mov	v1.16b, v3.16b
 937	beq	L_chacha_crypt_bytes_arm64_done
 938L_chacha_crypt_bytes_arm64_lt_32:
 939	cmp	x3, #16
 940	blt	L_chacha_crypt_bytes_arm64_lt_16
 941	# Encipher 16 bytes
 942	ld1	{v24.16b}, [x2], #16
 943	eor	v24.16b, v24.16b, v0.16b
 944	st1	{v24.16b}, [x1], #16
 945	subs	x3, x3, #16
 946	mov	v0.16b, v1.16b
 947	beq	L_chacha_crypt_bytes_arm64_done
 948L_chacha_crypt_bytes_arm64_lt_16:
 949	cmp	x3, #8
 950	blt	L_chacha_crypt_bytes_arm64_lt_8
 951	# Encipher 8 bytes
 952	ld1	{v24.8b}, [x2], #8
 953	eor	v24.8b, v24.8b, v0.8b
 954	st1	{v24.8b}, [x1], #8
 955	subs	x3, x3, #8
 956	mov	v0.d[0], v0.d[1]
 957	beq	L_chacha_crypt_bytes_arm64_done
 958L_chacha_crypt_bytes_arm64_lt_8:
 959	mov	x5, v0.d[0]
 960L_chacha_crypt_bytes_arm64_loop_lt_8:
 961	# Encipher 1 byte at a time
 962	ldrb	w6, [x2], #1
 963	eor	w6, w6, w5
 964	strb	w6, [x1], #1
 965	subs	x3, x3, #1
 966	lsr	x5, x5, #8
 967	bgt	L_chacha_crypt_bytes_arm64_loop_lt_8
 968L_chacha_crypt_bytes_arm64_done:
 969L_chacha_crypt_bytes_arm64_done_all:
 970	st1	{v16.4s, v17.4s, v18.4s, v19.4s}, [x0]
 971	ldp	x17, x19, [x29, #24]
 972	ldp	x20, x21, [x29, #40]
 973	ldp	x22, x23, [x29, #56]
 974	ldp	x24, x25, [x29, #72]
 975	ldr	x26, [x29, #88]
 976	ldp	d8, d9, [x29, #96]
 977	ldp	d10, d11, [x29, #112]
 978	ldp	d12, d13, [x29, #128]
 979	ldp	d14, d15, [x29, #144]
 980	ldp	x29, x30, [sp], #0xa0
 981	ret
 982#ifndef __APPLE__
 983	.size	wc_chacha_crypt_bytes,.-wc_chacha_crypt_bytes
 984#endif /* __APPLE__ */
 985#ifndef __APPLE__
 986.text
 987.globl	wc_chacha_setiv
 988.type	wc_chacha_setiv,@function
 989.align	2
 990wc_chacha_setiv:
 991#else
 992.section	__TEXT,__text
 993.globl	_wc_chacha_setiv
 994.p2align	2
 995_wc_chacha_setiv:
 996#endif /* __APPLE__ */
 997	ldr	x3, [x1]
 998	ldr	w4, [x1, #8]
 999	str	x2, [x0, #48]
1000	str	x3, [x0, #52]
1001	str	w4, [x0, #60]
1002	ret
1003#ifndef __APPLE__
1004	.size	wc_chacha_setiv,.-wc_chacha_setiv
1005#endif /* __APPLE__ */
1006#ifndef __APPLE__
1007	.text
1008	.section	.rodata
1009	.type	L_chacha_setkey_arm64_constant, %object
1010	.size	L_chacha_setkey_arm64_constant, 32
1011#else
1012	.section	__DATA,__data
1013#endif /* __APPLE__ */
1014	# 8-byte aligned, 64-bit aligned
1015#ifndef __APPLE__
1016	.align	3
1017#else
1018	.p2align	3
1019#endif /* __APPLE__ */
1020L_chacha_setkey_arm64_constant:
1021	.long	0x61707865,0x3120646e,0x79622d36,0x6b206574
1022	.long	0x61707865,0x3320646e,0x79622d32,0x6b206574
1023#ifndef __APPLE__
1024.text
1025.globl	wc_chacha_setkey
1026.type	wc_chacha_setkey,@function
1027.align	2
1028wc_chacha_setkey:
1029#else
1030.section	__TEXT,__text
1031.globl	_wc_chacha_setkey
1032.p2align	2
1033_wc_chacha_setkey:
1034#endif /* __APPLE__ */
1035#ifndef __APPLE__
1036	adrp x3, L_chacha_setkey_arm64_constant
1037	add  x3, x3, :lo12:L_chacha_setkey_arm64_constant
1038#else
1039	adrp x3, L_chacha_setkey_arm64_constant@PAGE
1040	add  x3, x3, L_chacha_setkey_arm64_constant@PAGEOFF
1041#endif /* __APPLE__ */
1042	subs	x2, x2, #16
1043	add	x3, x3, x2
1044	# Start with constants
1045	ld1	{v0.4s}, [x3]
1046	ld1	{v1.16b}, [x1], #16
1047#ifdef BIG_ENDIAN_ORDER
1048	rev32	v1.8h, v1.8h
1049#endif /* BIG_ENDIAN_ORDER */
1050	st1	{v0.4s}, [x0], #16
1051	st1	{v1.4s}, [x0], #16
1052	beq	L_chacha_setkey_arm64_done
1053	ld1	{v1.16b}, [x1]
1054#ifdef BIG_ENDIAN_ORDER
1055	rev32	v1.8h, v1.8h
1056#endif /* BIG_ENDIAN_ORDER */
1057L_chacha_setkey_arm64_done:
1058	st1	{v1.4s}, [x0]
1059	ret
1060#ifndef __APPLE__
1061	.size	wc_chacha_setkey,.-wc_chacha_setkey
1062#endif /* __APPLE__ */
1063#ifndef __APPLE__
1064.text
1065.globl	wc_chacha_use_over
1066.type	wc_chacha_use_over,@function
1067.align	2
1068wc_chacha_use_over:
1069#else
1070.section	__TEXT,__text
1071.globl	_wc_chacha_use_over
1072.p2align	2
1073_wc_chacha_use_over:
1074#endif /* __APPLE__ */
1075L_chacha_use_over_arm64_16byte_loop:
1076	cmp	x3, #16
1077	blt	L_chacha_use_over_arm64_word_loop
1078	# 16 bytes of state XORed into message.
1079	ld1	{v0.16b}, [x0], #16
1080	ld1	{v1.16b}, [x2], #16
1081	eor	v1.16b, v1.16b, v0.16b
1082	subs	x3, x3, #16
1083	st1	{v1.16b}, [x1], #16
1084	beq	L_chacha_use_over_arm64_done
1085	b	L_chacha_use_over_arm64_16byte_loop
1086L_chacha_use_over_arm64_word_loop:
1087	cmp	x3, #4
1088	blt	L_chacha_use_over_arm64_byte_loop
1089	# 4 bytes of state XORed into message.
1090	ldr	w4, [x0], #4
1091	ldr	w5, [x2], #4
1092	eor	w5, w5, w4
1093	subs	x3, x3, #4
1094	str	w5, [x1], #4
1095	beq	L_chacha_use_over_arm64_done
1096	b	L_chacha_use_over_arm64_word_loop
1097L_chacha_use_over_arm64_byte_loop:
1098	# 1 bytes of state XORed into message.
1099	ldrb	w4, [x0], #1
1100	ldrb	w5, [x2], #1
1101	eor	w5, w5, w4
1102	subs	x3, x3, #1
1103	strb	w5, [x1], #1
1104	beq	L_chacha_use_over_arm64_done
1105	b	L_chacha_use_over_arm64_byte_loop
1106L_chacha_use_over_arm64_done:
1107	ret
1108#ifndef __APPLE__
1109	.size	wc_chacha_use_over,.-wc_chacha_use_over
1110#endif /* __APPLE__ */
1111#endif /* !WOLFSSL_ARMASM_NO_NEON */
1112#endif /* HAVE_CHACHA */
1113#endif /* __aarch64__ */
1114#endif /* WOLFSSL_ARMASM */
1115
1116#if defined(__linux__) && defined(__ELF__)
1117.section	.note.GNU-stack,"",%progbits
1118#endif
1119#endif /* !WOLFSSL_ARMASM_INLINE */