/* armv8-aes-asm * * Copyright (C) 2006-2026 wolfSSL Inc. * * This file is part of wolfSSL. * * wolfSSL is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 3 of the License, or * (at your option) any later version. * * wolfSSL is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA */ #include /* Generated using (from wolfssl): * cd ../scripts * ruby ./aes/aes.rb arm64 \ * ../wolfssl/wolfcrypt/src/port/arm/armv8-aes-asm.S */ #ifdef WOLFSSL_ARMASM #ifdef __aarch64__ #ifndef WOLFSSL_ARMASM_INLINE #if !defined(NO_AES) && defined(WOLFSSL_ARMASM) #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO #ifndef __APPLE__ .text .globl AES_set_key_AARCH64 .type AES_set_key_AARCH64,@function .align 2 AES_set_key_AARCH64: #else .section __TEXT,__text .globl _AES_set_key_AARCH64 .p2align 2 _AES_set_key_AARCH64: #endif /* __APPLE__ */ cmp x1, #24 blt L_aes_set_key_arm64_crypto_start_128 bgt L_aes_set_key_arm64_crypto_start_256 ldr x4, [x0], #8 ldr x6, [x0], #8 ldr x8, [x0], #8 stp x4, x6, [x2], #16 str x8, [x2], #8 lsr x5, x4, #32 lsr x7, x6, #32 lsr x9, x8, #32 dup v1.4s, w9 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #1 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 eor w8, w8, w7 eor w9, w9, w8 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 dup v1.4s, w9 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #2 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 eor w8, w8, w7 eor w9, w9, w8 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 dup v1.4s, w9 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #4 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 eor w8, w8, w7 eor w9, w9, w8 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 dup v1.4s, w9 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #8 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 eor w8, w8, w7 eor w9, w9, w8 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 dup v1.4s, w9 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #16 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 eor w8, w8, w7 eor w9, w9, w8 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 dup v1.4s, w9 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #32 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 eor w8, w8, w7 eor w9, w9, w8 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 dup v1.4s, w9 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #0x40 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 eor w8, w8, w7 eor w9, w9, w8 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 dup v1.4s, w9 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #0x80 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 cmp x3, #0 beq L_aes_set_key_arm64_crypto_done sub x2, x2, #0xd0 ldur q0, [x2] ldur q1, [x2, #192] stur q1, [x2] stur q0, [x2, #192] ldur q0, [x2, #16] ldur q1, [x2, #176] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #16] stur q0, [x2, #176] ldur q0, [x2, #32] ldur q1, [x2, #160] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #32] stur q0, [x2, #160] ldur q0, [x2, #48] ldur q1, [x2, #144] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #48] stur q0, [x2, #144] ldur q0, [x2, #64] ldur q1, [x2, #128] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #64] stur q0, [x2, #128] ldur q0, [x2, #80] ldur q1, [x2, #112] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #80] stur q0, [x2, #112] ldur q0, [x2, #96] aesimc v0.16b, v0.16b stur q0, [x2, #96] b L_aes_set_key_arm64_crypto_done L_aes_set_key_arm64_crypto_start_256: ldr x4, [x0], #8 ldr x6, [x0], #8 ldr x8, [x0], #8 ldr x10, [x0], #8 stp x4, x6, [x2], #16 stp x8, x10, [x2], #16 lsr x5, x4, #32 lsr x7, x6, #32 lsr x9, x8, #32 lsr x11, x10, #32 dup v1.4s, w11 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #1 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] eor w8, w8, w12 eor w9, w9, w8 eor w10, w10, w9 eor w11, w11, w10 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 stp w10, w11, [x2], #8 dup v1.4s, w11 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #2 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] eor w8, w8, w12 eor w9, w9, w8 eor w10, w10, w9 eor w11, w11, w10 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 stp w10, w11, [x2], #8 dup v1.4s, w11 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #4 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] eor w8, w8, w12 eor w9, w9, w8 eor w10, w10, w9 eor w11, w11, w10 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 stp w10, w11, [x2], #8 dup v1.4s, w11 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #8 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] eor w8, w8, w12 eor w9, w9, w8 eor w10, w10, w9 eor w11, w11, w10 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 stp w10, w11, [x2], #8 dup v1.4s, w11 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #16 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] eor w8, w8, w12 eor w9, w9, w8 eor w10, w10, w9 eor w11, w11, w10 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 stp w10, w11, [x2], #8 dup v1.4s, w11 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #32 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] eor w8, w8, w12 eor w9, w9, w8 eor w10, w10, w9 eor w11, w11, w10 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 stp w10, w11, [x2], #8 dup v1.4s, w11 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #0x40 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 cmp x3, #0 beq L_aes_set_key_arm64_crypto_done sub x2, x2, #0xf0 ldur q0, [x2] ldur q1, [x2, #224] stur q1, [x2] stur q0, [x2, #224] ldur q0, [x2, #16] ldur q1, [x2, #208] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #16] stur q0, [x2, #208] ldur q0, [x2, #32] ldur q1, [x2, #192] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #32] stur q0, [x2, #192] ldur q0, [x2, #48] ldur q1, [x2, #176] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #48] stur q0, [x2, #176] ldur q0, [x2, #64] ldur q1, [x2, #160] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #64] stur q0, [x2, #160] ldur q0, [x2, #80] ldur q1, [x2, #144] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #80] stur q0, [x2, #144] ldur q0, [x2, #96] ldur q1, [x2, #128] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #96] stur q0, [x2, #128] ldur q0, [x2, #112] aesimc v0.16b, v0.16b stur q0, [x2, #112] b L_aes_set_key_arm64_crypto_done L_aes_set_key_arm64_crypto_start_128: ldr x4, [x0], #8 ldr x6, [x0], #8 stp x4, x6, [x2], #16 lsr x5, x4, #32 lsr x7, x6, #32 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #1 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #2 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #4 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #8 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #16 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #32 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #0x40 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] ror w12, w12, #8 eor w4, w4, #0x80 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] mov w13, #27 ror w12, w12, #8 eor w4, w4, w13 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 dup v1.4s, w7 movi v0.16b, #0 aese v0.16b, v1.16b mov w12, v0.s[0] mov w13, #54 ror w12, w12, #8 eor w4, w4, w13 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 cmp x3, #0 beq L_aes_set_key_arm64_crypto_done sub x2, x2, #0xb0 ldur q0, [x2] ldur q1, [x2, #160] stur q1, [x2] stur q0, [x2, #160] ldur q0, [x2, #16] ldur q1, [x2, #144] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #16] stur q0, [x2, #144] ldur q0, [x2, #32] ldur q1, [x2, #128] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #32] stur q0, [x2, #128] ldur q0, [x2, #48] ldur q1, [x2, #112] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #48] stur q0, [x2, #112] ldur q0, [x2, #64] ldur q1, [x2, #96] aesimc v0.16b, v0.16b aesimc v1.16b, v1.16b stur q1, [x2, #64] stur q0, [x2, #96] ldur q0, [x2, #80] aesimc v0.16b, v0.16b stur q0, [x2, #80] L_aes_set_key_arm64_crypto_done: ret #ifndef __APPLE__ .size AES_set_key_AARCH64,.-AES_set_key_AARCH64 #endif /* __APPLE__ */ #if defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || defined(HAVE_AES_CBC) #ifndef __APPLE__ .text .globl AES_encrypt_AARCH64 .type AES_encrypt_AARCH64,@function .align 2 AES_encrypt_AARCH64: #else .section __TEXT,__text .globl _AES_encrypt_AARCH64 .p2align 2 _AES_encrypt_AARCH64: #endif /* __APPLE__ */ ld1 {v0.16b}, [x0] ld1 {v1.2d, v2.2d, v3.2d, v4.2d}, [x2], #0x40 aese v0.16b, v1.16b aesmc v0.16b, v0.16b aese v0.16b, v2.16b aesmc v0.16b, v0.16b aese v0.16b, v3.16b aesmc v0.16b, v0.16b aese v0.16b, v4.16b aesmc v0.16b, v0.16b ld1 {v1.2d, v2.2d, v3.2d, v4.2d}, [x2], #0x40 aese v0.16b, v1.16b aesmc v0.16b, v0.16b aese v0.16b, v2.16b aesmc v0.16b, v0.16b aese v0.16b, v3.16b aesmc v0.16b, v0.16b aese v0.16b, v4.16b aesmc v0.16b, v0.16b subs w3, w3, #10 ld1 {v1.2d, v2.2d}, [x2], #32 aese v0.16b, v1.16b aesmc v0.16b, v0.16b aese v0.16b, v2.16b beq L_aes_encrypt_arm64_crypto_round_done ld1 {v1.2d, v2.2d}, [x2], #32 subs w3, w3, #2 aesmc v0.16b, v0.16b aese v0.16b, v1.16b aesmc v0.16b, v0.16b aese v0.16b, v2.16b beq L_aes_encrypt_arm64_crypto_round_done ld1 {v1.2d, v2.2d}, [x2], #32 aesmc v0.16b, v0.16b aese v0.16b, v1.16b aesmc v0.16b, v0.16b aese v0.16b, v2.16b L_aes_encrypt_arm64_crypto_round_done: ld1 {v1.2d}, [x2] eor v0.16b, v0.16b, v1.16b st1 {v0.16b}, [x1] ret #ifndef __APPLE__ .size AES_encrypt_AARCH64,.-AES_encrypt_AARCH64 #endif /* __APPLE__ */ #endif /* defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || defined(HAVE_AES_CBC) */ #if !defined(WC_AES_BITSLICED) || defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) #ifdef HAVE_AES_DECRYPT #ifndef __APPLE__ .text .globl AES_decrypt_AARCH64 .type AES_decrypt_AARCH64,@function .align 2 AES_decrypt_AARCH64: #else .section __TEXT,__text .globl _AES_decrypt_AARCH64 .p2align 2 _AES_decrypt_AARCH64: #endif /* __APPLE__ */ ld1 {v0.16b}, [x0] ld1 {v1.2d, v2.2d, v3.2d, v4.2d}, [x2], #0x40 aesd v0.16b, v1.16b aesimc v0.16b, v0.16b aesd v0.16b, v2.16b aesimc v0.16b, v0.16b aesd v0.16b, v3.16b aesimc v0.16b, v0.16b aesd v0.16b, v4.16b aesimc v0.16b, v0.16b ld1 {v1.2d, v2.2d, v3.2d, v4.2d}, [x2], #0x40 aesd v0.16b, v1.16b aesimc v0.16b, v0.16b aesd v0.16b, v2.16b aesimc v0.16b, v0.16b aesd v0.16b, v3.16b aesimc v0.16b, v0.16b aesd v0.16b, v4.16b aesimc v0.16b, v0.16b ld1 {v1.2d, v2.2d}, [x2], #32 aesd v0.16b, v1.16b aesimc v0.16b, v0.16b aesd v0.16b, v2.16b subs w3, w3, #10 beq L_aes_decrypt_arm64_crypto_round_done ld1 {v1.2d, v2.2d}, [x2], #32 aesimc v0.16b, v0.16b aesd v0.16b, v1.16b aesimc v0.16b, v0.16b aesd v0.16b, v2.16b subs w3, w3, #2 beq L_aes_decrypt_arm64_crypto_round_done ld1 {v1.2d, v2.2d}, [x2], #32 aesimc v0.16b, v0.16b aesd v0.16b, v1.16b aesimc v0.16b, v0.16b aesd v0.16b, v2.16b L_aes_decrypt_arm64_crypto_round_done: ld1 {v1.2d}, [x2] eor v0.16b, v0.16b, v1.16b st1 {v0.16b}, [x1] ret #ifndef __APPLE__ .size AES_decrypt_AARCH64,.-AES_decrypt_AARCH64 #endif /* __APPLE__ */ #endif /* HAVE_AES_DECRYPT */ #endif /* !defined(WC_AES_BITSLICED) || defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) */ #ifdef HAVE_AES_ECB #ifndef __APPLE__ .text .globl AES_encrypt_blocks_AARCH64 .type AES_encrypt_blocks_AARCH64,@function .align 2 AES_encrypt_blocks_AARCH64: #else .section __TEXT,__text .globl _AES_encrypt_blocks_AARCH64 .p2align 2 _AES_encrypt_blocks_AARCH64: #endif /* __APPLE__ */ stp x29, x30, [sp, #-32]! add x29, sp, #0 ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x3], #0x40 ld1 {v24.2d, v25.2d, v26.2d}, [x3], #48 lsr w2, w2, #4 cmp w4, #12 blt L_aes_encrypt_blocks_arm64_crypto_start_128 bgt L_aes_encrypt_blocks_arm64_crypto_start_256 # AES_ECB_192 #ifndef NO_AES_192 ld1 {v27.2d, v28.2d}, [x3], #32 cmp w2, #1 beq L_aes_encrypt_blocks_arm64_crypto_192_start_1 cmp w2, #8 blt L_aes_encrypt_blocks_arm64_crypto_192_start_4 L_aes_encrypt_blocks_arm64_crypto_192_start_8: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v2.16b, v16.16b aesmc v2.16b, v2.16b aese v3.16b, v16.16b aesmc v3.16b, v3.16b aese v4.16b, v16.16b aesmc v4.16b, v4.16b aese v5.16b, v16.16b aesmc v5.16b, v5.16b aese v6.16b, v16.16b aesmc v6.16b, v6.16b aese v7.16b, v16.16b aesmc v7.16b, v7.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v2.16b, v17.16b aesmc v2.16b, v2.16b aese v3.16b, v17.16b aesmc v3.16b, v3.16b aese v4.16b, v17.16b aesmc v4.16b, v4.16b aese v5.16b, v17.16b aesmc v5.16b, v5.16b aese v6.16b, v17.16b aesmc v6.16b, v6.16b aese v7.16b, v17.16b aesmc v7.16b, v7.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v2.16b, v18.16b aesmc v2.16b, v2.16b aese v3.16b, v18.16b aesmc v3.16b, v3.16b aese v4.16b, v18.16b aesmc v4.16b, v4.16b aese v5.16b, v18.16b aesmc v5.16b, v5.16b aese v6.16b, v18.16b aesmc v6.16b, v6.16b aese v7.16b, v18.16b aesmc v7.16b, v7.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v2.16b, v19.16b aesmc v2.16b, v2.16b aese v3.16b, v19.16b aesmc v3.16b, v3.16b aese v4.16b, v19.16b aesmc v4.16b, v4.16b aese v5.16b, v19.16b aesmc v5.16b, v5.16b aese v6.16b, v19.16b aesmc v6.16b, v6.16b aese v7.16b, v19.16b aesmc v7.16b, v7.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v2.16b, v20.16b aesmc v2.16b, v2.16b aese v3.16b, v20.16b aesmc v3.16b, v3.16b aese v4.16b, v20.16b aesmc v4.16b, v4.16b aese v5.16b, v20.16b aesmc v5.16b, v5.16b aese v6.16b, v20.16b aesmc v6.16b, v6.16b aese v7.16b, v20.16b aesmc v7.16b, v7.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v2.16b, v21.16b aesmc v2.16b, v2.16b aese v3.16b, v21.16b aesmc v3.16b, v3.16b aese v4.16b, v21.16b aesmc v4.16b, v4.16b aese v5.16b, v21.16b aesmc v5.16b, v5.16b aese v6.16b, v21.16b aesmc v6.16b, v6.16b aese v7.16b, v21.16b aesmc v7.16b, v7.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v2.16b, v22.16b aesmc v2.16b, v2.16b aese v3.16b, v22.16b aesmc v3.16b, v3.16b aese v4.16b, v22.16b aesmc v4.16b, v4.16b aese v5.16b, v22.16b aesmc v5.16b, v5.16b aese v6.16b, v22.16b aesmc v6.16b, v6.16b aese v7.16b, v22.16b aesmc v7.16b, v7.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v2.16b, v23.16b aesmc v2.16b, v2.16b aese v3.16b, v23.16b aesmc v3.16b, v3.16b aese v4.16b, v23.16b aesmc v4.16b, v4.16b aese v5.16b, v23.16b aesmc v5.16b, v5.16b aese v6.16b, v23.16b aesmc v6.16b, v6.16b aese v7.16b, v23.16b aesmc v7.16b, v7.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v2.16b, v24.16b aesmc v2.16b, v2.16b aese v3.16b, v24.16b aesmc v3.16b, v3.16b aese v4.16b, v24.16b aesmc v4.16b, v4.16b aese v5.16b, v24.16b aesmc v5.16b, v5.16b aese v6.16b, v24.16b aesmc v6.16b, v6.16b aese v7.16b, v24.16b aesmc v7.16b, v7.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v1.16b, v25.16b aesmc v1.16b, v1.16b aese v2.16b, v25.16b aesmc v2.16b, v2.16b aese v3.16b, v25.16b aesmc v3.16b, v3.16b aese v4.16b, v25.16b aesmc v4.16b, v4.16b aese v5.16b, v25.16b aesmc v5.16b, v5.16b aese v6.16b, v25.16b aesmc v6.16b, v6.16b aese v7.16b, v25.16b aesmc v7.16b, v7.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v1.16b, v26.16b aesmc v1.16b, v1.16b aese v2.16b, v26.16b aesmc v2.16b, v2.16b aese v3.16b, v26.16b aesmc v3.16b, v3.16b aese v4.16b, v26.16b aesmc v4.16b, v4.16b aese v5.16b, v26.16b aesmc v5.16b, v5.16b aese v6.16b, v26.16b aesmc v6.16b, v6.16b aese v7.16b, v26.16b aesmc v7.16b, v7.16b aese v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b aese v1.16b, v27.16b eor v1.16b, v1.16b, v28.16b aese v2.16b, v27.16b eor v2.16b, v2.16b, v28.16b aese v3.16b, v27.16b eor v3.16b, v3.16b, v28.16b aese v4.16b, v27.16b eor v4.16b, v4.16b, v28.16b aese v5.16b, v27.16b eor v5.16b, v5.16b, v28.16b aese v6.16b, v27.16b eor v6.16b, v6.16b, v28.16b aese v7.16b, v27.16b eor v7.16b, v7.16b, v28.16b sub w2, w2, #8 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 cmp w2, #8 bge L_aes_encrypt_blocks_arm64_crypto_192_start_8 L_aes_encrypt_blocks_arm64_crypto_192_start_4: cmp w2, #4 blt L_aes_encrypt_blocks_arm64_crypto_192_start_2 ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v2.16b, v16.16b aesmc v2.16b, v2.16b aese v3.16b, v16.16b aesmc v3.16b, v3.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v2.16b, v17.16b aesmc v2.16b, v2.16b aese v3.16b, v17.16b aesmc v3.16b, v3.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v2.16b, v18.16b aesmc v2.16b, v2.16b aese v3.16b, v18.16b aesmc v3.16b, v3.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v2.16b, v19.16b aesmc v2.16b, v2.16b aese v3.16b, v19.16b aesmc v3.16b, v3.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v2.16b, v20.16b aesmc v2.16b, v2.16b aese v3.16b, v20.16b aesmc v3.16b, v3.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v2.16b, v21.16b aesmc v2.16b, v2.16b aese v3.16b, v21.16b aesmc v3.16b, v3.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v2.16b, v22.16b aesmc v2.16b, v2.16b aese v3.16b, v22.16b aesmc v3.16b, v3.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v2.16b, v23.16b aesmc v2.16b, v2.16b aese v3.16b, v23.16b aesmc v3.16b, v3.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v2.16b, v24.16b aesmc v2.16b, v2.16b aese v3.16b, v24.16b aesmc v3.16b, v3.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v1.16b, v25.16b aesmc v1.16b, v1.16b aese v2.16b, v25.16b aesmc v2.16b, v2.16b aese v3.16b, v25.16b aesmc v3.16b, v3.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v1.16b, v26.16b aesmc v1.16b, v1.16b aese v2.16b, v26.16b aesmc v2.16b, v2.16b aese v3.16b, v26.16b aesmc v3.16b, v3.16b aese v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b aese v1.16b, v27.16b eor v1.16b, v1.16b, v28.16b aese v2.16b, v27.16b eor v2.16b, v2.16b, v28.16b aese v3.16b, v27.16b eor v3.16b, v3.16b, v28.16b sub w2, w2, #4 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 L_aes_encrypt_blocks_arm64_crypto_192_start_2: cmp w2, #2 blt L_aes_encrypt_blocks_arm64_crypto_192_start_1 ld1 {v0.16b, v1.16b}, [x0], #32 aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v1.16b, v25.16b aesmc v1.16b, v1.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v1.16b, v26.16b aesmc v1.16b, v1.16b aese v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b aese v1.16b, v27.16b eor v1.16b, v1.16b, v28.16b sub w2, w2, #2 st1 {v0.16b, v1.16b}, [x1], #32 L_aes_encrypt_blocks_arm64_crypto_192_start_1: cbz w2, L_aes_encrypt_blocks_arm64_crypto_192_done ld1 {v0.16b}, [x0], #16 aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b st1 {v0.16b}, [x1], #16 L_aes_encrypt_blocks_arm64_crypto_192_done: #endif /* !NO_AES_192 */ b L_aes_encrypt_blocks_arm64_crypto_done # AES_ECB_256 L_aes_encrypt_blocks_arm64_crypto_start_256: #ifndef NO_AES_256 ld1 {v27.2d, v28.2d, v29.2d, v30.2d}, [x3], #0x40 cmp w2, #1 beq L_aes_encrypt_blocks_arm64_crypto_256_start_1 cmp w2, #8 blt L_aes_encrypt_blocks_arm64_crypto_256_start_4 L_aes_encrypt_blocks_arm64_crypto_256_start_8: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v2.16b, v16.16b aesmc v2.16b, v2.16b aese v3.16b, v16.16b aesmc v3.16b, v3.16b aese v4.16b, v16.16b aesmc v4.16b, v4.16b aese v5.16b, v16.16b aesmc v5.16b, v5.16b aese v6.16b, v16.16b aesmc v6.16b, v6.16b aese v7.16b, v16.16b aesmc v7.16b, v7.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v2.16b, v17.16b aesmc v2.16b, v2.16b aese v3.16b, v17.16b aesmc v3.16b, v3.16b aese v4.16b, v17.16b aesmc v4.16b, v4.16b aese v5.16b, v17.16b aesmc v5.16b, v5.16b aese v6.16b, v17.16b aesmc v6.16b, v6.16b aese v7.16b, v17.16b aesmc v7.16b, v7.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v2.16b, v18.16b aesmc v2.16b, v2.16b aese v3.16b, v18.16b aesmc v3.16b, v3.16b aese v4.16b, v18.16b aesmc v4.16b, v4.16b aese v5.16b, v18.16b aesmc v5.16b, v5.16b aese v6.16b, v18.16b aesmc v6.16b, v6.16b aese v7.16b, v18.16b aesmc v7.16b, v7.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v2.16b, v19.16b aesmc v2.16b, v2.16b aese v3.16b, v19.16b aesmc v3.16b, v3.16b aese v4.16b, v19.16b aesmc v4.16b, v4.16b aese v5.16b, v19.16b aesmc v5.16b, v5.16b aese v6.16b, v19.16b aesmc v6.16b, v6.16b aese v7.16b, v19.16b aesmc v7.16b, v7.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v2.16b, v20.16b aesmc v2.16b, v2.16b aese v3.16b, v20.16b aesmc v3.16b, v3.16b aese v4.16b, v20.16b aesmc v4.16b, v4.16b aese v5.16b, v20.16b aesmc v5.16b, v5.16b aese v6.16b, v20.16b aesmc v6.16b, v6.16b aese v7.16b, v20.16b aesmc v7.16b, v7.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v2.16b, v21.16b aesmc v2.16b, v2.16b aese v3.16b, v21.16b aesmc v3.16b, v3.16b aese v4.16b, v21.16b aesmc v4.16b, v4.16b aese v5.16b, v21.16b aesmc v5.16b, v5.16b aese v6.16b, v21.16b aesmc v6.16b, v6.16b aese v7.16b, v21.16b aesmc v7.16b, v7.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v2.16b, v22.16b aesmc v2.16b, v2.16b aese v3.16b, v22.16b aesmc v3.16b, v3.16b aese v4.16b, v22.16b aesmc v4.16b, v4.16b aese v5.16b, v22.16b aesmc v5.16b, v5.16b aese v6.16b, v22.16b aesmc v6.16b, v6.16b aese v7.16b, v22.16b aesmc v7.16b, v7.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v2.16b, v23.16b aesmc v2.16b, v2.16b aese v3.16b, v23.16b aesmc v3.16b, v3.16b aese v4.16b, v23.16b aesmc v4.16b, v4.16b aese v5.16b, v23.16b aesmc v5.16b, v5.16b aese v6.16b, v23.16b aesmc v6.16b, v6.16b aese v7.16b, v23.16b aesmc v7.16b, v7.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v2.16b, v24.16b aesmc v2.16b, v2.16b aese v3.16b, v24.16b aesmc v3.16b, v3.16b aese v4.16b, v24.16b aesmc v4.16b, v4.16b aese v5.16b, v24.16b aesmc v5.16b, v5.16b aese v6.16b, v24.16b aesmc v6.16b, v6.16b aese v7.16b, v24.16b aesmc v7.16b, v7.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v1.16b, v25.16b aesmc v1.16b, v1.16b aese v2.16b, v25.16b aesmc v2.16b, v2.16b aese v3.16b, v25.16b aesmc v3.16b, v3.16b aese v4.16b, v25.16b aesmc v4.16b, v4.16b aese v5.16b, v25.16b aesmc v5.16b, v5.16b aese v6.16b, v25.16b aesmc v6.16b, v6.16b aese v7.16b, v25.16b aesmc v7.16b, v7.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v1.16b, v26.16b aesmc v1.16b, v1.16b aese v2.16b, v26.16b aesmc v2.16b, v2.16b aese v3.16b, v26.16b aesmc v3.16b, v3.16b aese v4.16b, v26.16b aesmc v4.16b, v4.16b aese v5.16b, v26.16b aesmc v5.16b, v5.16b aese v6.16b, v26.16b aesmc v6.16b, v6.16b aese v7.16b, v26.16b aesmc v7.16b, v7.16b aese v0.16b, v27.16b aesmc v0.16b, v0.16b aese v1.16b, v27.16b aesmc v1.16b, v1.16b aese v2.16b, v27.16b aesmc v2.16b, v2.16b aese v3.16b, v27.16b aesmc v3.16b, v3.16b aese v4.16b, v27.16b aesmc v4.16b, v4.16b aese v5.16b, v27.16b aesmc v5.16b, v5.16b aese v6.16b, v27.16b aesmc v6.16b, v6.16b aese v7.16b, v27.16b aesmc v7.16b, v7.16b aese v0.16b, v28.16b aesmc v0.16b, v0.16b aese v1.16b, v28.16b aesmc v1.16b, v1.16b aese v2.16b, v28.16b aesmc v2.16b, v2.16b aese v3.16b, v28.16b aesmc v3.16b, v3.16b aese v4.16b, v28.16b aesmc v4.16b, v4.16b aese v5.16b, v28.16b aesmc v5.16b, v5.16b aese v6.16b, v28.16b aesmc v6.16b, v6.16b aese v7.16b, v28.16b aesmc v7.16b, v7.16b aese v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b aese v1.16b, v29.16b eor v1.16b, v1.16b, v30.16b aese v2.16b, v29.16b eor v2.16b, v2.16b, v30.16b aese v3.16b, v29.16b eor v3.16b, v3.16b, v30.16b aese v4.16b, v29.16b eor v4.16b, v4.16b, v30.16b aese v5.16b, v29.16b eor v5.16b, v5.16b, v30.16b aese v6.16b, v29.16b eor v6.16b, v6.16b, v30.16b aese v7.16b, v29.16b eor v7.16b, v7.16b, v30.16b sub w2, w2, #8 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 cmp w2, #8 bge L_aes_encrypt_blocks_arm64_crypto_256_start_8 L_aes_encrypt_blocks_arm64_crypto_256_start_4: cmp w2, #4 blt L_aes_encrypt_blocks_arm64_crypto_256_start_2 ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v2.16b, v16.16b aesmc v2.16b, v2.16b aese v3.16b, v16.16b aesmc v3.16b, v3.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v2.16b, v17.16b aesmc v2.16b, v2.16b aese v3.16b, v17.16b aesmc v3.16b, v3.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v2.16b, v18.16b aesmc v2.16b, v2.16b aese v3.16b, v18.16b aesmc v3.16b, v3.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v2.16b, v19.16b aesmc v2.16b, v2.16b aese v3.16b, v19.16b aesmc v3.16b, v3.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v2.16b, v20.16b aesmc v2.16b, v2.16b aese v3.16b, v20.16b aesmc v3.16b, v3.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v2.16b, v21.16b aesmc v2.16b, v2.16b aese v3.16b, v21.16b aesmc v3.16b, v3.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v2.16b, v22.16b aesmc v2.16b, v2.16b aese v3.16b, v22.16b aesmc v3.16b, v3.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v2.16b, v23.16b aesmc v2.16b, v2.16b aese v3.16b, v23.16b aesmc v3.16b, v3.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v2.16b, v24.16b aesmc v2.16b, v2.16b aese v3.16b, v24.16b aesmc v3.16b, v3.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v1.16b, v25.16b aesmc v1.16b, v1.16b aese v2.16b, v25.16b aesmc v2.16b, v2.16b aese v3.16b, v25.16b aesmc v3.16b, v3.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v1.16b, v26.16b aesmc v1.16b, v1.16b aese v2.16b, v26.16b aesmc v2.16b, v2.16b aese v3.16b, v26.16b aesmc v3.16b, v3.16b aese v0.16b, v27.16b aesmc v0.16b, v0.16b aese v1.16b, v27.16b aesmc v1.16b, v1.16b aese v2.16b, v27.16b aesmc v2.16b, v2.16b aese v3.16b, v27.16b aesmc v3.16b, v3.16b aese v0.16b, v28.16b aesmc v0.16b, v0.16b aese v1.16b, v28.16b aesmc v1.16b, v1.16b aese v2.16b, v28.16b aesmc v2.16b, v2.16b aese v3.16b, v28.16b aesmc v3.16b, v3.16b aese v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b aese v1.16b, v29.16b eor v1.16b, v1.16b, v30.16b aese v2.16b, v29.16b eor v2.16b, v2.16b, v30.16b aese v3.16b, v29.16b eor v3.16b, v3.16b, v30.16b sub w2, w2, #4 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 L_aes_encrypt_blocks_arm64_crypto_256_start_2: cmp w2, #2 blt L_aes_encrypt_blocks_arm64_crypto_256_start_1 ld1 {v0.16b, v1.16b}, [x0], #32 aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v1.16b, v25.16b aesmc v1.16b, v1.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v1.16b, v26.16b aesmc v1.16b, v1.16b aese v0.16b, v27.16b aesmc v0.16b, v0.16b aese v1.16b, v27.16b aesmc v1.16b, v1.16b aese v0.16b, v28.16b aesmc v0.16b, v0.16b aese v1.16b, v28.16b aesmc v1.16b, v1.16b aese v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b aese v1.16b, v29.16b eor v1.16b, v1.16b, v30.16b sub w2, w2, #2 st1 {v0.16b, v1.16b}, [x1], #32 L_aes_encrypt_blocks_arm64_crypto_256_start_1: cbz w2, L_aes_encrypt_blocks_arm64_crypto_256_done ld1 {v0.16b}, [x0], #16 aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v0.16b, v27.16b aesmc v0.16b, v0.16b aese v0.16b, v28.16b aesmc v0.16b, v0.16b aese v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b st1 {v0.16b}, [x1], #16 L_aes_encrypt_blocks_arm64_crypto_256_done: #endif /* !NO_AES_256 */ b L_aes_encrypt_blocks_arm64_crypto_done # AES_ECB_128 L_aes_encrypt_blocks_arm64_crypto_start_128: #ifndef NO_AES_128 cmp w2, #1 beq L_aes_encrypt_blocks_arm64_crypto_128_start_1 cmp w2, #8 blt L_aes_encrypt_blocks_arm64_crypto_128_start_4 L_aes_encrypt_blocks_arm64_crypto_128_start_8: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v2.16b, v16.16b aesmc v2.16b, v2.16b aese v3.16b, v16.16b aesmc v3.16b, v3.16b aese v4.16b, v16.16b aesmc v4.16b, v4.16b aese v5.16b, v16.16b aesmc v5.16b, v5.16b aese v6.16b, v16.16b aesmc v6.16b, v6.16b aese v7.16b, v16.16b aesmc v7.16b, v7.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v2.16b, v17.16b aesmc v2.16b, v2.16b aese v3.16b, v17.16b aesmc v3.16b, v3.16b aese v4.16b, v17.16b aesmc v4.16b, v4.16b aese v5.16b, v17.16b aesmc v5.16b, v5.16b aese v6.16b, v17.16b aesmc v6.16b, v6.16b aese v7.16b, v17.16b aesmc v7.16b, v7.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v2.16b, v18.16b aesmc v2.16b, v2.16b aese v3.16b, v18.16b aesmc v3.16b, v3.16b aese v4.16b, v18.16b aesmc v4.16b, v4.16b aese v5.16b, v18.16b aesmc v5.16b, v5.16b aese v6.16b, v18.16b aesmc v6.16b, v6.16b aese v7.16b, v18.16b aesmc v7.16b, v7.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v2.16b, v19.16b aesmc v2.16b, v2.16b aese v3.16b, v19.16b aesmc v3.16b, v3.16b aese v4.16b, v19.16b aesmc v4.16b, v4.16b aese v5.16b, v19.16b aesmc v5.16b, v5.16b aese v6.16b, v19.16b aesmc v6.16b, v6.16b aese v7.16b, v19.16b aesmc v7.16b, v7.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v2.16b, v20.16b aesmc v2.16b, v2.16b aese v3.16b, v20.16b aesmc v3.16b, v3.16b aese v4.16b, v20.16b aesmc v4.16b, v4.16b aese v5.16b, v20.16b aesmc v5.16b, v5.16b aese v6.16b, v20.16b aesmc v6.16b, v6.16b aese v7.16b, v20.16b aesmc v7.16b, v7.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v2.16b, v21.16b aesmc v2.16b, v2.16b aese v3.16b, v21.16b aesmc v3.16b, v3.16b aese v4.16b, v21.16b aesmc v4.16b, v4.16b aese v5.16b, v21.16b aesmc v5.16b, v5.16b aese v6.16b, v21.16b aesmc v6.16b, v6.16b aese v7.16b, v21.16b aesmc v7.16b, v7.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v2.16b, v22.16b aesmc v2.16b, v2.16b aese v3.16b, v22.16b aesmc v3.16b, v3.16b aese v4.16b, v22.16b aesmc v4.16b, v4.16b aese v5.16b, v22.16b aesmc v5.16b, v5.16b aese v6.16b, v22.16b aesmc v6.16b, v6.16b aese v7.16b, v22.16b aesmc v7.16b, v7.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v2.16b, v23.16b aesmc v2.16b, v2.16b aese v3.16b, v23.16b aesmc v3.16b, v3.16b aese v4.16b, v23.16b aesmc v4.16b, v4.16b aese v5.16b, v23.16b aesmc v5.16b, v5.16b aese v6.16b, v23.16b aesmc v6.16b, v6.16b aese v7.16b, v23.16b aesmc v7.16b, v7.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v2.16b, v24.16b aesmc v2.16b, v2.16b aese v3.16b, v24.16b aesmc v3.16b, v3.16b aese v4.16b, v24.16b aesmc v4.16b, v4.16b aese v5.16b, v24.16b aesmc v5.16b, v5.16b aese v6.16b, v24.16b aesmc v6.16b, v6.16b aese v7.16b, v24.16b aesmc v7.16b, v7.16b aese v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b aese v1.16b, v25.16b eor v1.16b, v1.16b, v26.16b aese v2.16b, v25.16b eor v2.16b, v2.16b, v26.16b aese v3.16b, v25.16b eor v3.16b, v3.16b, v26.16b aese v4.16b, v25.16b eor v4.16b, v4.16b, v26.16b aese v5.16b, v25.16b eor v5.16b, v5.16b, v26.16b aese v6.16b, v25.16b eor v6.16b, v6.16b, v26.16b aese v7.16b, v25.16b eor v7.16b, v7.16b, v26.16b sub w2, w2, #8 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 cmp w2, #8 bge L_aes_encrypt_blocks_arm64_crypto_128_start_8 L_aes_encrypt_blocks_arm64_crypto_128_start_4: cmp w2, #4 blt L_aes_encrypt_blocks_arm64_crypto_128_start_2 ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v2.16b, v16.16b aesmc v2.16b, v2.16b aese v3.16b, v16.16b aesmc v3.16b, v3.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v2.16b, v17.16b aesmc v2.16b, v2.16b aese v3.16b, v17.16b aesmc v3.16b, v3.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v2.16b, v18.16b aesmc v2.16b, v2.16b aese v3.16b, v18.16b aesmc v3.16b, v3.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v2.16b, v19.16b aesmc v2.16b, v2.16b aese v3.16b, v19.16b aesmc v3.16b, v3.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v2.16b, v20.16b aesmc v2.16b, v2.16b aese v3.16b, v20.16b aesmc v3.16b, v3.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v2.16b, v21.16b aesmc v2.16b, v2.16b aese v3.16b, v21.16b aesmc v3.16b, v3.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v2.16b, v22.16b aesmc v2.16b, v2.16b aese v3.16b, v22.16b aesmc v3.16b, v3.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v2.16b, v23.16b aesmc v2.16b, v2.16b aese v3.16b, v23.16b aesmc v3.16b, v3.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v2.16b, v24.16b aesmc v2.16b, v2.16b aese v3.16b, v24.16b aesmc v3.16b, v3.16b aese v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b aese v1.16b, v25.16b eor v1.16b, v1.16b, v26.16b aese v2.16b, v25.16b eor v2.16b, v2.16b, v26.16b aese v3.16b, v25.16b eor v3.16b, v3.16b, v26.16b sub w2, w2, #4 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 L_aes_encrypt_blocks_arm64_crypto_128_start_2: cmp w2, #2 blt L_aes_encrypt_blocks_arm64_crypto_128_start_1 ld1 {v0.16b, v1.16b}, [x0], #32 aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b aese v1.16b, v25.16b eor v1.16b, v1.16b, v26.16b sub w2, w2, #2 st1 {v0.16b, v1.16b}, [x1], #32 L_aes_encrypt_blocks_arm64_crypto_128_start_1: cbz w2, L_aes_encrypt_blocks_arm64_crypto_128_done ld1 {v0.16b}, [x0], #16 aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b st1 {v0.16b}, [x1], #16 L_aes_encrypt_blocks_arm64_crypto_128_done: #endif /* !NO_AES_128 */ L_aes_encrypt_blocks_arm64_crypto_done: ldp x29, x30, [sp], #32 ret #ifndef __APPLE__ .size AES_encrypt_blocks_AARCH64,.-AES_encrypt_blocks_AARCH64 #endif /* __APPLE__ */ #ifdef HAVE_AES_DECRYPT #ifndef __APPLE__ .text .globl AES_decrypt_blocks_AARCH64 .type AES_decrypt_blocks_AARCH64,@function .align 2 AES_decrypt_blocks_AARCH64: #else .section __TEXT,__text .globl _AES_decrypt_blocks_AARCH64 .p2align 2 _AES_decrypt_blocks_AARCH64: #endif /* __APPLE__ */ stp x29, x30, [sp, #-32]! add x29, sp, #0 ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x3], #0x40 ld1 {v24.2d, v25.2d, v26.2d}, [x3], #48 lsr w2, w2, #4 cmp w4, #12 blt L_aes_decrypt_blocks_arm64_crypto_start_128 bgt L_aes_decrypt_blocks_arm64_crypto_start_256 # AES_ECB_192 #ifndef NO_AES_192 ld1 {v27.2d, v28.2d}, [x3], #32 cmp w2, #1 beq L_aes_decrypt_blocks_arm64_crypto_192_start_1 cmp w2, #8 blt L_aes_decrypt_blocks_arm64_crypto_192_start_4 L_aes_decrypt_blocks_arm64_crypto_192_start_8: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v2.16b, v16.16b aesimc v2.16b, v2.16b aesd v3.16b, v16.16b aesimc v3.16b, v3.16b aesd v4.16b, v16.16b aesimc v4.16b, v4.16b aesd v5.16b, v16.16b aesimc v5.16b, v5.16b aesd v6.16b, v16.16b aesimc v6.16b, v6.16b aesd v7.16b, v16.16b aesimc v7.16b, v7.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v2.16b, v17.16b aesimc v2.16b, v2.16b aesd v3.16b, v17.16b aesimc v3.16b, v3.16b aesd v4.16b, v17.16b aesimc v4.16b, v4.16b aesd v5.16b, v17.16b aesimc v5.16b, v5.16b aesd v6.16b, v17.16b aesimc v6.16b, v6.16b aesd v7.16b, v17.16b aesimc v7.16b, v7.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v2.16b, v18.16b aesimc v2.16b, v2.16b aesd v3.16b, v18.16b aesimc v3.16b, v3.16b aesd v4.16b, v18.16b aesimc v4.16b, v4.16b aesd v5.16b, v18.16b aesimc v5.16b, v5.16b aesd v6.16b, v18.16b aesimc v6.16b, v6.16b aesd v7.16b, v18.16b aesimc v7.16b, v7.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v2.16b, v19.16b aesimc v2.16b, v2.16b aesd v3.16b, v19.16b aesimc v3.16b, v3.16b aesd v4.16b, v19.16b aesimc v4.16b, v4.16b aesd v5.16b, v19.16b aesimc v5.16b, v5.16b aesd v6.16b, v19.16b aesimc v6.16b, v6.16b aesd v7.16b, v19.16b aesimc v7.16b, v7.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v2.16b, v20.16b aesimc v2.16b, v2.16b aesd v3.16b, v20.16b aesimc v3.16b, v3.16b aesd v4.16b, v20.16b aesimc v4.16b, v4.16b aesd v5.16b, v20.16b aesimc v5.16b, v5.16b aesd v6.16b, v20.16b aesimc v6.16b, v6.16b aesd v7.16b, v20.16b aesimc v7.16b, v7.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v2.16b, v21.16b aesimc v2.16b, v2.16b aesd v3.16b, v21.16b aesimc v3.16b, v3.16b aesd v4.16b, v21.16b aesimc v4.16b, v4.16b aesd v5.16b, v21.16b aesimc v5.16b, v5.16b aesd v6.16b, v21.16b aesimc v6.16b, v6.16b aesd v7.16b, v21.16b aesimc v7.16b, v7.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v2.16b, v22.16b aesimc v2.16b, v2.16b aesd v3.16b, v22.16b aesimc v3.16b, v3.16b aesd v4.16b, v22.16b aesimc v4.16b, v4.16b aesd v5.16b, v22.16b aesimc v5.16b, v5.16b aesd v6.16b, v22.16b aesimc v6.16b, v6.16b aesd v7.16b, v22.16b aesimc v7.16b, v7.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v2.16b, v23.16b aesimc v2.16b, v2.16b aesd v3.16b, v23.16b aesimc v3.16b, v3.16b aesd v4.16b, v23.16b aesimc v4.16b, v4.16b aesd v5.16b, v23.16b aesimc v5.16b, v5.16b aesd v6.16b, v23.16b aesimc v6.16b, v6.16b aesd v7.16b, v23.16b aesimc v7.16b, v7.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v2.16b, v24.16b aesimc v2.16b, v2.16b aesd v3.16b, v24.16b aesimc v3.16b, v3.16b aesd v4.16b, v24.16b aesimc v4.16b, v4.16b aesd v5.16b, v24.16b aesimc v5.16b, v5.16b aesd v6.16b, v24.16b aesimc v6.16b, v6.16b aesd v7.16b, v24.16b aesimc v7.16b, v7.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v1.16b, v25.16b aesimc v1.16b, v1.16b aesd v2.16b, v25.16b aesimc v2.16b, v2.16b aesd v3.16b, v25.16b aesimc v3.16b, v3.16b aesd v4.16b, v25.16b aesimc v4.16b, v4.16b aesd v5.16b, v25.16b aesimc v5.16b, v5.16b aesd v6.16b, v25.16b aesimc v6.16b, v6.16b aesd v7.16b, v25.16b aesimc v7.16b, v7.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v1.16b, v26.16b aesimc v1.16b, v1.16b aesd v2.16b, v26.16b aesimc v2.16b, v2.16b aesd v3.16b, v26.16b aesimc v3.16b, v3.16b aesd v4.16b, v26.16b aesimc v4.16b, v4.16b aesd v5.16b, v26.16b aesimc v5.16b, v5.16b aesd v6.16b, v26.16b aesimc v6.16b, v6.16b aesd v7.16b, v26.16b aesimc v7.16b, v7.16b aesd v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b aesd v1.16b, v27.16b eor v1.16b, v1.16b, v28.16b aesd v2.16b, v27.16b eor v2.16b, v2.16b, v28.16b aesd v3.16b, v27.16b eor v3.16b, v3.16b, v28.16b aesd v4.16b, v27.16b eor v4.16b, v4.16b, v28.16b aesd v5.16b, v27.16b eor v5.16b, v5.16b, v28.16b aesd v6.16b, v27.16b eor v6.16b, v6.16b, v28.16b aesd v7.16b, v27.16b eor v7.16b, v7.16b, v28.16b sub w2, w2, #8 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 cmp w2, #8 bge L_aes_decrypt_blocks_arm64_crypto_192_start_8 L_aes_decrypt_blocks_arm64_crypto_192_start_4: cmp w2, #4 blt L_aes_decrypt_blocks_arm64_crypto_192_start_2 ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v2.16b, v16.16b aesimc v2.16b, v2.16b aesd v3.16b, v16.16b aesimc v3.16b, v3.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v2.16b, v17.16b aesimc v2.16b, v2.16b aesd v3.16b, v17.16b aesimc v3.16b, v3.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v2.16b, v18.16b aesimc v2.16b, v2.16b aesd v3.16b, v18.16b aesimc v3.16b, v3.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v2.16b, v19.16b aesimc v2.16b, v2.16b aesd v3.16b, v19.16b aesimc v3.16b, v3.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v2.16b, v20.16b aesimc v2.16b, v2.16b aesd v3.16b, v20.16b aesimc v3.16b, v3.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v2.16b, v21.16b aesimc v2.16b, v2.16b aesd v3.16b, v21.16b aesimc v3.16b, v3.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v2.16b, v22.16b aesimc v2.16b, v2.16b aesd v3.16b, v22.16b aesimc v3.16b, v3.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v2.16b, v23.16b aesimc v2.16b, v2.16b aesd v3.16b, v23.16b aesimc v3.16b, v3.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v2.16b, v24.16b aesimc v2.16b, v2.16b aesd v3.16b, v24.16b aesimc v3.16b, v3.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v1.16b, v25.16b aesimc v1.16b, v1.16b aesd v2.16b, v25.16b aesimc v2.16b, v2.16b aesd v3.16b, v25.16b aesimc v3.16b, v3.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v1.16b, v26.16b aesimc v1.16b, v1.16b aesd v2.16b, v26.16b aesimc v2.16b, v2.16b aesd v3.16b, v26.16b aesimc v3.16b, v3.16b aesd v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b aesd v1.16b, v27.16b eor v1.16b, v1.16b, v28.16b aesd v2.16b, v27.16b eor v2.16b, v2.16b, v28.16b aesd v3.16b, v27.16b eor v3.16b, v3.16b, v28.16b sub w2, w2, #4 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 L_aes_decrypt_blocks_arm64_crypto_192_start_2: cmp w2, #2 blt L_aes_decrypt_blocks_arm64_crypto_192_start_1 ld1 {v0.16b, v1.16b}, [x0], #32 aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v1.16b, v25.16b aesimc v1.16b, v1.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v1.16b, v26.16b aesimc v1.16b, v1.16b aesd v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b aesd v1.16b, v27.16b eor v1.16b, v1.16b, v28.16b sub w2, w2, #2 st1 {v0.16b, v1.16b}, [x1], #32 L_aes_decrypt_blocks_arm64_crypto_192_start_1: cbz w2, L_aes_decrypt_blocks_arm64_crypto_192_done ld1 {v0.16b}, [x0], #16 aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b st1 {v0.16b}, [x1], #16 L_aes_decrypt_blocks_arm64_crypto_192_done: #endif /* !NO_AES_192 */ b L_aes_decrypt_blocks_arm64_crypto_done # AES_ECB_256 L_aes_decrypt_blocks_arm64_crypto_start_256: #ifndef NO_AES_256 ld1 {v27.2d, v28.2d, v29.2d, v30.2d}, [x3], #0x40 cmp w2, #1 beq L_aes_decrypt_blocks_arm64_crypto_256_start_1 cmp w2, #8 blt L_aes_decrypt_blocks_arm64_crypto_256_start_4 L_aes_decrypt_blocks_arm64_crypto_256_start_8: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v2.16b, v16.16b aesimc v2.16b, v2.16b aesd v3.16b, v16.16b aesimc v3.16b, v3.16b aesd v4.16b, v16.16b aesimc v4.16b, v4.16b aesd v5.16b, v16.16b aesimc v5.16b, v5.16b aesd v6.16b, v16.16b aesimc v6.16b, v6.16b aesd v7.16b, v16.16b aesimc v7.16b, v7.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v2.16b, v17.16b aesimc v2.16b, v2.16b aesd v3.16b, v17.16b aesimc v3.16b, v3.16b aesd v4.16b, v17.16b aesimc v4.16b, v4.16b aesd v5.16b, v17.16b aesimc v5.16b, v5.16b aesd v6.16b, v17.16b aesimc v6.16b, v6.16b aesd v7.16b, v17.16b aesimc v7.16b, v7.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v2.16b, v18.16b aesimc v2.16b, v2.16b aesd v3.16b, v18.16b aesimc v3.16b, v3.16b aesd v4.16b, v18.16b aesimc v4.16b, v4.16b aesd v5.16b, v18.16b aesimc v5.16b, v5.16b aesd v6.16b, v18.16b aesimc v6.16b, v6.16b aesd v7.16b, v18.16b aesimc v7.16b, v7.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v2.16b, v19.16b aesimc v2.16b, v2.16b aesd v3.16b, v19.16b aesimc v3.16b, v3.16b aesd v4.16b, v19.16b aesimc v4.16b, v4.16b aesd v5.16b, v19.16b aesimc v5.16b, v5.16b aesd v6.16b, v19.16b aesimc v6.16b, v6.16b aesd v7.16b, v19.16b aesimc v7.16b, v7.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v2.16b, v20.16b aesimc v2.16b, v2.16b aesd v3.16b, v20.16b aesimc v3.16b, v3.16b aesd v4.16b, v20.16b aesimc v4.16b, v4.16b aesd v5.16b, v20.16b aesimc v5.16b, v5.16b aesd v6.16b, v20.16b aesimc v6.16b, v6.16b aesd v7.16b, v20.16b aesimc v7.16b, v7.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v2.16b, v21.16b aesimc v2.16b, v2.16b aesd v3.16b, v21.16b aesimc v3.16b, v3.16b aesd v4.16b, v21.16b aesimc v4.16b, v4.16b aesd v5.16b, v21.16b aesimc v5.16b, v5.16b aesd v6.16b, v21.16b aesimc v6.16b, v6.16b aesd v7.16b, v21.16b aesimc v7.16b, v7.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v2.16b, v22.16b aesimc v2.16b, v2.16b aesd v3.16b, v22.16b aesimc v3.16b, v3.16b aesd v4.16b, v22.16b aesimc v4.16b, v4.16b aesd v5.16b, v22.16b aesimc v5.16b, v5.16b aesd v6.16b, v22.16b aesimc v6.16b, v6.16b aesd v7.16b, v22.16b aesimc v7.16b, v7.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v2.16b, v23.16b aesimc v2.16b, v2.16b aesd v3.16b, v23.16b aesimc v3.16b, v3.16b aesd v4.16b, v23.16b aesimc v4.16b, v4.16b aesd v5.16b, v23.16b aesimc v5.16b, v5.16b aesd v6.16b, v23.16b aesimc v6.16b, v6.16b aesd v7.16b, v23.16b aesimc v7.16b, v7.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v2.16b, v24.16b aesimc v2.16b, v2.16b aesd v3.16b, v24.16b aesimc v3.16b, v3.16b aesd v4.16b, v24.16b aesimc v4.16b, v4.16b aesd v5.16b, v24.16b aesimc v5.16b, v5.16b aesd v6.16b, v24.16b aesimc v6.16b, v6.16b aesd v7.16b, v24.16b aesimc v7.16b, v7.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v1.16b, v25.16b aesimc v1.16b, v1.16b aesd v2.16b, v25.16b aesimc v2.16b, v2.16b aesd v3.16b, v25.16b aesimc v3.16b, v3.16b aesd v4.16b, v25.16b aesimc v4.16b, v4.16b aesd v5.16b, v25.16b aesimc v5.16b, v5.16b aesd v6.16b, v25.16b aesimc v6.16b, v6.16b aesd v7.16b, v25.16b aesimc v7.16b, v7.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v1.16b, v26.16b aesimc v1.16b, v1.16b aesd v2.16b, v26.16b aesimc v2.16b, v2.16b aesd v3.16b, v26.16b aesimc v3.16b, v3.16b aesd v4.16b, v26.16b aesimc v4.16b, v4.16b aesd v5.16b, v26.16b aesimc v5.16b, v5.16b aesd v6.16b, v26.16b aesimc v6.16b, v6.16b aesd v7.16b, v26.16b aesimc v7.16b, v7.16b aesd v0.16b, v27.16b aesimc v0.16b, v0.16b aesd v1.16b, v27.16b aesimc v1.16b, v1.16b aesd v2.16b, v27.16b aesimc v2.16b, v2.16b aesd v3.16b, v27.16b aesimc v3.16b, v3.16b aesd v4.16b, v27.16b aesimc v4.16b, v4.16b aesd v5.16b, v27.16b aesimc v5.16b, v5.16b aesd v6.16b, v27.16b aesimc v6.16b, v6.16b aesd v7.16b, v27.16b aesimc v7.16b, v7.16b aesd v0.16b, v28.16b aesimc v0.16b, v0.16b aesd v1.16b, v28.16b aesimc v1.16b, v1.16b aesd v2.16b, v28.16b aesimc v2.16b, v2.16b aesd v3.16b, v28.16b aesimc v3.16b, v3.16b aesd v4.16b, v28.16b aesimc v4.16b, v4.16b aesd v5.16b, v28.16b aesimc v5.16b, v5.16b aesd v6.16b, v28.16b aesimc v6.16b, v6.16b aesd v7.16b, v28.16b aesimc v7.16b, v7.16b aesd v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b aesd v1.16b, v29.16b eor v1.16b, v1.16b, v30.16b aesd v2.16b, v29.16b eor v2.16b, v2.16b, v30.16b aesd v3.16b, v29.16b eor v3.16b, v3.16b, v30.16b aesd v4.16b, v29.16b eor v4.16b, v4.16b, v30.16b aesd v5.16b, v29.16b eor v5.16b, v5.16b, v30.16b aesd v6.16b, v29.16b eor v6.16b, v6.16b, v30.16b aesd v7.16b, v29.16b eor v7.16b, v7.16b, v30.16b sub w2, w2, #8 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 cmp w2, #8 bge L_aes_decrypt_blocks_arm64_crypto_256_start_8 L_aes_decrypt_blocks_arm64_crypto_256_start_4: cmp w2, #4 blt L_aes_decrypt_blocks_arm64_crypto_256_start_2 ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v2.16b, v16.16b aesimc v2.16b, v2.16b aesd v3.16b, v16.16b aesimc v3.16b, v3.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v2.16b, v17.16b aesimc v2.16b, v2.16b aesd v3.16b, v17.16b aesimc v3.16b, v3.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v2.16b, v18.16b aesimc v2.16b, v2.16b aesd v3.16b, v18.16b aesimc v3.16b, v3.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v2.16b, v19.16b aesimc v2.16b, v2.16b aesd v3.16b, v19.16b aesimc v3.16b, v3.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v2.16b, v20.16b aesimc v2.16b, v2.16b aesd v3.16b, v20.16b aesimc v3.16b, v3.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v2.16b, v21.16b aesimc v2.16b, v2.16b aesd v3.16b, v21.16b aesimc v3.16b, v3.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v2.16b, v22.16b aesimc v2.16b, v2.16b aesd v3.16b, v22.16b aesimc v3.16b, v3.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v2.16b, v23.16b aesimc v2.16b, v2.16b aesd v3.16b, v23.16b aesimc v3.16b, v3.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v2.16b, v24.16b aesimc v2.16b, v2.16b aesd v3.16b, v24.16b aesimc v3.16b, v3.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v1.16b, v25.16b aesimc v1.16b, v1.16b aesd v2.16b, v25.16b aesimc v2.16b, v2.16b aesd v3.16b, v25.16b aesimc v3.16b, v3.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v1.16b, v26.16b aesimc v1.16b, v1.16b aesd v2.16b, v26.16b aesimc v2.16b, v2.16b aesd v3.16b, v26.16b aesimc v3.16b, v3.16b aesd v0.16b, v27.16b aesimc v0.16b, v0.16b aesd v1.16b, v27.16b aesimc v1.16b, v1.16b aesd v2.16b, v27.16b aesimc v2.16b, v2.16b aesd v3.16b, v27.16b aesimc v3.16b, v3.16b aesd v0.16b, v28.16b aesimc v0.16b, v0.16b aesd v1.16b, v28.16b aesimc v1.16b, v1.16b aesd v2.16b, v28.16b aesimc v2.16b, v2.16b aesd v3.16b, v28.16b aesimc v3.16b, v3.16b aesd v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b aesd v1.16b, v29.16b eor v1.16b, v1.16b, v30.16b aesd v2.16b, v29.16b eor v2.16b, v2.16b, v30.16b aesd v3.16b, v29.16b eor v3.16b, v3.16b, v30.16b sub w2, w2, #4 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 L_aes_decrypt_blocks_arm64_crypto_256_start_2: cmp w2, #2 blt L_aes_decrypt_blocks_arm64_crypto_256_start_1 ld1 {v0.16b, v1.16b}, [x0], #32 aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v1.16b, v25.16b aesimc v1.16b, v1.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v1.16b, v26.16b aesimc v1.16b, v1.16b aesd v0.16b, v27.16b aesimc v0.16b, v0.16b aesd v1.16b, v27.16b aesimc v1.16b, v1.16b aesd v0.16b, v28.16b aesimc v0.16b, v0.16b aesd v1.16b, v28.16b aesimc v1.16b, v1.16b aesd v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b aesd v1.16b, v29.16b eor v1.16b, v1.16b, v30.16b sub w2, w2, #2 st1 {v0.16b, v1.16b}, [x1], #32 L_aes_decrypt_blocks_arm64_crypto_256_start_1: cbz w2, L_aes_decrypt_blocks_arm64_crypto_256_done ld1 {v0.16b}, [x0], #16 aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v0.16b, v27.16b aesimc v0.16b, v0.16b aesd v0.16b, v28.16b aesimc v0.16b, v0.16b aesd v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b st1 {v0.16b}, [x1], #16 L_aes_decrypt_blocks_arm64_crypto_256_done: #endif /* !NO_AES_256 */ b L_aes_decrypt_blocks_arm64_crypto_done # AES_ECB_128 L_aes_decrypt_blocks_arm64_crypto_start_128: #ifndef NO_AES_128 cmp w2, #1 beq L_aes_decrypt_blocks_arm64_crypto_128_start_1 cmp w2, #8 blt L_aes_decrypt_blocks_arm64_crypto_128_start_4 L_aes_decrypt_blocks_arm64_crypto_128_start_8: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v2.16b, v16.16b aesimc v2.16b, v2.16b aesd v3.16b, v16.16b aesimc v3.16b, v3.16b aesd v4.16b, v16.16b aesimc v4.16b, v4.16b aesd v5.16b, v16.16b aesimc v5.16b, v5.16b aesd v6.16b, v16.16b aesimc v6.16b, v6.16b aesd v7.16b, v16.16b aesimc v7.16b, v7.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v2.16b, v17.16b aesimc v2.16b, v2.16b aesd v3.16b, v17.16b aesimc v3.16b, v3.16b aesd v4.16b, v17.16b aesimc v4.16b, v4.16b aesd v5.16b, v17.16b aesimc v5.16b, v5.16b aesd v6.16b, v17.16b aesimc v6.16b, v6.16b aesd v7.16b, v17.16b aesimc v7.16b, v7.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v2.16b, v18.16b aesimc v2.16b, v2.16b aesd v3.16b, v18.16b aesimc v3.16b, v3.16b aesd v4.16b, v18.16b aesimc v4.16b, v4.16b aesd v5.16b, v18.16b aesimc v5.16b, v5.16b aesd v6.16b, v18.16b aesimc v6.16b, v6.16b aesd v7.16b, v18.16b aesimc v7.16b, v7.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v2.16b, v19.16b aesimc v2.16b, v2.16b aesd v3.16b, v19.16b aesimc v3.16b, v3.16b aesd v4.16b, v19.16b aesimc v4.16b, v4.16b aesd v5.16b, v19.16b aesimc v5.16b, v5.16b aesd v6.16b, v19.16b aesimc v6.16b, v6.16b aesd v7.16b, v19.16b aesimc v7.16b, v7.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v2.16b, v20.16b aesimc v2.16b, v2.16b aesd v3.16b, v20.16b aesimc v3.16b, v3.16b aesd v4.16b, v20.16b aesimc v4.16b, v4.16b aesd v5.16b, v20.16b aesimc v5.16b, v5.16b aesd v6.16b, v20.16b aesimc v6.16b, v6.16b aesd v7.16b, v20.16b aesimc v7.16b, v7.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v2.16b, v21.16b aesimc v2.16b, v2.16b aesd v3.16b, v21.16b aesimc v3.16b, v3.16b aesd v4.16b, v21.16b aesimc v4.16b, v4.16b aesd v5.16b, v21.16b aesimc v5.16b, v5.16b aesd v6.16b, v21.16b aesimc v6.16b, v6.16b aesd v7.16b, v21.16b aesimc v7.16b, v7.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v2.16b, v22.16b aesimc v2.16b, v2.16b aesd v3.16b, v22.16b aesimc v3.16b, v3.16b aesd v4.16b, v22.16b aesimc v4.16b, v4.16b aesd v5.16b, v22.16b aesimc v5.16b, v5.16b aesd v6.16b, v22.16b aesimc v6.16b, v6.16b aesd v7.16b, v22.16b aesimc v7.16b, v7.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v2.16b, v23.16b aesimc v2.16b, v2.16b aesd v3.16b, v23.16b aesimc v3.16b, v3.16b aesd v4.16b, v23.16b aesimc v4.16b, v4.16b aesd v5.16b, v23.16b aesimc v5.16b, v5.16b aesd v6.16b, v23.16b aesimc v6.16b, v6.16b aesd v7.16b, v23.16b aesimc v7.16b, v7.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v2.16b, v24.16b aesimc v2.16b, v2.16b aesd v3.16b, v24.16b aesimc v3.16b, v3.16b aesd v4.16b, v24.16b aesimc v4.16b, v4.16b aesd v5.16b, v24.16b aesimc v5.16b, v5.16b aesd v6.16b, v24.16b aesimc v6.16b, v6.16b aesd v7.16b, v24.16b aesimc v7.16b, v7.16b aesd v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b aesd v1.16b, v25.16b eor v1.16b, v1.16b, v26.16b aesd v2.16b, v25.16b eor v2.16b, v2.16b, v26.16b aesd v3.16b, v25.16b eor v3.16b, v3.16b, v26.16b aesd v4.16b, v25.16b eor v4.16b, v4.16b, v26.16b aesd v5.16b, v25.16b eor v5.16b, v5.16b, v26.16b aesd v6.16b, v25.16b eor v6.16b, v6.16b, v26.16b aesd v7.16b, v25.16b eor v7.16b, v7.16b, v26.16b sub w2, w2, #8 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 cmp w2, #8 bge L_aes_decrypt_blocks_arm64_crypto_128_start_8 L_aes_decrypt_blocks_arm64_crypto_128_start_4: cmp w2, #4 blt L_aes_decrypt_blocks_arm64_crypto_128_start_2 ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v2.16b, v16.16b aesimc v2.16b, v2.16b aesd v3.16b, v16.16b aesimc v3.16b, v3.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v2.16b, v17.16b aesimc v2.16b, v2.16b aesd v3.16b, v17.16b aesimc v3.16b, v3.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v2.16b, v18.16b aesimc v2.16b, v2.16b aesd v3.16b, v18.16b aesimc v3.16b, v3.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v2.16b, v19.16b aesimc v2.16b, v2.16b aesd v3.16b, v19.16b aesimc v3.16b, v3.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v2.16b, v20.16b aesimc v2.16b, v2.16b aesd v3.16b, v20.16b aesimc v3.16b, v3.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v2.16b, v21.16b aesimc v2.16b, v2.16b aesd v3.16b, v21.16b aesimc v3.16b, v3.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v2.16b, v22.16b aesimc v2.16b, v2.16b aesd v3.16b, v22.16b aesimc v3.16b, v3.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v2.16b, v23.16b aesimc v2.16b, v2.16b aesd v3.16b, v23.16b aesimc v3.16b, v3.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v2.16b, v24.16b aesimc v2.16b, v2.16b aesd v3.16b, v24.16b aesimc v3.16b, v3.16b aesd v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b aesd v1.16b, v25.16b eor v1.16b, v1.16b, v26.16b aesd v2.16b, v25.16b eor v2.16b, v2.16b, v26.16b aesd v3.16b, v25.16b eor v3.16b, v3.16b, v26.16b sub w2, w2, #4 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 L_aes_decrypt_blocks_arm64_crypto_128_start_2: cmp w2, #2 blt L_aes_decrypt_blocks_arm64_crypto_128_start_1 ld1 {v0.16b, v1.16b}, [x0], #32 aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b aesd v1.16b, v25.16b eor v1.16b, v1.16b, v26.16b sub w2, w2, #2 st1 {v0.16b, v1.16b}, [x1], #32 L_aes_decrypt_blocks_arm64_crypto_128_start_1: cbz w2, L_aes_decrypt_blocks_arm64_crypto_128_done ld1 {v0.16b}, [x0], #16 aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b st1 {v0.16b}, [x1], #16 L_aes_decrypt_blocks_arm64_crypto_128_done: #endif /* !NO_AES_128 */ L_aes_decrypt_blocks_arm64_crypto_done: ldp x29, x30, [sp], #32 ret #ifndef __APPLE__ .size AES_decrypt_blocks_AARCH64,.-AES_decrypt_blocks_AARCH64 #endif /* __APPLE__ */ #endif /* HAVE_AES_DECRYPT */ #endif /* HAVE_AES_ECB */ #ifdef HAVE_AES_CBC #ifndef __APPLE__ .text .globl AES_CBC_encrypt_AARCH64 .type AES_CBC_encrypt_AARCH64,@function .align 2 AES_CBC_encrypt_AARCH64: #else .section __TEXT,__text .globl _AES_CBC_encrypt_AARCH64 .p2align 2 _AES_CBC_encrypt_AARCH64: #endif /* __APPLE__ */ ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 ld1 {v0.2d}, [x3] subs w5, w5, #12 lsr w2, w2, #4 blt L_aes_cbc_encrypt_arm64_crypto_start_128 bgt L_aes_cbc_encrypt_arm64_crypto_start_256 # AES_CBC_192 #ifndef NO_AES_192 ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 L_aes_cbc_encrypt_arm64_crypto_loop_192: ld1 {v28.2d}, [x4] ld1 {v1.16b}, [x0], #16 subs w2, w2, #1 eor v0.16b, v0.16b, v1.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b st1 {v0.16b}, [x1], #16 bne L_aes_cbc_encrypt_arm64_crypto_loop_192 #endif /* !NO_AES_192 */ b L_aes_cbc_encrypt_arm64_crypto_done # AES_CBC_256 L_aes_cbc_encrypt_arm64_crypto_start_256: #ifndef NO_AES_256 ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 ld1 {v28.2d, v29.2d}, [x4], #32 L_aes_cbc_encrypt_arm64_crypto_loop_256: ld1 {v30.2d}, [x4] ld1 {v1.16b}, [x0], #16 subs w2, w2, #1 eor v0.16b, v0.16b, v1.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v0.16b, v27.16b aesmc v0.16b, v0.16b aese v0.16b, v28.16b aesmc v0.16b, v0.16b aese v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b st1 {v0.16b}, [x1], #16 bne L_aes_cbc_encrypt_arm64_crypto_loop_256 #endif /* !NO_AES_256 */ b L_aes_cbc_encrypt_arm64_crypto_done # AES_CBC_128 L_aes_cbc_encrypt_arm64_crypto_start_128: #ifndef NO_AES_128 ld1 {v24.2d, v25.2d}, [x4], #32 L_aes_cbc_encrypt_arm64_crypto_loop_128: ld1 {v26.2d}, [x4] ld1 {v1.16b}, [x0], #16 subs w2, w2, #1 eor v0.16b, v0.16b, v1.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b st1 {v0.16b}, [x1], #16 bne L_aes_cbc_encrypt_arm64_crypto_loop_128 #endif /* !NO_AES_128 */ L_aes_cbc_encrypt_arm64_crypto_done: st1 {v0.2d}, [x3] ret #ifndef __APPLE__ .size AES_CBC_encrypt_AARCH64,.-AES_CBC_encrypt_AARCH64 #endif /* __APPLE__ */ #ifdef HAVE_AES_DECRYPT #ifndef __APPLE__ .text .globl AES_CBC_decrypt_AARCH64 .type AES_CBC_decrypt_AARCH64,@function .align 2 AES_CBC_decrypt_AARCH64: #else .section __TEXT,__text .globl _AES_CBC_decrypt_AARCH64 .p2align 2 _AES_CBC_decrypt_AARCH64: #endif /* __APPLE__ */ ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 ld1 {v0.2d}, [x3] lsr w2, w2, #4 cmp w5, #12 blt L_aes_cbc_decrypt_blocks_arm64_crypto_start_128 bgt L_aes_cbc_decrypt_blocks_arm64_crypto_start_256 # AES_CBC_192 #ifndef NO_AES_192 ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 ld1 {v28.2d}, [x4] cmp w2, #10 ble L_aes_cbc_decrypt_blocks_arm64_crypto_192_start_1 L_aes_cbc_decrypt_blocks_arm64_crypto_192_start_1_long: ld1 {v1.16b}, [x0], #16 sub w2, w2, #1 mov v2.16b, v1.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v1.16b, v25.16b aesimc v1.16b, v1.16b aesd v1.16b, v26.16b aesimc v1.16b, v1.16b aesd v1.16b, v27.16b eor v0.16b, v0.16b, v28.16b eor v1.16b, v1.16b, v0.16b mov v0.16b, v2.16b st1 {v1.16b}, [x1], #16 cmp w2, #1 bge L_aes_cbc_decrypt_blocks_arm64_crypto_192_start_1_long b L_aes_cbc_decrypt_blocks_arm64_crypto_done L_aes_cbc_decrypt_blocks_arm64_crypto_192_start_1: ld1 {v1.16b}, [x0], #16 sub w2, w2, #1 eor v2.16b, v0.16b, v28.16b mov v0.16b, v1.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v1.16b, v25.16b aesimc v1.16b, v1.16b aesd v1.16b, v26.16b aesimc v1.16b, v1.16b aesd v1.16b, v27.16b eor v1.16b, v1.16b, v2.16b st1 {v1.16b}, [x1], #16 cmp w2, #1 bge L_aes_cbc_decrypt_blocks_arm64_crypto_192_start_1 #endif /* !NO_AES_192 */ b L_aes_cbc_decrypt_blocks_arm64_crypto_done # AES_CBC_256 L_aes_cbc_decrypt_blocks_arm64_crypto_start_256: #ifndef NO_AES_256 ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 ld1 {v28.2d, v29.2d}, [x4], #32 ld1 {v30.2d}, [x4] cmp w2, #5 ble L_aes_cbc_decrypt_blocks_arm64_crypto_256_start_1 L_aes_cbc_decrypt_blocks_arm64_crypto_256_start_1_long: ld1 {v1.16b}, [x0], #16 sub w2, w2, #1 mov v2.16b, v1.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v1.16b, v25.16b aesimc v1.16b, v1.16b aesd v1.16b, v26.16b aesimc v1.16b, v1.16b aesd v1.16b, v27.16b aesimc v1.16b, v1.16b aesd v1.16b, v28.16b aesimc v1.16b, v1.16b aesd v1.16b, v29.16b eor v0.16b, v0.16b, v30.16b eor v1.16b, v1.16b, v0.16b mov v0.16b, v2.16b st1 {v1.16b}, [x1], #16 cmp w2, #1 bge L_aes_cbc_decrypt_blocks_arm64_crypto_256_start_1_long b L_aes_cbc_decrypt_blocks_arm64_crypto_done L_aes_cbc_decrypt_blocks_arm64_crypto_256_start_1: ld1 {v1.16b}, [x0], #16 sub w2, w2, #1 eor v2.16b, v0.16b, v30.16b mov v0.16b, v1.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v1.16b, v25.16b aesimc v1.16b, v1.16b aesd v1.16b, v26.16b aesimc v1.16b, v1.16b aesd v1.16b, v27.16b aesimc v1.16b, v1.16b aesd v1.16b, v28.16b aesimc v1.16b, v1.16b aesd v1.16b, v29.16b eor v1.16b, v1.16b, v2.16b st1 {v1.16b}, [x1], #16 cmp w2, #1 bge L_aes_cbc_decrypt_blocks_arm64_crypto_256_start_1 #endif /* !NO_AES_256 */ b L_aes_cbc_decrypt_blocks_arm64_crypto_done # AES_CBC_128 L_aes_cbc_decrypt_blocks_arm64_crypto_start_128: #ifndef NO_AES_128 ld1 {v24.2d, v25.2d}, [x4], #32 ld1 {v26.2d}, [x4] cmp w2, #24 ble L_aes_cbc_decrypt_blocks_arm64_crypto_128_start_1 L_aes_cbc_decrypt_blocks_arm64_crypto_128_start_1_long: ld1 {v1.16b}, [x0], #16 sub w2, w2, #1 mov v2.16b, v1.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v1.16b, v25.16b eor v0.16b, v0.16b, v26.16b eor v1.16b, v1.16b, v0.16b mov v0.16b, v2.16b st1 {v1.16b}, [x1], #16 cmp w2, #1 bge L_aes_cbc_decrypt_blocks_arm64_crypto_128_start_1_long b L_aes_cbc_decrypt_blocks_arm64_crypto_done L_aes_cbc_decrypt_blocks_arm64_crypto_128_start_1: ld1 {v1.16b}, [x0], #16 sub w2, w2, #1 eor v2.16b, v0.16b, v26.16b mov v0.16b, v1.16b aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v1.16b, v25.16b eor v1.16b, v1.16b, v2.16b st1 {v1.16b}, [x1], #16 cmp w2, #1 bge L_aes_cbc_decrypt_blocks_arm64_crypto_128_start_1 #endif /* !NO_AES_128 */ L_aes_cbc_decrypt_blocks_arm64_crypto_done: st1 {v0.2d}, [x3] ret #ifndef __APPLE__ .size AES_CBC_decrypt_AARCH64,.-AES_CBC_decrypt_AARCH64 #endif /* __APPLE__ */ #endif /* HAVE_AES_DECRYPT */ #endif /* HAVE_AES_CBC */ #ifdef WOLFSSL_AES_COUNTER #ifndef __APPLE__ .text .globl AES_CTR_encrypt_AARCH64 .type AES_CTR_encrypt_AARCH64,@function .align 2 AES_CTR_encrypt_AARCH64: #else .section __TEXT,__text .globl _AES_CTR_encrypt_AARCH64 .p2align 2 _AES_CTR_encrypt_AARCH64: #endif /* __APPLE__ */ stp x29, x30, [sp, #-144]! add x29, sp, #0 stp x17, x19, [x29, #16] stp x20, x21, [x29, #32] stp x22, x23, [x29, #48] stp x24, x25, [x29, #64] stp d8, d9, [x29, #80] stp d10, d11, [x29, #96] stp d12, d13, [x29, #112] stp d14, d15, [x29, #128] ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x4], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x4], #0x40 ld1 {v15.2d}, [x3] rev64 v16.16b, v15.16b lsr w8, w2, #4 and w2, w2, #15 mov x9, v16.d[1] mov x10, v16.d[0] cmp w7, #12 blt L_aes_ctr_encrypt_arm64_crypto_start_128 bgt L_aes_ctr_encrypt_arm64_crypto_start_256 # AES_CTR_192 #ifndef NO_AES_192 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x4], #0x40 ld1 {v12.2d}, [x4] cmp w8, #1 ble L_aes_ctr_encrypt_arm64_crypto_192_start_1 adds x11, x9, #1 adc x12, x10, xzr cmp w8, #8 blt L_aes_ctr_encrypt_arm64_crypto_192_start_4 adds x13, x9, #2 adc x14, x10, xzr adds x15, x9, #3 adc x16, x10, xzr L_aes_ctr_encrypt_arm64_crypto_192_start_8: ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x0], #0x40 ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x0], #0x40 mov v17.d[0], x12 mov v17.d[1], x11 mov v18.d[0], x14 mov v18.d[1], x13 adds x17, x9, #4 mov v19.d[0], x16 adc x19, x10, xzr mov v19.d[1], x15 adds x20, x9, #5 mov v20.d[0], x19 adc x21, x10, xzr mov v20.d[1], x17 adds x22, x9, #6 mov v21.d[0], x21 adc x23, x10, xzr mov v21.d[1], x20 adds x24, x9, #7 mov v22.d[0], x23 adc x25, x10, xzr mov v22.d[1], x22 mov v23.d[0], x25 mov v23.d[1], x24 rev64 v16.16b, v16.16b rev64 v17.16b, v17.16b rev64 v18.16b, v18.16b rev64 v19.16b, v19.16b rev64 v20.16b, v20.16b rev64 v21.16b, v21.16b rev64 v22.16b, v22.16b rev64 v23.16b, v23.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v18.16b, v0.16b aesmc v18.16b, v18.16b aese v19.16b, v0.16b aesmc v19.16b, v19.16b aese v20.16b, v0.16b aesmc v20.16b, v20.16b aese v21.16b, v0.16b aesmc v21.16b, v21.16b aese v22.16b, v0.16b aesmc v22.16b, v22.16b aese v23.16b, v0.16b aesmc v23.16b, v23.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v18.16b, v1.16b aesmc v18.16b, v18.16b aese v19.16b, v1.16b aesmc v19.16b, v19.16b aese v20.16b, v1.16b aesmc v20.16b, v20.16b aese v21.16b, v1.16b aesmc v21.16b, v21.16b aese v22.16b, v1.16b aesmc v22.16b, v22.16b aese v23.16b, v1.16b aesmc v23.16b, v23.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v18.16b, v2.16b aesmc v18.16b, v18.16b aese v19.16b, v2.16b aesmc v19.16b, v19.16b aese v20.16b, v2.16b aesmc v20.16b, v20.16b aese v21.16b, v2.16b aesmc v21.16b, v21.16b aese v22.16b, v2.16b aesmc v22.16b, v22.16b aese v23.16b, v2.16b aesmc v23.16b, v23.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v18.16b, v3.16b aesmc v18.16b, v18.16b aese v19.16b, v3.16b aesmc v19.16b, v19.16b aese v20.16b, v3.16b aesmc v20.16b, v20.16b aese v21.16b, v3.16b aesmc v21.16b, v21.16b aese v22.16b, v3.16b aesmc v22.16b, v22.16b aese v23.16b, v3.16b aesmc v23.16b, v23.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v18.16b, v4.16b aesmc v18.16b, v18.16b aese v19.16b, v4.16b aesmc v19.16b, v19.16b aese v20.16b, v4.16b aesmc v20.16b, v20.16b aese v21.16b, v4.16b aesmc v21.16b, v21.16b aese v22.16b, v4.16b aesmc v22.16b, v22.16b aese v23.16b, v4.16b aesmc v23.16b, v23.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v18.16b, v5.16b aesmc v18.16b, v18.16b aese v19.16b, v5.16b aesmc v19.16b, v19.16b aese v20.16b, v5.16b aesmc v20.16b, v20.16b aese v21.16b, v5.16b aesmc v21.16b, v21.16b aese v22.16b, v5.16b aesmc v22.16b, v22.16b aese v23.16b, v5.16b aesmc v23.16b, v23.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b aese v18.16b, v6.16b aesmc v18.16b, v18.16b aese v19.16b, v6.16b aesmc v19.16b, v19.16b aese v20.16b, v6.16b aesmc v20.16b, v20.16b aese v21.16b, v6.16b aesmc v21.16b, v21.16b aese v22.16b, v6.16b aesmc v22.16b, v22.16b aese v23.16b, v6.16b aesmc v23.16b, v23.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v18.16b, v7.16b aesmc v18.16b, v18.16b aese v19.16b, v7.16b aesmc v19.16b, v19.16b aese v20.16b, v7.16b aesmc v20.16b, v20.16b aese v21.16b, v7.16b aesmc v21.16b, v21.16b aese v22.16b, v7.16b aesmc v22.16b, v22.16b aese v23.16b, v7.16b aesmc v23.16b, v23.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v18.16b, v8.16b aesmc v18.16b, v18.16b aese v19.16b, v8.16b aesmc v19.16b, v19.16b aese v20.16b, v8.16b aesmc v20.16b, v20.16b aese v21.16b, v8.16b aesmc v21.16b, v21.16b aese v22.16b, v8.16b aesmc v22.16b, v22.16b aese v23.16b, v8.16b aesmc v23.16b, v23.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v18.16b, v9.16b aesmc v18.16b, v18.16b aese v19.16b, v9.16b aesmc v19.16b, v19.16b aese v20.16b, v9.16b aesmc v20.16b, v20.16b aese v21.16b, v9.16b aesmc v21.16b, v21.16b aese v22.16b, v9.16b aesmc v22.16b, v22.16b aese v23.16b, v9.16b aesmc v23.16b, v23.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v18.16b, v10.16b aesmc v18.16b, v18.16b aese v19.16b, v10.16b aesmc v19.16b, v19.16b aese v20.16b, v10.16b aesmc v20.16b, v20.16b aese v21.16b, v10.16b aesmc v21.16b, v21.16b aese v22.16b, v10.16b aesmc v22.16b, v22.16b aese v23.16b, v10.16b aesmc v23.16b, v23.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b aese v18.16b, v11.16b eor v18.16b, v18.16b, v12.16b aese v19.16b, v11.16b eor v19.16b, v19.16b, v12.16b aese v20.16b, v11.16b eor v20.16b, v20.16b, v12.16b aese v21.16b, v11.16b eor v21.16b, v21.16b, v12.16b aese v22.16b, v11.16b eor v22.16b, v22.16b, v12.16b aese v23.16b, v11.16b eor v23.16b, v23.16b, v12.16b adds x9, x9, #8 eor v24.16b, v24.16b, v16.16b adc x10, x10, xzr eor v25.16b, v25.16b, v17.16b adds x11, x11, #8 eor v26.16b, v26.16b, v18.16b adc x12, x12, xzr eor v27.16b, v27.16b, v19.16b adds x13, x13, #8 eor v28.16b, v28.16b, v20.16b adc x14, x14, xzr eor v29.16b, v29.16b, v21.16b adds x15, x15, #8 eor v30.16b, v30.16b, v22.16b adc x16, x16, xzr eor v31.16b, v31.16b, v23.16b sub w8, w8, #8 st1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40 st1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x1], #0x40 mov v16.d[0], x10 mov v16.d[1], x9 cmp w8, #8 bge L_aes_ctr_encrypt_arm64_crypto_192_start_8 L_aes_ctr_encrypt_arm64_crypto_192_start_4: cmp w8, #4 blt L_aes_ctr_encrypt_arm64_crypto_192_start_2 ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x0], #0x40 adds x13, x9, #2 mov v17.d[0], x12 adc x14, x10, xzr mov v17.d[1], x11 adds x15, x9, #3 mov v18.d[0], x14 adc x16, x10, xzr mov v18.d[1], x13 mov v19.d[0], x16 mov v19.d[1], x15 rev64 v16.16b, v16.16b rev64 v17.16b, v17.16b rev64 v18.16b, v18.16b rev64 v19.16b, v19.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v18.16b, v0.16b aesmc v18.16b, v18.16b aese v19.16b, v0.16b aesmc v19.16b, v19.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v18.16b, v1.16b aesmc v18.16b, v18.16b aese v19.16b, v1.16b aesmc v19.16b, v19.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v18.16b, v2.16b aesmc v18.16b, v18.16b aese v19.16b, v2.16b aesmc v19.16b, v19.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v18.16b, v3.16b aesmc v18.16b, v18.16b aese v19.16b, v3.16b aesmc v19.16b, v19.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v18.16b, v4.16b aesmc v18.16b, v18.16b aese v19.16b, v4.16b aesmc v19.16b, v19.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v18.16b, v5.16b aesmc v18.16b, v18.16b aese v19.16b, v5.16b aesmc v19.16b, v19.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b aese v18.16b, v6.16b aesmc v18.16b, v18.16b aese v19.16b, v6.16b aesmc v19.16b, v19.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v18.16b, v7.16b aesmc v18.16b, v18.16b aese v19.16b, v7.16b aesmc v19.16b, v19.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v18.16b, v8.16b aesmc v18.16b, v18.16b aese v19.16b, v8.16b aesmc v19.16b, v19.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v18.16b, v9.16b aesmc v18.16b, v18.16b aese v19.16b, v9.16b aesmc v19.16b, v19.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v18.16b, v10.16b aesmc v18.16b, v18.16b aese v19.16b, v10.16b aesmc v19.16b, v19.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b aese v18.16b, v11.16b eor v18.16b, v18.16b, v12.16b aese v19.16b, v11.16b eor v19.16b, v19.16b, v12.16b adds x9, x9, #4 eor v24.16b, v24.16b, v16.16b adc x10, x10, xzr eor v25.16b, v25.16b, v17.16b adds x11, x11, #4 eor v26.16b, v26.16b, v18.16b adc x12, x12, xzr eor v27.16b, v27.16b, v19.16b sub w8, w8, #4 st1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40 mov v16.d[0], x10 mov v16.d[1], x9 L_aes_ctr_encrypt_arm64_crypto_192_start_2: cmp w8, #2 blt L_aes_ctr_encrypt_arm64_crypto_192_start_1 ld1 {v24.16b, v25.16b}, [x0], #32 eor v20.16b, v20.16b, v20.16b ext v19.16b, v16.16b, v16.16b, #8 movi v18.16b, #1 ext v18.16b, v18.16b, v20.16b, #15 add v17.2d, v19.2d, v18.2d cmeq v19.2d, v17.2d, #0 ext v19.16b, v20.16b, v19.16b, #8 sub v17.2d, v17.2d, v19.2d ext v17.16b, v17.16b, v17.16b, #8 rev64 v16.16b, v16.16b rev64 v17.16b, v17.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b adds x9, x9, #2 eor v24.16b, v24.16b, v16.16b adc x10, x10, xzr eor v25.16b, v25.16b, v17.16b sub w8, w8, #2 st1 {v24.16b, v25.16b}, [x1], #32 mov v16.d[0], x10 mov v16.d[1], x9 L_aes_ctr_encrypt_arm64_crypto_192_start_1: cbz w8, L_aes_ctr_encrypt_arm64_crypto_192_done ld1 {v24.16b}, [x0], #16 rev64 v16.16b, v16.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b adds x9, x9, #1 eor v24.16b, v24.16b, v16.16b adc x10, x10, xzr st1 {v24.16b}, [x1], #16 L_aes_ctr_encrypt_arm64_crypto_192_done: cbz w2, L_aes_ctr_encrypt_arm64_crypto_192_partial_done mov v16.d[0], x10 mov v16.d[1], x9 rev64 v16.16b, v16.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b adds x9, x9, #1 adc x10, x10, xzr st1 {v16.2d}, [x5] mov w13, #16 sub w13, w13, w2 L_aes_ctr_encrypt_arm64_crypto_192_start_byte: ldrb w11, [x5], #1 ldrb w12, [x0], #1 eor w11, w11, w12 subs w2, w2, #1 strb w11, [x1], #1 bgt L_aes_ctr_encrypt_arm64_crypto_192_start_byte str w13, [x6] L_aes_ctr_encrypt_arm64_crypto_192_partial_done: #endif /* !NO_AES_192 */ b L_aes_ctr_encrypt_arm64_crypto_done # AES_CTR_256 L_aes_ctr_encrypt_arm64_crypto_start_256: #ifndef NO_AES_256 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x4], #0x40 ld1 {v12.2d, v13.2d}, [x4], #32 ld1 {v14.2d}, [x4] cmp w8, #1 ble L_aes_ctr_encrypt_arm64_crypto_256_start_1 adds x11, x9, #1 adc x12, x10, xzr cmp w8, #8 blt L_aes_ctr_encrypt_arm64_crypto_256_start_4 adds x13, x9, #2 adc x14, x10, xzr adds x15, x9, #3 adc x16, x10, xzr L_aes_ctr_encrypt_arm64_crypto_256_start_8: ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x0], #0x40 ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x0], #0x40 mov v17.d[0], x12 mov v17.d[1], x11 mov v18.d[0], x14 mov v18.d[1], x13 adds x17, x9, #4 mov v19.d[0], x16 adc x19, x10, xzr mov v19.d[1], x15 adds x20, x9, #5 mov v20.d[0], x19 adc x21, x10, xzr mov v20.d[1], x17 adds x22, x9, #6 mov v21.d[0], x21 adc x23, x10, xzr mov v21.d[1], x20 adds x24, x9, #7 mov v22.d[0], x23 adc x25, x10, xzr mov v22.d[1], x22 mov v23.d[0], x25 mov v23.d[1], x24 rev64 v16.16b, v16.16b rev64 v17.16b, v17.16b rev64 v18.16b, v18.16b rev64 v19.16b, v19.16b rev64 v20.16b, v20.16b rev64 v21.16b, v21.16b rev64 v22.16b, v22.16b rev64 v23.16b, v23.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v18.16b, v0.16b aesmc v18.16b, v18.16b aese v19.16b, v0.16b aesmc v19.16b, v19.16b aese v20.16b, v0.16b aesmc v20.16b, v20.16b aese v21.16b, v0.16b aesmc v21.16b, v21.16b aese v22.16b, v0.16b aesmc v22.16b, v22.16b aese v23.16b, v0.16b aesmc v23.16b, v23.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v18.16b, v1.16b aesmc v18.16b, v18.16b aese v19.16b, v1.16b aesmc v19.16b, v19.16b aese v20.16b, v1.16b aesmc v20.16b, v20.16b aese v21.16b, v1.16b aesmc v21.16b, v21.16b aese v22.16b, v1.16b aesmc v22.16b, v22.16b aese v23.16b, v1.16b aesmc v23.16b, v23.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v18.16b, v2.16b aesmc v18.16b, v18.16b aese v19.16b, v2.16b aesmc v19.16b, v19.16b aese v20.16b, v2.16b aesmc v20.16b, v20.16b aese v21.16b, v2.16b aesmc v21.16b, v21.16b aese v22.16b, v2.16b aesmc v22.16b, v22.16b aese v23.16b, v2.16b aesmc v23.16b, v23.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v18.16b, v3.16b aesmc v18.16b, v18.16b aese v19.16b, v3.16b aesmc v19.16b, v19.16b aese v20.16b, v3.16b aesmc v20.16b, v20.16b aese v21.16b, v3.16b aesmc v21.16b, v21.16b aese v22.16b, v3.16b aesmc v22.16b, v22.16b aese v23.16b, v3.16b aesmc v23.16b, v23.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v18.16b, v4.16b aesmc v18.16b, v18.16b aese v19.16b, v4.16b aesmc v19.16b, v19.16b aese v20.16b, v4.16b aesmc v20.16b, v20.16b aese v21.16b, v4.16b aesmc v21.16b, v21.16b aese v22.16b, v4.16b aesmc v22.16b, v22.16b aese v23.16b, v4.16b aesmc v23.16b, v23.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v18.16b, v5.16b aesmc v18.16b, v18.16b aese v19.16b, v5.16b aesmc v19.16b, v19.16b aese v20.16b, v5.16b aesmc v20.16b, v20.16b aese v21.16b, v5.16b aesmc v21.16b, v21.16b aese v22.16b, v5.16b aesmc v22.16b, v22.16b aese v23.16b, v5.16b aesmc v23.16b, v23.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b aese v18.16b, v6.16b aesmc v18.16b, v18.16b aese v19.16b, v6.16b aesmc v19.16b, v19.16b aese v20.16b, v6.16b aesmc v20.16b, v20.16b aese v21.16b, v6.16b aesmc v21.16b, v21.16b aese v22.16b, v6.16b aesmc v22.16b, v22.16b aese v23.16b, v6.16b aesmc v23.16b, v23.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v18.16b, v7.16b aesmc v18.16b, v18.16b aese v19.16b, v7.16b aesmc v19.16b, v19.16b aese v20.16b, v7.16b aesmc v20.16b, v20.16b aese v21.16b, v7.16b aesmc v21.16b, v21.16b aese v22.16b, v7.16b aesmc v22.16b, v22.16b aese v23.16b, v7.16b aesmc v23.16b, v23.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v18.16b, v8.16b aesmc v18.16b, v18.16b aese v19.16b, v8.16b aesmc v19.16b, v19.16b aese v20.16b, v8.16b aesmc v20.16b, v20.16b aese v21.16b, v8.16b aesmc v21.16b, v21.16b aese v22.16b, v8.16b aesmc v22.16b, v22.16b aese v23.16b, v8.16b aesmc v23.16b, v23.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v18.16b, v9.16b aesmc v18.16b, v18.16b aese v19.16b, v9.16b aesmc v19.16b, v19.16b aese v20.16b, v9.16b aesmc v20.16b, v20.16b aese v21.16b, v9.16b aesmc v21.16b, v21.16b aese v22.16b, v9.16b aesmc v22.16b, v22.16b aese v23.16b, v9.16b aesmc v23.16b, v23.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v18.16b, v10.16b aesmc v18.16b, v18.16b aese v19.16b, v10.16b aesmc v19.16b, v19.16b aese v20.16b, v10.16b aesmc v20.16b, v20.16b aese v21.16b, v10.16b aesmc v21.16b, v21.16b aese v22.16b, v10.16b aesmc v22.16b, v22.16b aese v23.16b, v10.16b aesmc v23.16b, v23.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b aese v18.16b, v11.16b aesmc v18.16b, v18.16b aese v19.16b, v11.16b aesmc v19.16b, v19.16b aese v20.16b, v11.16b aesmc v20.16b, v20.16b aese v21.16b, v11.16b aesmc v21.16b, v21.16b aese v22.16b, v11.16b aesmc v22.16b, v22.16b aese v23.16b, v11.16b aesmc v23.16b, v23.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v18.16b, v12.16b aesmc v18.16b, v18.16b aese v19.16b, v12.16b aesmc v19.16b, v19.16b aese v20.16b, v12.16b aesmc v20.16b, v20.16b aese v21.16b, v12.16b aesmc v21.16b, v21.16b aese v22.16b, v12.16b aesmc v22.16b, v22.16b aese v23.16b, v12.16b aesmc v23.16b, v23.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v14.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v14.16b aese v18.16b, v13.16b eor v18.16b, v18.16b, v14.16b aese v19.16b, v13.16b eor v19.16b, v19.16b, v14.16b aese v20.16b, v13.16b eor v20.16b, v20.16b, v14.16b aese v21.16b, v13.16b eor v21.16b, v21.16b, v14.16b aese v22.16b, v13.16b eor v22.16b, v22.16b, v14.16b aese v23.16b, v13.16b eor v23.16b, v23.16b, v14.16b adds x9, x9, #8 eor v24.16b, v24.16b, v16.16b adc x10, x10, xzr eor v25.16b, v25.16b, v17.16b adds x11, x11, #8 eor v26.16b, v26.16b, v18.16b adc x12, x12, xzr eor v27.16b, v27.16b, v19.16b adds x13, x13, #8 eor v28.16b, v28.16b, v20.16b adc x14, x14, xzr eor v29.16b, v29.16b, v21.16b adds x15, x15, #8 eor v30.16b, v30.16b, v22.16b adc x16, x16, xzr eor v31.16b, v31.16b, v23.16b sub w8, w8, #8 st1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40 st1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x1], #0x40 mov v16.d[0], x10 mov v16.d[1], x9 cmp w8, #8 bge L_aes_ctr_encrypt_arm64_crypto_256_start_8 L_aes_ctr_encrypt_arm64_crypto_256_start_4: cmp w8, #4 blt L_aes_ctr_encrypt_arm64_crypto_256_start_2 ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x0], #0x40 adds x13, x9, #2 mov v17.d[0], x12 adc x14, x10, xzr mov v17.d[1], x11 adds x15, x9, #3 mov v18.d[0], x14 adc x16, x10, xzr mov v18.d[1], x13 mov v19.d[0], x16 mov v19.d[1], x15 rev64 v16.16b, v16.16b rev64 v17.16b, v17.16b rev64 v18.16b, v18.16b rev64 v19.16b, v19.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v18.16b, v0.16b aesmc v18.16b, v18.16b aese v19.16b, v0.16b aesmc v19.16b, v19.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v18.16b, v1.16b aesmc v18.16b, v18.16b aese v19.16b, v1.16b aesmc v19.16b, v19.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v18.16b, v2.16b aesmc v18.16b, v18.16b aese v19.16b, v2.16b aesmc v19.16b, v19.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v18.16b, v3.16b aesmc v18.16b, v18.16b aese v19.16b, v3.16b aesmc v19.16b, v19.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v18.16b, v4.16b aesmc v18.16b, v18.16b aese v19.16b, v4.16b aesmc v19.16b, v19.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v18.16b, v5.16b aesmc v18.16b, v18.16b aese v19.16b, v5.16b aesmc v19.16b, v19.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b aese v18.16b, v6.16b aesmc v18.16b, v18.16b aese v19.16b, v6.16b aesmc v19.16b, v19.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v18.16b, v7.16b aesmc v18.16b, v18.16b aese v19.16b, v7.16b aesmc v19.16b, v19.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v18.16b, v8.16b aesmc v18.16b, v18.16b aese v19.16b, v8.16b aesmc v19.16b, v19.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v18.16b, v9.16b aesmc v18.16b, v18.16b aese v19.16b, v9.16b aesmc v19.16b, v19.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v18.16b, v10.16b aesmc v18.16b, v18.16b aese v19.16b, v10.16b aesmc v19.16b, v19.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b aese v18.16b, v11.16b aesmc v18.16b, v18.16b aese v19.16b, v11.16b aesmc v19.16b, v19.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v18.16b, v12.16b aesmc v18.16b, v18.16b aese v19.16b, v12.16b aesmc v19.16b, v19.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v14.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v14.16b aese v18.16b, v13.16b eor v18.16b, v18.16b, v14.16b aese v19.16b, v13.16b eor v19.16b, v19.16b, v14.16b adds x9, x9, #4 eor v24.16b, v24.16b, v16.16b adc x10, x10, xzr eor v25.16b, v25.16b, v17.16b adds x11, x11, #4 eor v26.16b, v26.16b, v18.16b adc x12, x12, xzr eor v27.16b, v27.16b, v19.16b sub w8, w8, #4 st1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40 mov v16.d[0], x10 mov v16.d[1], x9 L_aes_ctr_encrypt_arm64_crypto_256_start_2: cmp w8, #2 blt L_aes_ctr_encrypt_arm64_crypto_256_start_1 ld1 {v24.16b, v25.16b}, [x0], #32 eor v20.16b, v20.16b, v20.16b ext v19.16b, v16.16b, v16.16b, #8 movi v18.16b, #1 ext v18.16b, v18.16b, v20.16b, #15 add v17.2d, v19.2d, v18.2d cmeq v19.2d, v17.2d, #0 ext v19.16b, v20.16b, v19.16b, #8 sub v17.2d, v17.2d, v19.2d ext v17.16b, v17.16b, v17.16b, #8 rev64 v16.16b, v16.16b rev64 v17.16b, v17.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v14.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v14.16b adds x9, x9, #2 eor v24.16b, v24.16b, v16.16b adc x10, x10, xzr eor v25.16b, v25.16b, v17.16b sub w8, w8, #2 st1 {v24.16b, v25.16b}, [x1], #32 mov v16.d[0], x10 mov v16.d[1], x9 L_aes_ctr_encrypt_arm64_crypto_256_start_1: cbz w8, L_aes_ctr_encrypt_arm64_crypto_256_done ld1 {v24.16b}, [x0], #16 rev64 v16.16b, v16.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v14.16b adds x9, x9, #1 eor v24.16b, v24.16b, v16.16b adc x10, x10, xzr st1 {v24.16b}, [x1], #16 L_aes_ctr_encrypt_arm64_crypto_256_done: cbz w2, L_aes_ctr_encrypt_arm64_crypto_256_partial_done mov v16.d[0], x10 mov v16.d[1], x9 rev64 v16.16b, v16.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v14.16b adds x9, x9, #1 adc x10, x10, xzr st1 {v16.2d}, [x5] mov w13, #16 sub w13, w13, w2 L_aes_ctr_encrypt_arm64_crypto_256_start_byte: ldrb w11, [x5], #1 ldrb w12, [x0], #1 eor w11, w11, w12 subs w2, w2, #1 strb w11, [x1], #1 bgt L_aes_ctr_encrypt_arm64_crypto_256_start_byte str w13, [x6] L_aes_ctr_encrypt_arm64_crypto_256_partial_done: #endif /* !NO_AES_256 */ b L_aes_ctr_encrypt_arm64_crypto_done # AES_CTR_128 L_aes_ctr_encrypt_arm64_crypto_start_128: #ifndef NO_AES_128 ld1 {v8.2d, v9.2d}, [x4], #32 ld1 {v10.2d}, [x4] cmp w8, #1 ble L_aes_ctr_encrypt_arm64_crypto_128_start_1 adds x11, x9, #1 adc x12, x10, xzr cmp w8, #8 blt L_aes_ctr_encrypt_arm64_crypto_128_start_4 adds x13, x9, #2 adc x14, x10, xzr adds x15, x9, #3 adc x16, x10, xzr L_aes_ctr_encrypt_arm64_crypto_128_start_8: ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x0], #0x40 ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x0], #0x40 mov v17.d[0], x12 mov v17.d[1], x11 mov v18.d[0], x14 mov v18.d[1], x13 adds x17, x9, #4 mov v19.d[0], x16 adc x19, x10, xzr mov v19.d[1], x15 adds x20, x9, #5 mov v20.d[0], x19 adc x21, x10, xzr mov v20.d[1], x17 adds x22, x9, #6 mov v21.d[0], x21 adc x23, x10, xzr mov v21.d[1], x20 adds x24, x9, #7 mov v22.d[0], x23 adc x25, x10, xzr mov v22.d[1], x22 mov v23.d[0], x25 mov v23.d[1], x24 rev64 v16.16b, v16.16b rev64 v17.16b, v17.16b rev64 v18.16b, v18.16b rev64 v19.16b, v19.16b rev64 v20.16b, v20.16b rev64 v21.16b, v21.16b rev64 v22.16b, v22.16b rev64 v23.16b, v23.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v18.16b, v0.16b aesmc v18.16b, v18.16b aese v19.16b, v0.16b aesmc v19.16b, v19.16b aese v20.16b, v0.16b aesmc v20.16b, v20.16b aese v21.16b, v0.16b aesmc v21.16b, v21.16b aese v22.16b, v0.16b aesmc v22.16b, v22.16b aese v23.16b, v0.16b aesmc v23.16b, v23.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v18.16b, v1.16b aesmc v18.16b, v18.16b aese v19.16b, v1.16b aesmc v19.16b, v19.16b aese v20.16b, v1.16b aesmc v20.16b, v20.16b aese v21.16b, v1.16b aesmc v21.16b, v21.16b aese v22.16b, v1.16b aesmc v22.16b, v22.16b aese v23.16b, v1.16b aesmc v23.16b, v23.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v18.16b, v2.16b aesmc v18.16b, v18.16b aese v19.16b, v2.16b aesmc v19.16b, v19.16b aese v20.16b, v2.16b aesmc v20.16b, v20.16b aese v21.16b, v2.16b aesmc v21.16b, v21.16b aese v22.16b, v2.16b aesmc v22.16b, v22.16b aese v23.16b, v2.16b aesmc v23.16b, v23.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v18.16b, v3.16b aesmc v18.16b, v18.16b aese v19.16b, v3.16b aesmc v19.16b, v19.16b aese v20.16b, v3.16b aesmc v20.16b, v20.16b aese v21.16b, v3.16b aesmc v21.16b, v21.16b aese v22.16b, v3.16b aesmc v22.16b, v22.16b aese v23.16b, v3.16b aesmc v23.16b, v23.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v18.16b, v4.16b aesmc v18.16b, v18.16b aese v19.16b, v4.16b aesmc v19.16b, v19.16b aese v20.16b, v4.16b aesmc v20.16b, v20.16b aese v21.16b, v4.16b aesmc v21.16b, v21.16b aese v22.16b, v4.16b aesmc v22.16b, v22.16b aese v23.16b, v4.16b aesmc v23.16b, v23.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v18.16b, v5.16b aesmc v18.16b, v18.16b aese v19.16b, v5.16b aesmc v19.16b, v19.16b aese v20.16b, v5.16b aesmc v20.16b, v20.16b aese v21.16b, v5.16b aesmc v21.16b, v21.16b aese v22.16b, v5.16b aesmc v22.16b, v22.16b aese v23.16b, v5.16b aesmc v23.16b, v23.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b aese v18.16b, v6.16b aesmc v18.16b, v18.16b aese v19.16b, v6.16b aesmc v19.16b, v19.16b aese v20.16b, v6.16b aesmc v20.16b, v20.16b aese v21.16b, v6.16b aesmc v21.16b, v21.16b aese v22.16b, v6.16b aesmc v22.16b, v22.16b aese v23.16b, v6.16b aesmc v23.16b, v23.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v18.16b, v7.16b aesmc v18.16b, v18.16b aese v19.16b, v7.16b aesmc v19.16b, v19.16b aese v20.16b, v7.16b aesmc v20.16b, v20.16b aese v21.16b, v7.16b aesmc v21.16b, v21.16b aese v22.16b, v7.16b aesmc v22.16b, v22.16b aese v23.16b, v7.16b aesmc v23.16b, v23.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v18.16b, v8.16b aesmc v18.16b, v18.16b aese v19.16b, v8.16b aesmc v19.16b, v19.16b aese v20.16b, v8.16b aesmc v20.16b, v20.16b aese v21.16b, v8.16b aesmc v21.16b, v21.16b aese v22.16b, v8.16b aesmc v22.16b, v22.16b aese v23.16b, v8.16b aesmc v23.16b, v23.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b aese v18.16b, v9.16b eor v18.16b, v18.16b, v10.16b aese v19.16b, v9.16b eor v19.16b, v19.16b, v10.16b aese v20.16b, v9.16b eor v20.16b, v20.16b, v10.16b aese v21.16b, v9.16b eor v21.16b, v21.16b, v10.16b aese v22.16b, v9.16b eor v22.16b, v22.16b, v10.16b aese v23.16b, v9.16b eor v23.16b, v23.16b, v10.16b adds x9, x9, #8 eor v24.16b, v24.16b, v16.16b adc x10, x10, xzr eor v25.16b, v25.16b, v17.16b adds x11, x11, #8 eor v26.16b, v26.16b, v18.16b adc x12, x12, xzr eor v27.16b, v27.16b, v19.16b adds x13, x13, #8 eor v28.16b, v28.16b, v20.16b adc x14, x14, xzr eor v29.16b, v29.16b, v21.16b adds x15, x15, #8 eor v30.16b, v30.16b, v22.16b adc x16, x16, xzr eor v31.16b, v31.16b, v23.16b sub w8, w8, #8 st1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40 st1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x1], #0x40 mov v16.d[0], x10 mov v16.d[1], x9 cmp w8, #8 bge L_aes_ctr_encrypt_arm64_crypto_128_start_8 L_aes_ctr_encrypt_arm64_crypto_128_start_4: cmp w8, #4 blt L_aes_ctr_encrypt_arm64_crypto_128_start_2 ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x0], #0x40 adds x13, x9, #2 mov v17.d[0], x12 adc x14, x10, xzr mov v17.d[1], x11 adds x15, x9, #3 mov v18.d[0], x14 adc x16, x10, xzr mov v18.d[1], x13 mov v19.d[0], x16 mov v19.d[1], x15 rev64 v16.16b, v16.16b rev64 v17.16b, v17.16b rev64 v18.16b, v18.16b rev64 v19.16b, v19.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v18.16b, v0.16b aesmc v18.16b, v18.16b aese v19.16b, v0.16b aesmc v19.16b, v19.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v18.16b, v1.16b aesmc v18.16b, v18.16b aese v19.16b, v1.16b aesmc v19.16b, v19.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v18.16b, v2.16b aesmc v18.16b, v18.16b aese v19.16b, v2.16b aesmc v19.16b, v19.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v18.16b, v3.16b aesmc v18.16b, v18.16b aese v19.16b, v3.16b aesmc v19.16b, v19.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v18.16b, v4.16b aesmc v18.16b, v18.16b aese v19.16b, v4.16b aesmc v19.16b, v19.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v18.16b, v5.16b aesmc v18.16b, v18.16b aese v19.16b, v5.16b aesmc v19.16b, v19.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b aese v18.16b, v6.16b aesmc v18.16b, v18.16b aese v19.16b, v6.16b aesmc v19.16b, v19.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v18.16b, v7.16b aesmc v18.16b, v18.16b aese v19.16b, v7.16b aesmc v19.16b, v19.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v18.16b, v8.16b aesmc v18.16b, v18.16b aese v19.16b, v8.16b aesmc v19.16b, v19.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b aese v18.16b, v9.16b eor v18.16b, v18.16b, v10.16b aese v19.16b, v9.16b eor v19.16b, v19.16b, v10.16b adds x9, x9, #4 eor v24.16b, v24.16b, v16.16b adc x10, x10, xzr eor v25.16b, v25.16b, v17.16b adds x11, x11, #4 eor v26.16b, v26.16b, v18.16b adc x12, x12, xzr eor v27.16b, v27.16b, v19.16b sub w8, w8, #4 st1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40 mov v16.d[0], x10 mov v16.d[1], x9 L_aes_ctr_encrypt_arm64_crypto_128_start_2: cmp w8, #2 blt L_aes_ctr_encrypt_arm64_crypto_128_start_1 ld1 {v24.16b, v25.16b}, [x0], #32 eor v20.16b, v20.16b, v20.16b ext v19.16b, v16.16b, v16.16b, #8 movi v18.16b, #1 ext v18.16b, v18.16b, v20.16b, #15 add v17.2d, v19.2d, v18.2d cmeq v19.2d, v17.2d, #0 ext v19.16b, v20.16b, v19.16b, #8 sub v17.2d, v17.2d, v19.2d ext v17.16b, v17.16b, v17.16b, #8 rev64 v16.16b, v16.16b rev64 v17.16b, v17.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b adds x9, x9, #2 eor v24.16b, v24.16b, v16.16b adc x10, x10, xzr eor v25.16b, v25.16b, v17.16b sub w8, w8, #2 st1 {v24.16b, v25.16b}, [x1], #32 mov v16.d[0], x10 mov v16.d[1], x9 L_aes_ctr_encrypt_arm64_crypto_128_start_1: cbz w8, L_aes_ctr_encrypt_arm64_crypto_128_done ld1 {v24.16b}, [x0], #16 rev64 v16.16b, v16.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b adds x9, x9, #1 eor v24.16b, v24.16b, v16.16b adc x10, x10, xzr st1 {v24.16b}, [x1], #16 L_aes_ctr_encrypt_arm64_crypto_128_done: cbz w2, L_aes_ctr_encrypt_arm64_crypto_128_partial_done mov v16.d[0], x10 mov v16.d[1], x9 rev64 v16.16b, v16.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b adds x9, x9, #1 adc x10, x10, xzr st1 {v16.2d}, [x5] mov w13, #16 sub w13, w13, w2 L_aes_ctr_encrypt_arm64_crypto_128_start_byte: ldrb w11, [x5], #1 ldrb w12, [x0], #1 eor w11, w11, w12 subs w2, w2, #1 strb w11, [x1], #1 bgt L_aes_ctr_encrypt_arm64_crypto_128_start_byte str w13, [x6] L_aes_ctr_encrypt_arm64_crypto_128_partial_done: #endif /* !NO_AES_128 */ L_aes_ctr_encrypt_arm64_crypto_done: rev x11, x10 rev x12, x9 stp x11, x12, [x3] ldp x17, x19, [x29, #16] ldp x20, x21, [x29, #32] ldp x22, x23, [x29, #48] ldp x24, x25, [x29, #64] ldp d8, d9, [x29, #80] ldp d10, d11, [x29, #96] ldp d12, d13, [x29, #112] ldp d14, d15, [x29, #128] ldp x29, x30, [sp], #0x90 ret #ifndef __APPLE__ .size AES_CTR_encrypt_AARCH64,.-AES_CTR_encrypt_AARCH64 #endif /* __APPLE__ */ #endif /* WOLFSSL_AES_COUNTER */ #ifdef HAVE_AESGCM #ifndef __APPLE__ .text .globl AES_GCM_set_key_AARCH64 .type AES_GCM_set_key_AARCH64,@function .align 2 AES_GCM_set_key_AARCH64: #else .section __TEXT,__text .globl _AES_GCM_set_key_AARCH64 .p2align 2 _AES_GCM_set_key_AARCH64: #endif /* __APPLE__ */ ld1 {v0.16b}, [x0] ld1 {v1.2d, v2.2d, v3.2d, v4.2d}, [x1], #0x40 aese v0.16b, v1.16b aesmc v0.16b, v0.16b aese v0.16b, v2.16b aesmc v0.16b, v0.16b aese v0.16b, v3.16b aesmc v0.16b, v0.16b aese v0.16b, v4.16b aesmc v0.16b, v0.16b ld1 {v1.2d, v2.2d, v3.2d, v4.2d}, [x1], #0x40 aese v0.16b, v1.16b aesmc v0.16b, v0.16b aese v0.16b, v2.16b aesmc v0.16b, v0.16b aese v0.16b, v3.16b aesmc v0.16b, v0.16b aese v0.16b, v4.16b aesmc v0.16b, v0.16b subs w3, w3, #10 ld1 {v1.2d, v2.2d}, [x1], #32 aese v0.16b, v1.16b aesmc v0.16b, v0.16b aese v0.16b, v2.16b beq L_aes_gcm_set_key_arm64_crypto_round_done ld1 {v1.2d, v2.2d}, [x1], #32 subs w3, w3, #2 aesmc v0.16b, v0.16b aese v0.16b, v1.16b aesmc v0.16b, v0.16b aese v0.16b, v2.16b beq L_aes_gcm_set_key_arm64_crypto_round_done ld1 {v1.2d, v2.2d}, [x1], #32 aesmc v0.16b, v0.16b aese v0.16b, v1.16b aesmc v0.16b, v0.16b aese v0.16b, v2.16b L_aes_gcm_set_key_arm64_crypto_round_done: ld1 {v1.2d}, [x1] eor v0.16b, v0.16b, v1.16b rbit v0.16b, v0.16b st1 {v0.2d}, [x2] ret #ifndef __APPLE__ .size AES_GCM_set_key_AARCH64,.-AES_GCM_set_key_AARCH64 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_encrypt_AARCH64 .type AES_GCM_encrypt_AARCH64,@function .align 2 AES_GCM_encrypt_AARCH64: #else .section __TEXT,__text .globl _AES_GCM_encrypt_AARCH64 .p2align 2 _AES_GCM_encrypt_AARCH64: #endif /* __APPLE__ */ stp x29, x30, [sp, #-144]! add x29, sp, #0 stp x17, x19, [x29, #24] stp x20, x21, [x29, #40] stp x22, x23, [x29, #56] str x24, [x29, #72] stp d8, d9, [x29, #80] stp d10, d11, [x29, #96] stp d12, d13, [x29, #112] stp d14, d15, [x29, #128] ldr w8, [x29, #144] ldr x9, [x29, #152] ldr x10, [x29, #160] ldr x11, [x29, #168] ldr x12, [x29, #176] ldr w13, [x29, #184] movi v27.16b, #0x87 eor v26.16b, v26.16b, v26.16b ushr v27.2d, v27.2d, #56 ld1 {v22.2d}, [x10] cmp w8, #0x40 csetm x16, lt cmp w2, #32 csetm x17, lt ands x16, x16, x17 bne L_aes_gcm_encrypt_arm64_crypto_h_done # Square H => H^2 pmull2 v31.1q, v22.2d, v22.2d pmull v30.1q, v22.1d, v22.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v23.16b, v30.16b, v31.16b cmp w8, #0x100 csetm x16, lt cmp w2, #0x40 csetm x17, lt ands x16, x16, x17 bne L_aes_gcm_encrypt_arm64_crypto_h_done # Multiply H and H^2 => H^3 pmull v28.1q, v22.1d, v23.1d pmull2 v29.1q, v22.2d, v23.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v23.1d pmull2 v31.1q, v31.2d, v23.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v24.16b, v28.16b, v30.16b # Square H^2 => H^4 pmull2 v31.1q, v23.2d, v23.2d pmull v30.1q, v23.1d, v23.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v25.16b, v30.16b, v31.16b # Done cmp w8, #0x400 csetm x16, lt cmp w2, #0x200 csetm x17, lt ands x16, x16, x17 bne L_aes_gcm_encrypt_arm64_crypto_h_done # Multiply H and H^4 => H^5 pmull v28.1q, v22.1d, v25.1d pmull2 v29.1q, v22.2d, v25.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v25.1d pmull2 v31.1q, v31.2d, v25.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v4.16b, v28.16b, v30.16b # Square H^3 => H^6 pmull2 v31.1q, v24.2d, v24.2d pmull v30.1q, v24.1d, v24.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v5.16b, v30.16b, v31.16b # Multiply H and H^6 => H^7 pmull v28.1q, v22.1d, v5.1d pmull2 v29.1q, v22.2d, v5.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v5.1d pmull2 v31.1q, v31.2d, v5.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v6.16b, v28.16b, v30.16b # Square H^4 => H^8 pmull2 v31.1q, v25.2d, v25.2d pmull v30.1q, v25.1d, v25.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v7.16b, v30.16b, v31.16b # Done L_aes_gcm_encrypt_arm64_crypto_h_done: lsr w14, w8, #4 cmp w14, #4 blt L_aes_gcm_encrypt_arm64_crypto_aad_start_1 cmp w14, #16 blt L_aes_gcm_encrypt_arm64_crypto_aad_start_2 cmp w14, #0x40 blt L_aes_gcm_encrypt_arm64_crypto_aad_start_4 L_aes_gcm_encrypt_arm64_crypto_aad_start_8: ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x7], #0x40 rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH sub w14, w14, #8 cmp w14, #8 bge L_aes_gcm_encrypt_arm64_crypto_aad_start_8 cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_aad_done beq L_aes_gcm_encrypt_arm64_crypto_aad_start_1 cmp w14, #16 blt L_aes_gcm_encrypt_arm64_crypto_aad_start_2 L_aes_gcm_encrypt_arm64_crypto_aad_start_4: ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH sub w14, w14, #4 cmp w14, #4 bge L_aes_gcm_encrypt_arm64_crypto_aad_start_4 cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_aad_done beq L_aes_gcm_encrypt_arm64_crypto_aad_start_1 L_aes_gcm_encrypt_arm64_crypto_aad_start_2: ld1 {v18.16b, v19.16b}, [x7], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH sub w14, w14, #2 cmp w14, #1 bgt L_aes_gcm_encrypt_arm64_crypto_aad_start_2 blt L_aes_gcm_encrypt_arm64_crypto_aad_done L_aes_gcm_encrypt_arm64_crypto_aad_start_1: cbz w14, L_aes_gcm_encrypt_arm64_crypto_aad_done L_aes_gcm_encrypt_arm64_crypto_aad_both_1: ld1 {v18.16b}, [x7], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH subs w14, w14, #1 bne L_aes_gcm_encrypt_arm64_crypto_aad_both_1 L_aes_gcm_encrypt_arm64_crypto_aad_done: and w14, w8, #15 cbz w14, L_aes_gcm_encrypt_arm64_crypto_aad_partial_done eor v28.16b, v28.16b, v28.16b mov w20, w14 st1 {v28.2d}, [x11] cmp w20, #8 blt L_aes_gcm_encrypt_arm64_crypto_aad_start_dw ldr x19, [x7], #8 sub w20, w20, #8 str x19, [x11], #8 L_aes_gcm_encrypt_arm64_crypto_aad_start_dw: cmp w20, #4 blt L_aes_gcm_encrypt_arm64_crypto_aad_start_sw ldr w19, [x7], #4 sub w20, w20, #4 str w19, [x11], #4 L_aes_gcm_encrypt_arm64_crypto_aad_start_sw: cmp w20, #2 blt L_aes_gcm_encrypt_arm64_crypto_aad_start_byte ldrh w19, [x7], #2 sub w20, w20, #2 strh w19, [x11], #2 L_aes_gcm_encrypt_arm64_crypto_aad_start_byte: cbz w20, L_aes_gcm_encrypt_arm64_crypto_aad_end_bytes ldrb w19, [x7], #1 subs w20, w20, #1 strb w19, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_aad_start_byte L_aes_gcm_encrypt_arm64_crypto_aad_end_bytes: sub x11, x11, x14 ld1 {v18.2d}, [x11] rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_aad_partial_done: # Load Nonce cmp w4, #12 bne L_aes_gcm_encrypt_arm64_crypto_ghash_nonce ldr x16, [x3] movi v13.4s, #1, lsl 24 ldr w17, [x3, #8] mov v13.d[0], x16 mov v13.s[2], w17 mov w15, #1 b L_aes_gcm_encrypt_arm64_crypto_done_nonce L_aes_gcm_encrypt_arm64_crypto_ghash_nonce: eor v13.16b, v13.16b, v13.16b lsr w14, w4, #4 cbz w14, L_aes_gcm_encrypt_arm64_crypto_nonce_done L_aes_gcm_encrypt_arm64_crypto_nonce_start_1: ld1 {v18.16b}, [x3], #16 rbit v18.16b, v18.16b eor v21.16b, v13.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v13.16b, v28.16b, v30.16b # Done GHASH subs w14, w14, #1 bne L_aes_gcm_encrypt_arm64_crypto_nonce_start_1 L_aes_gcm_encrypt_arm64_crypto_nonce_done: and w24, w4, #15 cbz x24, L_aes_gcm_encrypt_arm64_crypto_nonce_partial_done eor v28.16b, v28.16b, v28.16b mov w20, w24 st1 {v28.2d}, [x11] cmp w20, #8 blt L_aes_gcm_encrypt_arm64_crypto_nonce_start_dw ldr x19, [x3], #8 sub w20, w20, #8 str x19, [x11], #8 L_aes_gcm_encrypt_arm64_crypto_nonce_start_dw: cmp w20, #4 blt L_aes_gcm_encrypt_arm64_crypto_nonce_start_sw ldr w19, [x3], #4 sub w20, w20, #4 str w19, [x11], #4 L_aes_gcm_encrypt_arm64_crypto_nonce_start_sw: cmp w20, #2 blt L_aes_gcm_encrypt_arm64_crypto_nonce_start_byte ldrh w19, [x3], #2 sub w20, w20, #2 strh w19, [x11], #2 L_aes_gcm_encrypt_arm64_crypto_nonce_start_byte: cbz w20, L_aes_gcm_encrypt_arm64_crypto_nonce_end_bytes ldrb w19, [x3], #1 subs w20, w20, #1 strb w19, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_nonce_start_byte L_aes_gcm_encrypt_arm64_crypto_nonce_end_bytes: sub x11, x11, x24 ld1 {v18.2d}, [x11] rbit v18.16b, v18.16b eor v21.16b, v13.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v13.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_nonce_partial_done: eor x14, x14, x14 lsl x24, x4, #3 mov v28.d[0], x14 mov v28.d[1], x24 rev64 v28.16b, v28.16b rbit v28.16b, v28.16b eor v13.16b, v13.16b, v28.16b pmull v28.1q, v13.1d, v22.1d pmull2 v29.1q, v13.2d, v22.2d ext v31.16b, v13.16b, v13.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v13.16b, v28.16b, v30.16b rbit v13.16b, v13.16b mov w15, v13.s[3] rev w15, w15 L_aes_gcm_encrypt_arm64_crypto_done_nonce: st1 {v13.2d}, [x12] lsr w14, w2, #4 cmp w13, #12 blt L_aes_gcm_encrypt_arm64_crypto_start_128 bgt L_aes_gcm_encrypt_arm64_crypto_start_256 # AES_GCM_192 #ifndef NO_AES_192 cmp w14, #32 blt L_aes_gcm_encrypt_arm64_crypto_192_start_4 L_aes_gcm_encrypt_arm64_crypto_192_start_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rev w24, w24 rev w23, w23 rev w22, w22 rev w21, w21 rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 mov v14.s[3], w24 mov v15.s[3], w23 mov v16.s[3], w22 mov v17.s[3], w21 mov v8.s[3], w20 mov v9.s[3], w19 mov v10.s[3], w17 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v20.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v21.16b}, [x0], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v0.16b}, [x0], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v1.16b}, [x0], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v2.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v3.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b ld1 {v13.2d}, [x12] st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_encrypt_arm64_crypto_192_end_8 L_aes_gcm_encrypt_arm64_crypto_192_both_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w24, w24 rbit v19.16b, v19.16b rev w23, w23 rbit v20.16b, v20.16b rev w22, w22 rbit v21.16b, v21.16b rev w21, w21 rbit v0.16b, v0.16b rev w20, w20 rbit v1.16b, v1.16b rev w19, w19 rbit v2.16b, v2.16b rev w17, w17 rbit v3.16b, v3.16b rev w16, w15 mov v14.s[3], w24 mov v15.s[3], w23 mov v16.s[3], w22 mov v17.s[3], w21 mov v8.s[3], w20 mov v9.s[3], w19 mov v10.s[3], w17 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v31.16b, v3.16b, v3.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b ext v26.16b, v2.16b, v2.16b, #8 aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v26.16b, v26.16b, v31.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b ext v26.16b, v1.16b, v1.16b, #8 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b ext v26.16b, v0.16b, v0.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v31.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v30.16b, v30.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b ext v26.16b, v21.16b, v21.16b, #8 aese v9.16b, v13.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v26.16b, v26.16b, v31.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v8.16b, v12.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v26.16b, v26.16b, v31.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v30.16b, v30.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ext v26.16b, v19.16b, v19.16b, #8 aese v17.16b, v13.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v26.16b, v26.16b, v31.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v30.16b, v30.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull2 v29.1q, v29.2d, v27.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v31.16b, v31.16b, v30.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull2 v30.1q, v31.2d, v27.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b mov v28.d[1], v31.d[0] aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v18.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v19.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x0], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x0], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x12] eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_encrypt_arm64_crypto_192_both_8 L_aes_gcm_encrypt_arm64_crypto_192_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_192_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 ld1 {v12.2d}, [x9] cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_192_done beq L_aes_gcm_encrypt_arm64_crypto_192_start_1 cmp w14, #4 blt L_aes_gcm_encrypt_arm64_crypto_192_start_2 add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w19 mov v16.s[3], w17 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b ld1 {v19.16b}, [x0], #16 aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x0], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x0], #16 aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w14, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 blt L_aes_gcm_encrypt_arm64_crypto_192_end_4 L_aes_gcm_encrypt_arm64_crypto_192_both_4: add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w20, w20 rev w19, w19 rev w17, w17 rbit v19.16b, v19.16b rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w19 mov v16.s[3], w17 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b rbit v20.16b, v20.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b rbit v21.16b, v21.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b eor v18.16b, v18.16b, v26.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b ext v31.16b, v21.16b, v21.16b, #8 aese v15.16b, v1.16b aesmc v15.16b, v15.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v31.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b ext v26.16b, v20.16b, v20.16b, #8 aese v16.16b, v2.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v17.16b, v2.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v16.16b, v3.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b ext v26.16b, v19.16b, v19.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v4.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v5.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v17.16b, v5.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v29.1q, v29.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 eor v31.16b, v31.16b, v29.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v7.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v7.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w14, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 bge L_aes_gcm_encrypt_arm64_crypto_192_both_4 L_aes_gcm_encrypt_arm64_crypto_192_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w14, #1 beq L_aes_gcm_encrypt_arm64_crypto_192_start_1 blt L_aes_gcm_encrypt_arm64_crypto_192_done L_aes_gcm_encrypt_arm64_crypto_192_start_2: add w20, w15, #1 mov v14.16b, v13.16b add w15, w15, #2 mov v15.16b, v13.16b rev w20, w20 rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w14, w14, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x0], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b st1 {v18.16b, v19.16b}, [x1], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w14, L_aes_gcm_encrypt_arm64_crypto_192_done L_aes_gcm_encrypt_arm64_crypto_192_start_1: add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b eor v18.16b, v18.16b, v14.16b st1 {v18.16b}, [x1], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_192_done: ands w14, w2, #15 beq L_aes_gcm_encrypt_arm64_crypto_192_partial_done eor v16.16b, v16.16b, v16.16b mov w19, w14 st1 {v16.2d}, [x11] cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_192_start_dw ldr x17, [x0], #8 sub x19, x19, #8 str x17, [x11], #8 L_aes_gcm_encrypt_arm64_crypto_192_start_dw: cmp x19, #4 blt L_aes_gcm_encrypt_arm64_crypto_192_start_sw ldr w17, [x0], #4 sub x19, x19, #4 str w17, [x11], #4 L_aes_gcm_encrypt_arm64_crypto_192_start_sw: cmp x19, #2 blt L_aes_gcm_encrypt_arm64_crypto_192_start_byte ldrh w17, [x0], #2 sub x19, x19, #2 strh w17, [x11], #2 L_aes_gcm_encrypt_arm64_crypto_192_start_byte: cbz x19, L_aes_gcm_encrypt_arm64_crypto_192_end_bytes ldrb w17, [x0], #1 subs x19, x19, #1 strb w17, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_192_start_byte L_aes_gcm_encrypt_arm64_crypto_192_end_bytes: sub x11, x11, x14 ld1 {v16.2d}, [x11] add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b eor v16.16b, v16.16b, v14.16b st1 {v16.2d}, [x11] mov w19, w14 cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_192_out_start_dw ldr x17, [x11], #8 sub x19, x19, #8 str x17, [x1], #8 L_aes_gcm_encrypt_arm64_crypto_192_out_start_dw: cmp x19, #4 blt L_aes_gcm_encrypt_arm64_crypto_192_out_start_sw ldr w17, [x11], #4 sub x19, x19, #4 str w17, [x1], #4 L_aes_gcm_encrypt_arm64_crypto_192_out_start_sw: cmp x19, #2 blt L_aes_gcm_encrypt_arm64_crypto_192_out_start_byte ldrh w17, [x11], #2 sub x19, x19, #2 strh w17, [x1], #2 L_aes_gcm_encrypt_arm64_crypto_192_out_start_byte: cbz x19, L_aes_gcm_encrypt_arm64_crypto_192_out_end_bytes ldrb w17, [x11], #1 subs x19, x19, #1 strb w17, [x1], #1 bne L_aes_gcm_encrypt_arm64_crypto_192_out_start_byte L_aes_gcm_encrypt_arm64_crypto_192_out_end_bytes: mov x17, #16 sub x17, x17, x14 L_aes_gcm_encrypt_arm64_crypto_192_start_zero: subs x17, x17, #1 strb wzr, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_192_start_zero sub x11, x11, #16 ld1 {v14.2d}, [x11] rbit v14.16b, v14.16b eor v15.16b, v26.16b, v14.16b # X = C * H^1 pmull v28.1q, v15.1d, v22.1d pmull2 v29.1q, v15.2d, v22.2d ext v31.16b, v15.16b, v15.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_192_partial_done: ld1 {v14.2d}, [x12] lsl x8, x8, #3 rbit x8, x8 mov v28.d[0], x8 lsl x2, x2, #3 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b pmull v28.1q, v26.1d, v22.1d pmull2 v29.1q, v26.2d, v22.2d aese v14.16b, v0.16b aesmc v14.16b, v14.16b ext v31.16b, v26.16b, v26.16b, #8 aese v14.16b, v1.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v8.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v9.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b rbit v26.16b, v26.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b eor v26.16b, v26.16b, v14.16b cmp w6, #16 bne L_aes_gcm_encrypt_arm64_crypto_192_tag_partial st1 {v26.16b}, [x5] b L_aes_gcm_encrypt_arm64_crypto_done L_aes_gcm_encrypt_arm64_crypto_192_tag_partial: st1 {v26.16b}, [x11] cmp w6, #8 blt L_aes_gcm_encrypt_arm64_crypto_192_tag_start_dw ldr x16, [x11], #8 sub w6, w6, #8 str x16, [x5], #8 L_aes_gcm_encrypt_arm64_crypto_192_tag_start_dw: cmp w6, #4 blt L_aes_gcm_encrypt_arm64_crypto_192_tag_start_sw ldr w16, [x11], #4 sub w6, w6, #4 str w16, [x5], #4 L_aes_gcm_encrypt_arm64_crypto_192_tag_start_sw: cmp w6, #2 blt L_aes_gcm_encrypt_arm64_crypto_192_tag_start_byte ldrh w16, [x11], #2 sub w6, w6, #2 strh w16, [x5], #2 L_aes_gcm_encrypt_arm64_crypto_192_tag_start_byte: cbz w6, L_aes_gcm_encrypt_arm64_crypto_192_tag_end_bytes ldrb w16, [x11], #1 subs w6, w6, #1 strb w16, [x5], #1 bne L_aes_gcm_encrypt_arm64_crypto_192_tag_start_byte L_aes_gcm_encrypt_arm64_crypto_192_tag_end_bytes: #endif /* !NO_AES_192 */ b L_aes_gcm_encrypt_arm64_crypto_done # AES_GCM_256 L_aes_gcm_encrypt_arm64_crypto_start_256: #ifndef NO_AES_256 cmp w14, #32 blt L_aes_gcm_encrypt_arm64_crypto_256_start_4 L_aes_gcm_encrypt_arm64_crypto_256_start_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rev w24, w24 rev w23, w23 rev w22, w22 rev w21, w21 rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 mov v14.s[3], w24 mov v15.s[3], w23 mov v16.s[3], w22 mov v17.s[3], w21 mov v8.s[3], w20 mov v9.s[3], w19 mov v10.s[3], w17 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v20.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v21.16b}, [x0], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v0.16b}, [x0], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v1.16b}, [x0], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v2.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v3.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b ld1 {v13.2d}, [x12] st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_encrypt_arm64_crypto_256_end_8 L_aes_gcm_encrypt_arm64_crypto_256_both_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w24, w24 rbit v19.16b, v19.16b rev w23, w23 rbit v20.16b, v20.16b rev w22, w22 rbit v21.16b, v21.16b rev w21, w21 rbit v0.16b, v0.16b rev w20, w20 rbit v1.16b, v1.16b rev w19, w19 rbit v2.16b, v2.16b rev w17, w17 rbit v3.16b, v3.16b rev w16, w15 mov v14.s[3], w24 mov v15.s[3], w23 mov v16.s[3], w22 mov v17.s[3], w21 mov v8.s[3], w20 mov v9.s[3], w19 mov v10.s[3], w17 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v31.16b, v3.16b, v3.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b ext v26.16b, v2.16b, v2.16b, #8 aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v26.16b, v26.16b, v31.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b ext v26.16b, v1.16b, v1.16b, #8 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b ext v26.16b, v0.16b, v0.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v31.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v30.16b, v30.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b ext v26.16b, v21.16b, v21.16b, #8 aese v9.16b, v13.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v26.16b, v26.16b, v31.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v8.16b, v12.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v26.16b, v26.16b, v31.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v30.16b, v30.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ext v26.16b, v19.16b, v19.16b, #8 aese v17.16b, v13.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v26.16b, v26.16b, v31.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v30.16b, v30.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull2 v29.1q, v29.2d, v27.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v31.16b, v31.16b, v30.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull2 v30.1q, v31.2d, v27.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b mov v28.d[1], v31.d[0] aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v18.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v19.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x0], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x0], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x12] eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_encrypt_arm64_crypto_256_both_8 L_aes_gcm_encrypt_arm64_crypto_256_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_256_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 ld1 {v12.2d}, [x9], #16 cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_256_done beq L_aes_gcm_encrypt_arm64_crypto_256_start_1 cmp w14, #4 blt L_aes_gcm_encrypt_arm64_crypto_256_start_2 add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w19 mov v16.s[3], w17 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b ld1 {v19.16b}, [x0], #16 aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x0], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x0], #16 aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w14, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 blt L_aes_gcm_encrypt_arm64_crypto_256_end_4 L_aes_gcm_encrypt_arm64_crypto_256_both_4: add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w20, w20 rev w19, w19 rev w17, w17 rbit v19.16b, v19.16b rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w19 mov v16.s[3], w17 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b rbit v20.16b, v20.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b rbit v21.16b, v21.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b eor v18.16b, v18.16b, v26.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b ext v31.16b, v21.16b, v21.16b, #8 aese v15.16b, v1.16b aesmc v15.16b, v15.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v31.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b ext v26.16b, v20.16b, v20.16b, #8 aese v16.16b, v2.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v17.16b, v2.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v16.16b, v3.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b ext v26.16b, v19.16b, v19.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v4.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v5.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v17.16b, v5.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v29.1q, v29.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 eor v31.16b, v31.16b, v29.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v7.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v7.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w14, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 bge L_aes_gcm_encrypt_arm64_crypto_256_both_4 L_aes_gcm_encrypt_arm64_crypto_256_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w14, #1 beq L_aes_gcm_encrypt_arm64_crypto_256_start_1 blt L_aes_gcm_encrypt_arm64_crypto_256_done L_aes_gcm_encrypt_arm64_crypto_256_start_2: add w20, w15, #1 mov v14.16b, v13.16b add w15, w15, #2 mov v15.16b, v13.16b rev w20, w20 rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w14, w14, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x0], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b ld1 {v29.2d, v30.2d}, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b st1 {v18.16b, v19.16b}, [x1], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w14, L_aes_gcm_encrypt_arm64_crypto_256_done L_aes_gcm_encrypt_arm64_crypto_256_start_1: add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b ldr q29, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ldr q30, [x9, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v18.16b, v18.16b, v14.16b st1 {v18.16b}, [x1], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_256_done: ands w14, w2, #15 beq L_aes_gcm_encrypt_arm64_crypto_256_partial_done eor v16.16b, v16.16b, v16.16b mov w19, w14 st1 {v16.2d}, [x11] cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_256_start_dw ldr x17, [x0], #8 sub x19, x19, #8 str x17, [x11], #8 L_aes_gcm_encrypt_arm64_crypto_256_start_dw: cmp x19, #4 blt L_aes_gcm_encrypt_arm64_crypto_256_start_sw ldr w17, [x0], #4 sub x19, x19, #4 str w17, [x11], #4 L_aes_gcm_encrypt_arm64_crypto_256_start_sw: cmp x19, #2 blt L_aes_gcm_encrypt_arm64_crypto_256_start_byte ldrh w17, [x0], #2 sub x19, x19, #2 strh w17, [x11], #2 L_aes_gcm_encrypt_arm64_crypto_256_start_byte: cbz x19, L_aes_gcm_encrypt_arm64_crypto_256_end_bytes ldrb w17, [x0], #1 subs x19, x19, #1 strb w17, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_256_start_byte L_aes_gcm_encrypt_arm64_crypto_256_end_bytes: sub x11, x11, x14 ld1 {v16.2d}, [x11] add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b ldr q29, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ldr q30, [x9, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v16.16b, v16.16b, v14.16b st1 {v16.2d}, [x11] mov w19, w14 cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_256_out_start_dw ldr x17, [x11], #8 sub x19, x19, #8 str x17, [x1], #8 L_aes_gcm_encrypt_arm64_crypto_256_out_start_dw: cmp x19, #4 blt L_aes_gcm_encrypt_arm64_crypto_256_out_start_sw ldr w17, [x11], #4 sub x19, x19, #4 str w17, [x1], #4 L_aes_gcm_encrypt_arm64_crypto_256_out_start_sw: cmp x19, #2 blt L_aes_gcm_encrypt_arm64_crypto_256_out_start_byte ldrh w17, [x11], #2 sub x19, x19, #2 strh w17, [x1], #2 L_aes_gcm_encrypt_arm64_crypto_256_out_start_byte: cbz x19, L_aes_gcm_encrypt_arm64_crypto_256_out_end_bytes ldrb w17, [x11], #1 subs x19, x19, #1 strb w17, [x1], #1 bne L_aes_gcm_encrypt_arm64_crypto_256_out_start_byte L_aes_gcm_encrypt_arm64_crypto_256_out_end_bytes: mov x17, #16 sub x17, x17, x14 L_aes_gcm_encrypt_arm64_crypto_256_start_zero: subs x17, x17, #1 strb wzr, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_256_start_zero sub x11, x11, #16 ld1 {v14.2d}, [x11] rbit v14.16b, v14.16b eor v15.16b, v26.16b, v14.16b # X = C * H^1 pmull v28.1q, v15.1d, v22.1d pmull2 v29.1q, v15.2d, v22.2d ext v31.16b, v15.16b, v15.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_256_partial_done: ld1 {v14.2d}, [x12] lsl x8, x8, #3 rbit x8, x8 mov v28.d[0], x8 lsl x2, x2, #3 rbit x2, x2 mov v28.d[1], x2 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v28.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b pmull v28.1q, v26.1d, v22.1d pmull2 v29.1q, v26.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b ext v31.16b, v26.16b, v26.16b, #8 aese v14.16b, v3.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d ldr q11, [x9, #-32] aese v14.16b, v10.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] ldr q12, [x9, #-16] aese v14.16b, v11.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b ldr q29, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b rbit v26.16b, v26.16b ldr q30, [x9, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v26.16b, v26.16b, v14.16b cmp w6, #16 bne L_aes_gcm_encrypt_arm64_crypto_256_tag_partial st1 {v26.16b}, [x5] b L_aes_gcm_encrypt_arm64_crypto_done L_aes_gcm_encrypt_arm64_crypto_256_tag_partial: st1 {v26.16b}, [x11] cmp w6, #8 blt L_aes_gcm_encrypt_arm64_crypto_256_tag_start_dw ldr x16, [x11], #8 sub w6, w6, #8 str x16, [x5], #8 L_aes_gcm_encrypt_arm64_crypto_256_tag_start_dw: cmp w6, #4 blt L_aes_gcm_encrypt_arm64_crypto_256_tag_start_sw ldr w16, [x11], #4 sub w6, w6, #4 str w16, [x5], #4 L_aes_gcm_encrypt_arm64_crypto_256_tag_start_sw: cmp w6, #2 blt L_aes_gcm_encrypt_arm64_crypto_256_tag_start_byte ldrh w16, [x11], #2 sub w6, w6, #2 strh w16, [x5], #2 L_aes_gcm_encrypt_arm64_crypto_256_tag_start_byte: cbz w6, L_aes_gcm_encrypt_arm64_crypto_256_tag_end_bytes ldrb w16, [x11], #1 subs w6, w6, #1 strb w16, [x5], #1 bne L_aes_gcm_encrypt_arm64_crypto_256_tag_start_byte L_aes_gcm_encrypt_arm64_crypto_256_tag_end_bytes: #endif /* !NO_AES_256 */ b L_aes_gcm_encrypt_arm64_crypto_done # AES_GCM_128 L_aes_gcm_encrypt_arm64_crypto_start_128: #ifndef NO_AES_128 cmp w14, #32 blt L_aes_gcm_encrypt_arm64_crypto_128_start_4 L_aes_gcm_encrypt_arm64_crypto_128_start_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rev w24, w24 rev w23, w23 rev w22, w22 rev w21, w21 rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 mov v14.s[3], w24 mov v15.s[3], w23 mov v16.s[3], w22 mov v17.s[3], w21 mov v8.s[3], w20 mov v9.s[3], w19 mov v10.s[3], w17 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v20.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v21.16b}, [x0], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b ld1 {v0.16b}, [x0], #16 aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b ld1 {v1.16b}, [x0], #16 aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b ld1 {v2.16b}, [x0], #16 aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b ld1 {v3.16b}, [x0], #16 aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b ld1 {v13.2d}, [x12] st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_encrypt_arm64_crypto_128_end_8 L_aes_gcm_encrypt_arm64_crypto_128_both_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w24, w24 rbit v19.16b, v19.16b rev w23, w23 rbit v20.16b, v20.16b rev w22, w22 rbit v21.16b, v21.16b rev w21, w21 rbit v0.16b, v0.16b rev w20, w20 rbit v1.16b, v1.16b rev w19, w19 rbit v2.16b, v2.16b rev w17, w17 rbit v3.16b, v3.16b rev w16, w15 mov v14.s[3], w24 mov v15.s[3], w23 mov v16.s[3], w22 mov v17.s[3], w21 mov v8.s[3], w20 mov v9.s[3], w19 mov v10.s[3], w17 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v31.16b, v3.16b, v3.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b ext v26.16b, v2.16b, v2.16b, #8 aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v26.16b, v26.16b, v31.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b ext v26.16b, v1.16b, v1.16b, #8 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b ext v26.16b, v0.16b, v0.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v31.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v30.16b, v30.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b ext v26.16b, v21.16b, v21.16b, #8 aese v9.16b, v13.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v26.16b, v26.16b, v31.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v8.16b, v12.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v26.16b, v26.16b, v31.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v30.16b, v30.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ext v26.16b, v19.16b, v19.16b, #8 aese v17.16b, v13.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v26.16b, v26.16b, v31.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v30.16b, v30.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull2 v29.1q, v29.2d, v27.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v31.16b, v31.16b, v30.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull2 v30.1q, v31.2d, v27.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b mov v28.d[1], v31.d[0] aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v18.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v19.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x0], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x0], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x12] eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_encrypt_arm64_crypto_128_both_8 L_aes_gcm_encrypt_arm64_crypto_128_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_128_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 ld1 {v8.2d, v9.2d}, [x9], #32 ld1 {v10.2d}, [x9] cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_128_done beq L_aes_gcm_encrypt_arm64_crypto_128_start_1 cmp w14, #4 blt L_aes_gcm_encrypt_arm64_crypto_128_start_2 add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w19 mov v16.s[3], w17 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b ld1 {v19.16b}, [x0], #16 aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v20.16b}, [x0], #16 aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b ld1 {v21.16b}, [x0], #16 aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w14, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 blt L_aes_gcm_encrypt_arm64_crypto_128_end_4 L_aes_gcm_encrypt_arm64_crypto_128_both_4: add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w20, w20 rev w19, w19 rev w17, w17 rbit v19.16b, v19.16b rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w19 mov v16.s[3], w17 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b rbit v20.16b, v20.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b rbit v21.16b, v21.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b eor v18.16b, v18.16b, v26.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b ext v31.16b, v21.16b, v21.16b, #8 aese v15.16b, v1.16b aesmc v15.16b, v15.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v31.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b ext v26.16b, v20.16b, v20.16b, #8 aese v16.16b, v2.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v17.16b, v2.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v16.16b, v3.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b ext v26.16b, v19.16b, v19.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v4.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v5.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v17.16b, v5.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v29.1q, v29.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 eor v31.16b, v31.16b, v29.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v7.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v7.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w14, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 bge L_aes_gcm_encrypt_arm64_crypto_128_both_4 L_aes_gcm_encrypt_arm64_crypto_128_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w14, #1 beq L_aes_gcm_encrypt_arm64_crypto_128_start_1 blt L_aes_gcm_encrypt_arm64_crypto_128_done L_aes_gcm_encrypt_arm64_crypto_128_start_2: add w20, w15, #1 mov v14.16b, v13.16b add w15, w15, #2 mov v15.16b, v13.16b rev w20, w20 rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w14, w14, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v19.16b}, [x0], #16 aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b st1 {v18.16b, v19.16b}, [x1], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w14, L_aes_gcm_encrypt_arm64_crypto_128_done L_aes_gcm_encrypt_arm64_crypto_128_start_1: add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v18.16b}, [x0], #16 eor v18.16b, v18.16b, v14.16b st1 {v18.16b}, [x1], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_128_done: ands w14, w2, #15 beq L_aes_gcm_encrypt_arm64_crypto_128_partial_done eor v16.16b, v16.16b, v16.16b mov w19, w14 st1 {v16.2d}, [x11] cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_128_start_dw ldr x17, [x0], #8 sub x19, x19, #8 str x17, [x11], #8 L_aes_gcm_encrypt_arm64_crypto_128_start_dw: cmp x19, #4 blt L_aes_gcm_encrypt_arm64_crypto_128_start_sw ldr w17, [x0], #4 sub x19, x19, #4 str w17, [x11], #4 L_aes_gcm_encrypt_arm64_crypto_128_start_sw: cmp x19, #2 blt L_aes_gcm_encrypt_arm64_crypto_128_start_byte ldrh w17, [x0], #2 sub x19, x19, #2 strh w17, [x11], #2 L_aes_gcm_encrypt_arm64_crypto_128_start_byte: cbz x19, L_aes_gcm_encrypt_arm64_crypto_128_end_bytes ldrb w17, [x0], #1 subs x19, x19, #1 strb w17, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_128_start_byte L_aes_gcm_encrypt_arm64_crypto_128_end_bytes: sub x11, x11, x14 ld1 {v16.2d}, [x11] add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b eor v16.16b, v16.16b, v14.16b st1 {v16.2d}, [x11] mov w19, w14 cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_128_out_start_dw ldr x17, [x11], #8 sub x19, x19, #8 str x17, [x1], #8 L_aes_gcm_encrypt_arm64_crypto_128_out_start_dw: cmp x19, #4 blt L_aes_gcm_encrypt_arm64_crypto_128_out_start_sw ldr w17, [x11], #4 sub x19, x19, #4 str w17, [x1], #4 L_aes_gcm_encrypt_arm64_crypto_128_out_start_sw: cmp x19, #2 blt L_aes_gcm_encrypt_arm64_crypto_128_out_start_byte ldrh w17, [x11], #2 sub x19, x19, #2 strh w17, [x1], #2 L_aes_gcm_encrypt_arm64_crypto_128_out_start_byte: cbz x19, L_aes_gcm_encrypt_arm64_crypto_128_out_end_bytes ldrb w17, [x11], #1 subs x19, x19, #1 strb w17, [x1], #1 bne L_aes_gcm_encrypt_arm64_crypto_128_out_start_byte L_aes_gcm_encrypt_arm64_crypto_128_out_end_bytes: mov x17, #16 sub x17, x17, x14 L_aes_gcm_encrypt_arm64_crypto_128_start_zero: subs x17, x17, #1 strb wzr, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_128_start_zero sub x11, x11, #16 ld1 {v14.2d}, [x11] rbit v14.16b, v14.16b eor v15.16b, v26.16b, v14.16b # X = C * H^1 pmull v28.1q, v15.1d, v22.1d pmull2 v29.1q, v15.2d, v22.2d ext v31.16b, v15.16b, v15.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_128_partial_done: ld1 {v14.2d}, [x12] lsl x8, x8, #3 rbit x8, x8 mov v28.d[0], x8 lsl x2, x2, #3 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b pmull v28.1q, v26.1d, v22.1d pmull2 v29.1q, v26.2d, v22.2d ext v31.16b, v26.16b, v26.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v6.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b rbit v26.16b, v26.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b eor v26.16b, v26.16b, v14.16b cmp w6, #16 bne L_aes_gcm_encrypt_arm64_crypto_128_tag_partial st1 {v26.16b}, [x5] b L_aes_gcm_encrypt_arm64_crypto_done L_aes_gcm_encrypt_arm64_crypto_128_tag_partial: st1 {v26.16b}, [x11] cmp w6, #8 blt L_aes_gcm_encrypt_arm64_crypto_128_tag_start_dw ldr x16, [x11], #8 sub w6, w6, #8 str x16, [x5], #8 L_aes_gcm_encrypt_arm64_crypto_128_tag_start_dw: cmp w6, #4 blt L_aes_gcm_encrypt_arm64_crypto_128_tag_start_sw ldr w16, [x11], #4 sub w6, w6, #4 str w16, [x5], #4 L_aes_gcm_encrypt_arm64_crypto_128_tag_start_sw: cmp w6, #2 blt L_aes_gcm_encrypt_arm64_crypto_128_tag_start_byte ldrh w16, [x11], #2 sub w6, w6, #2 strh w16, [x5], #2 L_aes_gcm_encrypt_arm64_crypto_128_tag_start_byte: cbz w6, L_aes_gcm_encrypt_arm64_crypto_128_tag_end_bytes ldrb w16, [x11], #1 subs w6, w6, #1 strb w16, [x5], #1 bne L_aes_gcm_encrypt_arm64_crypto_128_tag_start_byte L_aes_gcm_encrypt_arm64_crypto_128_tag_end_bytes: #endif /* !NO_AES_128 */ L_aes_gcm_encrypt_arm64_crypto_done: ldp x17, x19, [x29, #24] ldp x20, x21, [x29, #40] ldp x22, x23, [x29, #56] ldr x24, [x29, #72] ldp d8, d9, [x29, #80] ldp d10, d11, [x29, #96] ldp d12, d13, [x29, #112] ldp d14, d15, [x29, #128] ldp x29, x30, [sp], #0x90 ret #ifndef __APPLE__ .size AES_GCM_encrypt_AARCH64,.-AES_GCM_encrypt_AARCH64 #endif /* __APPLE__ */ #ifdef HAVE_AES_DECRYPT #ifndef __APPLE__ .text .globl AES_GCM_decrypt_AARCH64 .type AES_GCM_decrypt_AARCH64,@function .align 2 AES_GCM_decrypt_AARCH64: #else .section __TEXT,__text .globl _AES_GCM_decrypt_AARCH64 .p2align 2 _AES_GCM_decrypt_AARCH64: #endif /* __APPLE__ */ stp x29, x30, [sp, #-144]! add x29, sp, #0 stp x17, x19, [x29, #24] stp x20, x21, [x29, #40] stp x22, x23, [x29, #56] str x24, [x29, #72] stp d8, d9, [x29, #80] stp d10, d11, [x29, #96] stp d12, d13, [x29, #112] stp d14, d15, [x29, #128] ldr w8, [x29, #144] ldr x9, [x29, #152] ldr x10, [x29, #160] ldr x11, [x29, #168] ldr x12, [x29, #176] ldr w13, [x29, #184] movi v27.16b, #0x87 eor v26.16b, v26.16b, v26.16b ushr v27.2d, v27.2d, #56 ld1 {v22.2d}, [x10] cmp w8, #0x40 csetm x16, lt cmp w2, #32 csetm x17, lt ands x16, x16, x17 bne L_aes_gcm_decrypt_arm64_crypto_h_done # Square H => H^2 pmull2 v31.1q, v22.2d, v22.2d pmull v30.1q, v22.1d, v22.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v23.16b, v30.16b, v31.16b cmp w8, #0x100 csetm x16, lt cmp w2, #0x40 csetm x17, lt ands x16, x16, x17 bne L_aes_gcm_decrypt_arm64_crypto_h_done # Multiply H and H^2 => H^3 pmull v28.1q, v22.1d, v23.1d pmull2 v29.1q, v22.2d, v23.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v23.1d pmull2 v31.1q, v31.2d, v23.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v24.16b, v28.16b, v30.16b # Square H^2 => H^4 pmull2 v31.1q, v23.2d, v23.2d pmull v30.1q, v23.1d, v23.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v25.16b, v30.16b, v31.16b # Done cmp w8, #0x400 csetm x16, lt cmp w2, #0x200 csetm x17, lt ands x16, x16, x17 bne L_aes_gcm_decrypt_arm64_crypto_h_done # Multiply H and H^4 => H^5 pmull v28.1q, v22.1d, v25.1d pmull2 v29.1q, v22.2d, v25.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v25.1d pmull2 v31.1q, v31.2d, v25.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v4.16b, v28.16b, v30.16b # Square H^3 => H^6 pmull2 v31.1q, v24.2d, v24.2d pmull v30.1q, v24.1d, v24.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v5.16b, v30.16b, v31.16b # Multiply H and H^6 => H^7 pmull v28.1q, v22.1d, v5.1d pmull2 v29.1q, v22.2d, v5.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v5.1d pmull2 v31.1q, v31.2d, v5.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v6.16b, v28.16b, v30.16b # Square H^4 => H^8 pmull2 v31.1q, v25.2d, v25.2d pmull v30.1q, v25.1d, v25.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v7.16b, v30.16b, v31.16b # Done L_aes_gcm_decrypt_arm64_crypto_h_done: lsr w14, w8, #4 cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_aad_start_1 cmp w14, #16 blt L_aes_gcm_decrypt_arm64_crypto_aad_start_2 cmp w14, #0x40 blt L_aes_gcm_decrypt_arm64_crypto_aad_start_4 L_aes_gcm_decrypt_arm64_crypto_aad_start_8: ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x7], #0x40 rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH sub w14, w14, #8 cmp w14, #8 bge L_aes_gcm_decrypt_arm64_crypto_aad_start_8 cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_aad_done beq L_aes_gcm_decrypt_arm64_crypto_aad_start_1 cmp w14, #16 blt L_aes_gcm_decrypt_arm64_crypto_aad_start_2 L_aes_gcm_decrypt_arm64_crypto_aad_start_4: ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH sub w14, w14, #4 cmp w14, #4 bge L_aes_gcm_decrypt_arm64_crypto_aad_start_4 cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_aad_done beq L_aes_gcm_decrypt_arm64_crypto_aad_start_1 L_aes_gcm_decrypt_arm64_crypto_aad_start_2: ld1 {v18.16b, v19.16b}, [x7], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH sub w14, w14, #2 cmp w14, #1 bgt L_aes_gcm_decrypt_arm64_crypto_aad_start_2 blt L_aes_gcm_decrypt_arm64_crypto_aad_done L_aes_gcm_decrypt_arm64_crypto_aad_start_1: cbz w14, L_aes_gcm_decrypt_arm64_crypto_aad_done L_aes_gcm_decrypt_arm64_crypto_aad_both_1: ld1 {v18.16b}, [x7], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH subs w14, w14, #1 bne L_aes_gcm_decrypt_arm64_crypto_aad_both_1 L_aes_gcm_decrypt_arm64_crypto_aad_done: and w14, w8, #15 cbz w14, L_aes_gcm_decrypt_arm64_crypto_aad_partial_done eor v28.16b, v28.16b, v28.16b mov w20, w14 st1 {v28.2d}, [x11] cmp w20, #8 blt L_aes_gcm_decrypt_arm64_crypto_aad_start_dw ldr x19, [x7], #8 sub w20, w20, #8 str x19, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_aad_start_dw: cmp w20, #4 blt L_aes_gcm_decrypt_arm64_crypto_aad_start_sw ldr w19, [x7], #4 sub w20, w20, #4 str w19, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_aad_start_sw: cmp w20, #2 blt L_aes_gcm_decrypt_arm64_crypto_aad_start_byte ldrh w19, [x7], #2 sub w20, w20, #2 strh w19, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_aad_start_byte: cbz w20, L_aes_gcm_decrypt_arm64_crypto_aad_end_bytes ldrb w19, [x7], #1 subs w20, w20, #1 strb w19, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_aad_start_byte L_aes_gcm_decrypt_arm64_crypto_aad_end_bytes: sub x11, x11, x14 ld1 {v18.2d}, [x11] rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_arm64_crypto_aad_partial_done: # Load Nonce cmp w4, #12 bne L_aes_gcm_decrypt_arm64_crypto_ghash_nonce ldr x16, [x3] movi v13.4s, #1, lsl 24 ldr w17, [x3, #8] mov v13.d[0], x16 mov v13.s[2], w17 mov w15, #1 b L_aes_gcm_decrypt_arm64_crypto_done_nonce L_aes_gcm_decrypt_arm64_crypto_ghash_nonce: eor v13.16b, v13.16b, v13.16b lsr w14, w4, #4 cbz w14, L_aes_gcm_decrypt_arm64_crypto_nonce_done L_aes_gcm_decrypt_arm64_crypto_nonce_start_1: ld1 {v18.16b}, [x3], #16 rbit v18.16b, v18.16b eor v21.16b, v13.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v13.16b, v28.16b, v30.16b # Done GHASH subs w14, w14, #1 bne L_aes_gcm_decrypt_arm64_crypto_nonce_start_1 L_aes_gcm_decrypt_arm64_crypto_nonce_done: and w24, w4, #15 cbz x24, L_aes_gcm_decrypt_arm64_crypto_nonce_partial_done eor v28.16b, v28.16b, v28.16b mov w20, w24 st1 {v28.2d}, [x11] cmp w20, #8 blt L_aes_gcm_decrypt_arm64_crypto_nonce_start_dw ldr x19, [x3], #8 sub w20, w20, #8 str x19, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_nonce_start_dw: cmp w20, #4 blt L_aes_gcm_decrypt_arm64_crypto_nonce_start_sw ldr w19, [x3], #4 sub w20, w20, #4 str w19, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_nonce_start_sw: cmp w20, #2 blt L_aes_gcm_decrypt_arm64_crypto_nonce_start_byte ldrh w19, [x3], #2 sub w20, w20, #2 strh w19, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_nonce_start_byte: cbz w20, L_aes_gcm_decrypt_arm64_crypto_nonce_end_bytes ldrb w19, [x3], #1 subs w20, w20, #1 strb w19, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_nonce_start_byte L_aes_gcm_decrypt_arm64_crypto_nonce_end_bytes: sub x11, x11, x24 ld1 {v18.2d}, [x11] rbit v18.16b, v18.16b eor v21.16b, v13.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v13.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_arm64_crypto_nonce_partial_done: eor x14, x14, x14 lsl x24, x4, #3 mov v28.d[0], x14 mov v28.d[1], x24 rev64 v28.16b, v28.16b rbit v28.16b, v28.16b eor v13.16b, v13.16b, v28.16b pmull v28.1q, v13.1d, v22.1d pmull2 v29.1q, v13.2d, v22.2d ext v31.16b, v13.16b, v13.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v13.16b, v28.16b, v30.16b rbit v13.16b, v13.16b mov w15, v13.s[3] rev w15, w15 L_aes_gcm_decrypt_arm64_crypto_done_nonce: st1 {v13.2d}, [x12] lsr w14, w2, #4 cmp w13, #12 blt L_aes_gcm_decrypt_arm64_crypto_start_128 bgt L_aes_gcm_decrypt_arm64_crypto_start_256 # AES_GCM_192 #ifndef NO_AES_192 cmp w14, #32 blt L_aes_gcm_decrypt_arm64_crypto_192_start_4 L_aes_gcm_decrypt_arm64_crypto_192_start_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rev w24, w24 mov v14.s[3], w24 rev w23, w23 mov v15.s[3], w23 rev w22, w22 mov v16.s[3], w22 rev w21, w21 mov v17.s[3], w21 rev w20, w20 mov v8.s[3], w20 rev w19, w19 mov v9.s[3], w19 rev w17, w17 mov v10.s[3], w17 rev w16, w15 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v18.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v19.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v20.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v21.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x0], #16 aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x0], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x0], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b ld1 {v13.2d}, [x12] st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_192_end_8 L_aes_gcm_decrypt_arm64_crypto_192_both_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w24, w24 rbit v19.16b, v19.16b mov v14.s[3], w24 rev w23, w23 rbit v20.16b, v20.16b mov v15.s[3], w23 rev w22, w22 rbit v21.16b, v21.16b mov v16.s[3], w22 rev w21, w21 rbit v0.16b, v0.16b mov v17.s[3], w21 rev w20, w20 rbit v1.16b, v1.16b mov v8.s[3], w20 rev w19, w19 rbit v2.16b, v2.16b mov v9.s[3], w19 rev w17, w17 rbit v3.16b, v3.16b mov v10.s[3], w17 rev w16, w15 eor v18.16b, v18.16b, v26.16b mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v31.16b, v3.16b, v3.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b ext v26.16b, v2.16b, v2.16b, #8 aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b ext v26.16b, v1.16b, v1.16b, #8 aese v10.16b, v13.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v31.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v30.16b, v30.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b ext v26.16b, v0.16b, v0.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v26.16b, v26.16b, v31.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ext v26.16b, v21.16b, v21.16b, #8 aese v8.16b, v13.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v26.16b, v26.16b, v31.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v30.16b, v30.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v26.16b, v20.16b, v20.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v26.16b, v26.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v30.16b, v30.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ext v26.16b, v18.16b, v18.16b, #8 aese v15.16b, v12.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v26.16b, v26.16b, v31.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull2 v29.1q, v29.2d, v27.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v31.16b, v31.16b, v29.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v18.16b}, [x0], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v19.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v20.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v21.16b}, [x0], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v0.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v1.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v2.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v3.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x12] eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_decrypt_arm64_crypto_192_both_8 L_aes_gcm_decrypt_arm64_crypto_192_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_arm64_crypto_192_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 ld1 {v12.2d}, [x9] cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_192_done beq L_aes_gcm_decrypt_arm64_crypto_192_start_1 cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_192_start_2 add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rev w20, w20 mov v14.s[3], w20 rev w19, w19 mov v15.s[3], w19 rev w17, w17 mov v16.s[3], w17 rev w16, w15 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x0], #16 aese v17.16b, v7.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x0], #16 aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v20.16b}, [x0], #16 aese v17.16b, v8.16b aesmc v17.16b, v17.16b ld1 {v21.16b}, [x0], #16 aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w14, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 blt L_aes_gcm_decrypt_arm64_crypto_192_end_4 L_aes_gcm_decrypt_arm64_crypto_192_both_4: add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w20, w20 rbit v19.16b, v19.16b mov v14.s[3], w20 rev w19, w19 rbit v20.16b, v20.16b mov v15.s[3], w19 rev w17, w17 rbit v21.16b, v21.16b mov v16.s[3], w17 rev w16, w15 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v16.16b, v0.16b aesmc v16.16b, v16.16b ext v31.16b, v21.16b, v21.16b, #8 aese v17.16b, v0.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v2.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v3.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v17.16b, v3.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b ext v26.16b, v18.16b, v18.16b, #8 aese v14.16b, v5.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v15.16b, v5.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v15.16b, v6.16b aesmc v15.16b, v15.16b eor v31.16b, v31.16b, v29.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b eor v31.16b, v31.16b, v30.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v15.16b, v7.16b aesmc v15.16b, v15.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w14, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 bge L_aes_gcm_decrypt_arm64_crypto_192_both_4 L_aes_gcm_decrypt_arm64_crypto_192_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w14, #1 beq L_aes_gcm_decrypt_arm64_crypto_192_start_1 blt L_aes_gcm_decrypt_arm64_crypto_192_done L_aes_gcm_decrypt_arm64_crypto_192_start_2: add w20, w15, #1 mov v14.16b, v13.16b add w15, w15, #2 mov v15.16b, v13.16b rev w20, w20 mov v14.s[3], w20 rev w16, w15 mov v15.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w14, w14, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x0], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b st1 {v14.16b, v15.16b}, [x1], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w14, L_aes_gcm_decrypt_arm64_crypto_192_done L_aes_gcm_decrypt_arm64_crypto_192_start_1: ld1 {v15.16b}, [x0], #16 add w15, w15, #1 mov v14.16b, v13.16b rbit v15.16b, v15.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v16.16b, v26.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v16.1d, v22.1d pmull2 v29.1q, v16.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b ext v31.16b, v16.16b, v16.16b, #8 aese v14.16b, v3.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v10.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b eor v26.16b, v28.16b, v30.16b # Done GHASH rbit v15.16b, v15.16b eor v14.16b, v14.16b, v15.16b st1 {v14.16b}, [x1], #16 L_aes_gcm_decrypt_arm64_crypto_192_done: ands w14, w2, #15 beq L_aes_gcm_decrypt_arm64_crypto_192_partial_done eor v15.16b, v15.16b, v15.16b mov w19, w14 st1 {v15.2d}, [x11] cmp x19, #8 blt L_aes_gcm_decrypt_arm64_crypto_192_start_dw ldr x17, [x0], #8 sub x19, x19, #8 str x17, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_192_start_dw: cmp x19, #4 blt L_aes_gcm_decrypt_arm64_crypto_192_start_sw ldr w17, [x0], #4 sub x19, x19, #4 str w17, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_192_start_sw: cmp x19, #2 blt L_aes_gcm_decrypt_arm64_crypto_192_start_byte ldrh w17, [x0], #2 sub x19, x19, #2 strh w17, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_192_start_byte: cbz x19, L_aes_gcm_decrypt_arm64_crypto_192_end_bytes ldrb w17, [x0], #1 subs x19, x19, #1 strb w17, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_192_start_byte L_aes_gcm_decrypt_arm64_crypto_192_end_bytes: sub x11, x11, x14 ld1 {v15.2d}, [x11] add w15, w15, #1 mov v14.16b, v13.16b rbit v15.16b, v15.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v16.16b, v26.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v16.1d, v22.1d pmull2 v29.1q, v16.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b ext v31.16b, v16.16b, v16.16b, #8 aese v14.16b, v3.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v10.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b eor v26.16b, v28.16b, v30.16b # Done GHASH rbit v15.16b, v15.16b eor v14.16b, v14.16b, v15.16b st1 {v14.2d}, [x11] cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_192_out_start_dw ldr x17, [x11], #8 sub w14, w14, #8 str x17, [x1], #8 L_aes_gcm_decrypt_arm64_crypto_192_out_start_dw: cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_192_out_start_sw ldr w17, [x11], #4 sub w14, w14, #4 str w17, [x1], #4 L_aes_gcm_decrypt_arm64_crypto_192_out_start_sw: cmp w14, #2 blt L_aes_gcm_decrypt_arm64_crypto_192_out_start_byte ldrh w17, [x11], #2 sub w14, w14, #2 strh w17, [x1], #2 L_aes_gcm_decrypt_arm64_crypto_192_out_start_byte: cbz w14, L_aes_gcm_decrypt_arm64_crypto_192_out_end_bytes ldrb w17, [x11], #1 subs w14, w14, #1 strb w17, [x1], #1 bne L_aes_gcm_decrypt_arm64_crypto_192_out_start_byte L_aes_gcm_decrypt_arm64_crypto_192_out_end_bytes: L_aes_gcm_decrypt_arm64_crypto_192_partial_done: ld1 {v14.2d}, [x12] lsl x8, x8, #3 rbit x8, x8 mov v28.d[0], x8 lsl x2, x2, #3 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b pmull v28.1q, v26.1d, v22.1d pmull2 v29.1q, v26.2d, v22.2d aese v14.16b, v0.16b aesmc v14.16b, v14.16b ext v31.16b, v26.16b, v26.16b, #8 aese v14.16b, v1.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v8.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v9.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b rbit v26.16b, v26.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b eor v26.16b, v26.16b, v14.16b cmp w6, #16 blt L_aes_gcm_decrypt_arm64_crypto_192_part_tag ld1 {v28.16b}, [x5] b L_aes_gcm_decrypt_arm64_crypto_192_tag_loaded L_aes_gcm_decrypt_arm64_crypto_192_part_tag: eor v28.16b, v28.16b, v28.16b mov x17, x6 st1 {v28.2d}, [x11] cmp x17, #8 blt L_aes_gcm_decrypt_arm64_crypto_192_tag_start_dw ldr x16, [x5], #8 sub x17, x17, #8 str x16, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_192_tag_start_dw: cmp x17, #4 blt L_aes_gcm_decrypt_arm64_crypto_192_tag_start_sw ldr w16, [x5], #4 sub x17, x17, #4 str w16, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_192_tag_start_sw: cmp x17, #2 blt L_aes_gcm_decrypt_arm64_crypto_192_tag_start_byte ldrh w16, [x5], #2 sub x17, x17, #2 strh w16, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_192_tag_start_byte: cbz x17, L_aes_gcm_decrypt_arm64_crypto_192_tag_end_bytes ldrb w16, [x5], #1 subs x17, x17, #1 strb w16, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_192_tag_start_byte L_aes_gcm_decrypt_arm64_crypto_192_tag_end_bytes: sub x11, x11, x6 ld1 {v28.2d}, [x11] mov x17, #16 st1 {v26.2d}, [x11] sub x17, x17, x6 add x11, x11, x6 L_aes_gcm_decrypt_arm64_crypto_192_calc_tag_byte: strb wzr, [x11], #1 subs x17, x17, #1 bne L_aes_gcm_decrypt_arm64_crypto_192_calc_tag_byte subs x11, x11, #16 ld1 {v26.2d}, [x11] L_aes_gcm_decrypt_arm64_crypto_192_tag_loaded: eor v28.16b, v28.16b, v26.16b mov x16, v28.d[0] mov x17, v28.d[1] mov w19, #-180 orr x16, x16, x17 cmp x16, #0 csetm x0, ne and x0, x0, x19 #endif /* !NO_AES_192 */ b L_aes_gcm_decrypt_arm64_crypto_done # AES_GCM_256 L_aes_gcm_decrypt_arm64_crypto_start_256: #ifndef NO_AES_256 cmp w14, #32 blt L_aes_gcm_decrypt_arm64_crypto_256_start_4 L_aes_gcm_decrypt_arm64_crypto_256_start_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rev w24, w24 mov v14.s[3], w24 rev w23, w23 mov v15.s[3], w23 rev w22, w22 mov v16.s[3], w22 rev w21, w21 mov v17.s[3], w21 rev w20, w20 mov v8.s[3], w20 rev w19, w19 mov v9.s[3], w19 rev w17, w17 mov v10.s[3], w17 rev w16, w15 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v18.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v19.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v20.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v21.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x0], #16 aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x0], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x0], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b ld1 {v13.2d}, [x12] st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_256_end_8 L_aes_gcm_decrypt_arm64_crypto_256_both_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w24, w24 rbit v19.16b, v19.16b mov v14.s[3], w24 rev w23, w23 rbit v20.16b, v20.16b mov v15.s[3], w23 rev w22, w22 rbit v21.16b, v21.16b mov v16.s[3], w22 rev w21, w21 rbit v0.16b, v0.16b mov v17.s[3], w21 rev w20, w20 rbit v1.16b, v1.16b mov v8.s[3], w20 rev w19, w19 rbit v2.16b, v2.16b mov v9.s[3], w19 rev w17, w17 rbit v3.16b, v3.16b mov v10.s[3], w17 rev w16, w15 eor v18.16b, v18.16b, v26.16b mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v31.16b, v3.16b, v3.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b ext v26.16b, v2.16b, v2.16b, #8 aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b ext v26.16b, v1.16b, v1.16b, #8 aese v10.16b, v13.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v31.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v30.16b, v30.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b ext v26.16b, v0.16b, v0.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v26.16b, v26.16b, v31.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ext v26.16b, v21.16b, v21.16b, #8 aese v8.16b, v13.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v26.16b, v26.16b, v31.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v30.16b, v30.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v26.16b, v20.16b, v20.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v26.16b, v26.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v30.16b, v30.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ext v26.16b, v18.16b, v18.16b, #8 aese v15.16b, v12.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v26.16b, v26.16b, v31.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull2 v29.1q, v29.2d, v27.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v31.16b, v31.16b, v29.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v18.16b}, [x0], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v19.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v20.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v21.16b}, [x0], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v0.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v1.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v2.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v3.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x12] eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_decrypt_arm64_crypto_256_both_8 L_aes_gcm_decrypt_arm64_crypto_256_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_arm64_crypto_256_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 ld1 {v12.2d}, [x9], #16 cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_256_done beq L_aes_gcm_decrypt_arm64_crypto_256_start_1 cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_256_start_2 add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rev w20, w20 mov v14.s[3], w20 rev w19, w19 mov v15.s[3], w19 rev w17, w17 mov v16.s[3], w17 rev w16, w15 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x0], #16 aese v17.16b, v7.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x0], #16 aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v20.16b}, [x0], #16 aese v17.16b, v8.16b aesmc v17.16b, v17.16b ld1 {v21.16b}, [x0], #16 aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w14, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 blt L_aes_gcm_decrypt_arm64_crypto_256_end_4 L_aes_gcm_decrypt_arm64_crypto_256_both_4: add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w20, w20 rbit v19.16b, v19.16b mov v14.s[3], w20 rev w19, w19 rbit v20.16b, v20.16b mov v15.s[3], w19 rev w17, w17 rbit v21.16b, v21.16b mov v16.s[3], w17 rev w16, w15 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v16.16b, v0.16b aesmc v16.16b, v16.16b ext v31.16b, v21.16b, v21.16b, #8 aese v17.16b, v0.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v2.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v3.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v17.16b, v3.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b ext v26.16b, v18.16b, v18.16b, #8 aese v14.16b, v5.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v15.16b, v5.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v15.16b, v6.16b aesmc v15.16b, v15.16b eor v31.16b, v31.16b, v29.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b eor v31.16b, v31.16b, v30.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v15.16b, v7.16b aesmc v15.16b, v15.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w14, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 bge L_aes_gcm_decrypt_arm64_crypto_256_both_4 L_aes_gcm_decrypt_arm64_crypto_256_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w14, #1 beq L_aes_gcm_decrypt_arm64_crypto_256_start_1 blt L_aes_gcm_decrypt_arm64_crypto_256_done L_aes_gcm_decrypt_arm64_crypto_256_start_2: add w20, w15, #1 mov v14.16b, v13.16b add w15, w15, #2 mov v15.16b, v13.16b rev w20, w20 mov v14.s[3], w20 rev w16, w15 mov v15.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w14, w14, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x0], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b ld1 {v29.2d, v30.2d}, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b st1 {v14.16b, v15.16b}, [x1], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w14, L_aes_gcm_decrypt_arm64_crypto_256_done L_aes_gcm_decrypt_arm64_crypto_256_start_1: add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v14.16b, v11.16b aesmc v14.16b, v14.16b ldr q29, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ldr q30, [x9, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v14.16b, v14.16b, v18.16b st1 {v14.16b}, [x1], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_arm64_crypto_256_done: ands w14, w2, #15 beq L_aes_gcm_decrypt_arm64_crypto_256_partial_done eor v15.16b, v15.16b, v15.16b mov w19, w14 st1 {v15.2d}, [x11] cmp x19, #8 blt L_aes_gcm_decrypt_arm64_crypto_256_start_dw ldr x17, [x0], #8 sub x19, x19, #8 str x17, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_256_start_dw: cmp x19, #4 blt L_aes_gcm_decrypt_arm64_crypto_256_start_sw ldr w17, [x0], #4 sub x19, x19, #4 str w17, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_256_start_sw: cmp x19, #2 blt L_aes_gcm_decrypt_arm64_crypto_256_start_byte ldrh w17, [x0], #2 sub x19, x19, #2 strh w17, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_256_start_byte: cbz x19, L_aes_gcm_decrypt_arm64_crypto_256_end_bytes ldrb w17, [x0], #1 subs x19, x19, #1 strb w17, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_256_start_byte L_aes_gcm_decrypt_arm64_crypto_256_end_bytes: sub x11, x11, x14 ld1 {v15.2d}, [x11] add w15, w15, #1 mov v14.16b, v13.16b rbit v15.16b, v15.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v16.16b, v26.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v16.1d, v22.1d pmull2 v29.1q, v16.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b ext v31.16b, v16.16b, v16.16b, #8 aese v14.16b, v3.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v10.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v11.16b aesmc v14.16b, v14.16b ldr q29, [x9] eor v26.16b, v28.16b, v30.16b # Done GHASH aese v14.16b, v12.16b aesmc v14.16b, v14.16b ldr q30, [x9, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b rbit v15.16b, v15.16b eor v14.16b, v14.16b, v15.16b st1 {v14.2d}, [x11] cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_256_out_start_dw ldr x17, [x11], #8 sub w14, w14, #8 str x17, [x1], #8 L_aes_gcm_decrypt_arm64_crypto_256_out_start_dw: cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_256_out_start_sw ldr w17, [x11], #4 sub w14, w14, #4 str w17, [x1], #4 L_aes_gcm_decrypt_arm64_crypto_256_out_start_sw: cmp w14, #2 blt L_aes_gcm_decrypt_arm64_crypto_256_out_start_byte ldrh w17, [x11], #2 sub w14, w14, #2 strh w17, [x1], #2 L_aes_gcm_decrypt_arm64_crypto_256_out_start_byte: cbz w14, L_aes_gcm_decrypt_arm64_crypto_256_out_end_bytes ldrb w17, [x11], #1 subs w14, w14, #1 strb w17, [x1], #1 bne L_aes_gcm_decrypt_arm64_crypto_256_out_start_byte L_aes_gcm_decrypt_arm64_crypto_256_out_end_bytes: L_aes_gcm_decrypt_arm64_crypto_256_partial_done: ld1 {v14.2d}, [x12] lsl x8, x8, #3 rbit x8, x8 mov v28.d[0], x8 lsl x2, x2, #3 rbit x2, x2 mov v28.d[1], x2 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v28.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b pmull v28.1q, v26.1d, v22.1d pmull2 v29.1q, v26.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b ext v31.16b, v26.16b, v26.16b, #8 aese v14.16b, v3.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d ldr q11, [x9, #-32] aese v14.16b, v10.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] ldr q12, [x9, #-16] aese v14.16b, v11.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b ldr q29, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b rbit v26.16b, v26.16b ldr q30, [x9, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v26.16b, v26.16b, v14.16b cmp w6, #16 blt L_aes_gcm_decrypt_arm64_crypto_256_part_tag ld1 {v28.16b}, [x5] b L_aes_gcm_decrypt_arm64_crypto_256_tag_loaded L_aes_gcm_decrypt_arm64_crypto_256_part_tag: eor v28.16b, v28.16b, v28.16b mov x17, x6 st1 {v28.2d}, [x11] cmp x17, #8 blt L_aes_gcm_decrypt_arm64_crypto_256_tag_start_dw ldr x16, [x5], #8 sub x17, x17, #8 str x16, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_256_tag_start_dw: cmp x17, #4 blt L_aes_gcm_decrypt_arm64_crypto_256_tag_start_sw ldr w16, [x5], #4 sub x17, x17, #4 str w16, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_256_tag_start_sw: cmp x17, #2 blt L_aes_gcm_decrypt_arm64_crypto_256_tag_start_byte ldrh w16, [x5], #2 sub x17, x17, #2 strh w16, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_256_tag_start_byte: cbz x17, L_aes_gcm_decrypt_arm64_crypto_256_tag_end_bytes ldrb w16, [x5], #1 subs x17, x17, #1 strb w16, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_256_tag_start_byte L_aes_gcm_decrypt_arm64_crypto_256_tag_end_bytes: sub x11, x11, x6 ld1 {v28.2d}, [x11] mov x17, #16 st1 {v26.2d}, [x11] sub x17, x17, x6 add x11, x11, x6 L_aes_gcm_decrypt_arm64_crypto_256_calc_tag_byte: strb wzr, [x11], #1 subs x17, x17, #1 bne L_aes_gcm_decrypt_arm64_crypto_256_calc_tag_byte subs x11, x11, #16 ld1 {v26.2d}, [x11] L_aes_gcm_decrypt_arm64_crypto_256_tag_loaded: eor v28.16b, v28.16b, v26.16b mov x16, v28.d[0] mov x17, v28.d[1] mov w19, #-180 orr x16, x16, x17 cmp x16, #0 csetm x0, ne and x0, x0, x19 #endif /* !NO_AES_256 */ b L_aes_gcm_decrypt_arm64_crypto_done # AES_GCM_128 L_aes_gcm_decrypt_arm64_crypto_start_128: #ifndef NO_AES_128 cmp w14, #32 blt L_aes_gcm_decrypt_arm64_crypto_128_start_4 L_aes_gcm_decrypt_arm64_crypto_128_start_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rev w24, w24 mov v14.s[3], w24 rev w23, w23 mov v15.s[3], w23 rev w22, w22 mov v16.s[3], w22 rev w21, w21 mov v17.s[3], w21 rev w20, w20 mov v8.s[3], w20 rev w19, w19 mov v9.s[3], w19 rev w17, w17 mov v10.s[3], w17 rev w16, w15 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v18.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v19.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v20.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v21.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b ld1 {v0.16b}, [x0], #16 aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b ld1 {v1.16b}, [x0], #16 aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b ld1 {v2.16b}, [x0], #16 aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b ld1 {v3.16b}, [x0], #16 aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b ld1 {v13.2d}, [x12] st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_128_end_8 L_aes_gcm_decrypt_arm64_crypto_128_both_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w24, w24 rbit v19.16b, v19.16b mov v14.s[3], w24 rev w23, w23 rbit v20.16b, v20.16b mov v15.s[3], w23 rev w22, w22 rbit v21.16b, v21.16b mov v16.s[3], w22 rev w21, w21 rbit v0.16b, v0.16b mov v17.s[3], w21 rev w20, w20 rbit v1.16b, v1.16b mov v8.s[3], w20 rev w19, w19 rbit v2.16b, v2.16b mov v9.s[3], w19 rev w17, w17 rbit v3.16b, v3.16b mov v10.s[3], w17 rev w16, w15 eor v18.16b, v18.16b, v26.16b mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v31.16b, v3.16b, v3.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b ext v26.16b, v2.16b, v2.16b, #8 aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b ext v26.16b, v1.16b, v1.16b, #8 aese v10.16b, v13.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v31.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v30.16b, v30.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b ext v26.16b, v0.16b, v0.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v26.16b, v26.16b, v31.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ext v26.16b, v21.16b, v21.16b, #8 aese v8.16b, v13.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v26.16b, v26.16b, v31.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v30.16b, v30.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v26.16b, v20.16b, v20.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v26.16b, v26.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v30.16b, v30.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ext v26.16b, v18.16b, v18.16b, #8 aese v15.16b, v12.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v26.16b, v26.16b, v31.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull2 v29.1q, v29.2d, v27.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v31.16b, v31.16b, v29.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v18.16b}, [x0], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v19.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v20.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v21.16b}, [x0], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v0.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v1.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v2.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v3.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x12] eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_decrypt_arm64_crypto_128_both_8 L_aes_gcm_decrypt_arm64_crypto_128_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_arm64_crypto_128_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 ld1 {v8.2d, v9.2d}, [x9], #32 ld1 {v10.2d}, [x9] cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_128_done beq L_aes_gcm_decrypt_arm64_crypto_128_start_1 cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_128_start_2 add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rev w20, w20 mov v14.s[3], w20 rev w19, w19 mov v15.s[3], w19 rev w17, w17 mov v16.s[3], w17 rev w16, w15 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x0], #16 aese v17.16b, v7.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x0], #16 aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v20.16b}, [x0], #16 aese v17.16b, v8.16b aesmc v17.16b, v17.16b ld1 {v21.16b}, [x0], #16 aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w14, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 blt L_aes_gcm_decrypt_arm64_crypto_128_end_4 L_aes_gcm_decrypt_arm64_crypto_128_both_4: add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w20, w20 rbit v19.16b, v19.16b mov v14.s[3], w20 rev w19, w19 rbit v20.16b, v20.16b mov v15.s[3], w19 rev w17, w17 rbit v21.16b, v21.16b mov v16.s[3], w17 rev w16, w15 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v16.16b, v0.16b aesmc v16.16b, v16.16b ext v31.16b, v21.16b, v21.16b, #8 aese v17.16b, v0.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v2.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v3.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v17.16b, v3.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b ext v26.16b, v18.16b, v18.16b, #8 aese v14.16b, v5.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v15.16b, v5.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v15.16b, v6.16b aesmc v15.16b, v15.16b eor v31.16b, v31.16b, v29.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b eor v31.16b, v31.16b, v30.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v15.16b, v7.16b aesmc v15.16b, v15.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w14, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 bge L_aes_gcm_decrypt_arm64_crypto_128_both_4 L_aes_gcm_decrypt_arm64_crypto_128_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w14, #1 beq L_aes_gcm_decrypt_arm64_crypto_128_start_1 blt L_aes_gcm_decrypt_arm64_crypto_128_done L_aes_gcm_decrypt_arm64_crypto_128_start_2: add w20, w15, #1 mov v14.16b, v13.16b add w15, w15, #2 mov v15.16b, v13.16b rev w20, w20 mov v14.s[3], w20 rev w16, w15 mov v15.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w14, w14, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v19.16b}, [x0], #16 aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b st1 {v14.16b, v15.16b}, [x1], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w14, L_aes_gcm_decrypt_arm64_crypto_128_done L_aes_gcm_decrypt_arm64_crypto_128_start_1: add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v18.16b}, [x0], #16 eor v14.16b, v14.16b, v18.16b st1 {v14.16b}, [x1], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_arm64_crypto_128_done: ands w14, w2, #15 beq L_aes_gcm_decrypt_arm64_crypto_128_partial_done eor v15.16b, v15.16b, v15.16b mov w19, w14 st1 {v15.2d}, [x11] cmp x19, #8 blt L_aes_gcm_decrypt_arm64_crypto_128_start_dw ldr x17, [x0], #8 sub x19, x19, #8 str x17, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_128_start_dw: cmp x19, #4 blt L_aes_gcm_decrypt_arm64_crypto_128_start_sw ldr w17, [x0], #4 sub x19, x19, #4 str w17, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_128_start_sw: cmp x19, #2 blt L_aes_gcm_decrypt_arm64_crypto_128_start_byte ldrh w17, [x0], #2 sub x19, x19, #2 strh w17, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_128_start_byte: cbz x19, L_aes_gcm_decrypt_arm64_crypto_128_end_bytes ldrb w17, [x0], #1 subs x19, x19, #1 strb w17, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_128_start_byte L_aes_gcm_decrypt_arm64_crypto_128_end_bytes: sub x11, x11, x14 ld1 {v15.2d}, [x11] add w15, w15, #1 mov v14.16b, v13.16b rbit v15.16b, v15.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v16.16b, v26.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v16.1d, v22.1d pmull2 v29.1q, v16.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b ext v31.16b, v16.16b, v16.16b, #8 aese v14.16b, v3.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH rbit v15.16b, v15.16b eor v14.16b, v14.16b, v15.16b st1 {v14.2d}, [x11] cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_128_out_start_dw ldr x17, [x11], #8 sub w14, w14, #8 str x17, [x1], #8 L_aes_gcm_decrypt_arm64_crypto_128_out_start_dw: cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_128_out_start_sw ldr w17, [x11], #4 sub w14, w14, #4 str w17, [x1], #4 L_aes_gcm_decrypt_arm64_crypto_128_out_start_sw: cmp w14, #2 blt L_aes_gcm_decrypt_arm64_crypto_128_out_start_byte ldrh w17, [x11], #2 sub w14, w14, #2 strh w17, [x1], #2 L_aes_gcm_decrypt_arm64_crypto_128_out_start_byte: cbz w14, L_aes_gcm_decrypt_arm64_crypto_128_out_end_bytes ldrb w17, [x11], #1 subs w14, w14, #1 strb w17, [x1], #1 bne L_aes_gcm_decrypt_arm64_crypto_128_out_start_byte L_aes_gcm_decrypt_arm64_crypto_128_out_end_bytes: L_aes_gcm_decrypt_arm64_crypto_128_partial_done: ld1 {v14.2d}, [x12] lsl x8, x8, #3 rbit x8, x8 mov v28.d[0], x8 lsl x2, x2, #3 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b pmull v28.1q, v26.1d, v22.1d pmull2 v29.1q, v26.2d, v22.2d ext v31.16b, v26.16b, v26.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v6.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b rbit v26.16b, v26.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b eor v26.16b, v26.16b, v14.16b cmp w6, #16 blt L_aes_gcm_decrypt_arm64_crypto_128_part_tag ld1 {v28.16b}, [x5] b L_aes_gcm_decrypt_arm64_crypto_128_tag_loaded L_aes_gcm_decrypt_arm64_crypto_128_part_tag: eor v28.16b, v28.16b, v28.16b mov x17, x6 st1 {v28.2d}, [x11] cmp x17, #8 blt L_aes_gcm_decrypt_arm64_crypto_128_tag_start_dw ldr x16, [x5], #8 sub x17, x17, #8 str x16, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_128_tag_start_dw: cmp x17, #4 blt L_aes_gcm_decrypt_arm64_crypto_128_tag_start_sw ldr w16, [x5], #4 sub x17, x17, #4 str w16, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_128_tag_start_sw: cmp x17, #2 blt L_aes_gcm_decrypt_arm64_crypto_128_tag_start_byte ldrh w16, [x5], #2 sub x17, x17, #2 strh w16, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_128_tag_start_byte: cbz x17, L_aes_gcm_decrypt_arm64_crypto_128_tag_end_bytes ldrb w16, [x5], #1 subs x17, x17, #1 strb w16, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_128_tag_start_byte L_aes_gcm_decrypt_arm64_crypto_128_tag_end_bytes: sub x11, x11, x6 ld1 {v28.2d}, [x11] mov x17, #16 st1 {v26.2d}, [x11] sub x17, x17, x6 add x11, x11, x6 L_aes_gcm_decrypt_arm64_crypto_128_calc_tag_byte: strb wzr, [x11], #1 subs x17, x17, #1 bne L_aes_gcm_decrypt_arm64_crypto_128_calc_tag_byte subs x11, x11, #16 ld1 {v26.2d}, [x11] L_aes_gcm_decrypt_arm64_crypto_128_tag_loaded: eor v28.16b, v28.16b, v26.16b mov x16, v28.d[0] mov x17, v28.d[1] mov w19, #-180 orr x16, x16, x17 cmp x16, #0 csetm x0, ne and x0, x0, x19 #endif /* !NO_AES_128 */ L_aes_gcm_decrypt_arm64_crypto_done: ldp x17, x19, [x29, #24] ldp x20, x21, [x29, #40] ldp x22, x23, [x29, #56] ldr x24, [x29, #72] ldp d8, d9, [x29, #80] ldp d10, d11, [x29, #96] ldp d12, d13, [x29, #112] ldp d14, d15, [x29, #128] ldp x29, x30, [sp], #0x90 ret #ifndef __APPLE__ .size AES_GCM_decrypt_AARCH64,.-AES_GCM_decrypt_AARCH64 #endif /* __APPLE__ */ #endif /* HAVE_AES_DECRYPT */ #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 #ifndef __APPLE__ .text .globl AES_GCM_encrypt_AARCH64_EOR3 .type AES_GCM_encrypt_AARCH64_EOR3,@function .align 2 AES_GCM_encrypt_AARCH64_EOR3: #else .section __TEXT,__text .globl _AES_GCM_encrypt_AARCH64_EOR3 .p2align 2 _AES_GCM_encrypt_AARCH64_EOR3: #endif /* __APPLE__ */ stp x29, x30, [sp, #-144]! add x29, sp, #0 stp x17, x19, [x29, #24] stp x20, x21, [x29, #40] stp x22, x23, [x29, #56] str x24, [x29, #72] stp d8, d9, [x29, #80] stp d10, d11, [x29, #96] stp d12, d13, [x29, #112] stp d14, d15, [x29, #128] ldr w8, [x29, #144] ldr x9, [x29, #152] ldr x10, [x29, #160] ldr x11, [x29, #168] ldr x12, [x29, #176] ldr w13, [x29, #184] movi v27.16b, #0x87 eor v26.16b, v26.16b, v26.16b ushr v27.2d, v27.2d, #56 ld1 {v22.2d}, [x10] cmp w8, #0x40 csetm x16, lt cmp w2, #32 csetm x17, lt ands x16, x16, x17 bne L_aes_gcm_encrypt_arm64_crypto_eor3_h_done # Square H => H^2 pmull2 v31.1q, v22.2d, v22.2d pmull v30.1q, v22.1d, v22.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v23.16b, v30.16b, v31.16b cmp w8, #0x100 csetm x16, lt cmp w2, #0x40 csetm x17, lt ands x16, x16, x17 bne L_aes_gcm_encrypt_arm64_crypto_eor3_h_done # Multiply H and H^2 => H^3 pmull v28.1q, v22.1d, v23.1d pmull2 v29.1q, v22.2d, v23.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v23.1d pmull2 v31.1q, v31.2d, v23.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v24.16b, v28.16b, v30.16b # Square H^2 => H^4 pmull2 v31.1q, v23.2d, v23.2d pmull v30.1q, v23.1d, v23.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v25.16b, v30.16b, v31.16b # Done cmp w8, #0x400 csetm x16, lt cmp w2, #0x200 csetm x17, lt ands x16, x16, x17 bne L_aes_gcm_encrypt_arm64_crypto_eor3_h_done # Multiply H and H^4 => H^5 pmull v28.1q, v22.1d, v25.1d pmull2 v29.1q, v22.2d, v25.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v25.1d pmull2 v31.1q, v31.2d, v25.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v4.16b, v28.16b, v30.16b # Square H^3 => H^6 pmull2 v31.1q, v24.2d, v24.2d pmull v30.1q, v24.1d, v24.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v5.16b, v30.16b, v31.16b # Multiply H and H^6 => H^7 pmull v28.1q, v22.1d, v5.1d pmull2 v29.1q, v22.2d, v5.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v5.1d pmull2 v31.1q, v31.2d, v5.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v6.16b, v28.16b, v30.16b # Square H^4 => H^8 pmull2 v31.1q, v25.2d, v25.2d pmull v30.1q, v25.1d, v25.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v7.16b, v30.16b, v31.16b # Done L_aes_gcm_encrypt_arm64_crypto_eor3_h_done: lsr w14, w8, #4 cmp w14, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_1 cmp w14, #16 blt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_2 cmp w14, #0x40 blt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_4 L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_8: ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x7], #0x40 rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH sub w14, w14, #8 cmp w14, #8 bge L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_8 cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_done beq L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_1 cmp w14, #16 blt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_2 L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_4: ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH sub w14, w14, #4 cmp w14, #4 bge L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_4 cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_done beq L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_1 L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_2: ld1 {v18.16b, v19.16b}, [x7], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH sub w14, w14, #2 cmp w14, #1 bgt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_2 blt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_done L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_1: cbz w14, L_aes_gcm_encrypt_arm64_crypto_eor3_aad_done L_aes_gcm_encrypt_arm64_crypto_eor3_aad_both_1: ld1 {v18.16b}, [x7], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH subs w14, w14, #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_aad_both_1 L_aes_gcm_encrypt_arm64_crypto_eor3_aad_done: and w14, w8, #15 cbz w14, L_aes_gcm_encrypt_arm64_crypto_eor3_aad_partial_done eor v28.16b, v28.16b, v28.16b mov w20, w14 st1 {v28.2d}, [x11] cmp w20, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_dw ldr x19, [x7], #8 sub w20, w20, #8 str x19, [x11], #8 L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_dw: cmp w20, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_sw ldr w19, [x7], #4 sub w20, w20, #4 str w19, [x11], #4 L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_sw: cmp w20, #2 blt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_byte ldrh w19, [x7], #2 sub w20, w20, #2 strh w19, [x11], #2 L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_byte: cbz w20, L_aes_gcm_encrypt_arm64_crypto_eor3_aad_end_bytes ldrb w19, [x7], #1 subs w20, w20, #1 strb w19, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_aad_end_bytes: sub x11, x11, x14 ld1 {v18.2d}, [x11] rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_aad_partial_done: # Load Nonce cmp w4, #12 bne L_aes_gcm_encrypt_arm64_crypto_eor3_ghash_nonce ldr x16, [x3] movi v13.4s, #1, lsl 24 ldr w17, [x3, #8] mov v13.d[0], x16 mov v13.s[2], w17 mov w15, #1 b L_aes_gcm_encrypt_arm64_crypto_eor3_done_nonce L_aes_gcm_encrypt_arm64_crypto_eor3_ghash_nonce: eor v13.16b, v13.16b, v13.16b lsr w14, w4, #4 cbz w14, L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_done L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_1: ld1 {v18.16b}, [x3], #16 rbit v18.16b, v18.16b eor v21.16b, v13.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v13.16b, v28.16b, v30.16b # Done GHASH subs w14, w14, #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_1 L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_done: and w24, w4, #15 cbz x24, L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_partial_done eor v28.16b, v28.16b, v28.16b mov w20, w24 st1 {v28.2d}, [x11] cmp w20, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_dw ldr x19, [x3], #8 sub w20, w20, #8 str x19, [x11], #8 L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_dw: cmp w20, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_sw ldr w19, [x3], #4 sub w20, w20, #4 str w19, [x11], #4 L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_sw: cmp w20, #2 blt L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_byte ldrh w19, [x3], #2 sub w20, w20, #2 strh w19, [x11], #2 L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_byte: cbz w20, L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_end_bytes ldrb w19, [x3], #1 subs w20, w20, #1 strb w19, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_end_bytes: sub x11, x11, x24 ld1 {v18.2d}, [x11] rbit v18.16b, v18.16b eor v21.16b, v13.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v13.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_partial_done: eor x14, x14, x14 lsl x24, x4, #3 mov v28.d[0], x14 mov v28.d[1], x24 rev64 v28.16b, v28.16b rbit v28.16b, v28.16b eor v13.16b, v13.16b, v28.16b pmull v28.1q, v13.1d, v22.1d pmull2 v29.1q, v13.2d, v22.2d ext v31.16b, v13.16b, v13.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v13.16b, v28.16b, v30.16b rbit v13.16b, v13.16b mov w15, v13.s[3] rev w15, w15 L_aes_gcm_encrypt_arm64_crypto_eor3_done_nonce: st1 {v13.2d}, [x12] lsr w14, w2, #4 cmp w13, #12 blt L_aes_gcm_encrypt_arm64_crypto_eor3_start_128 bgt L_aes_gcm_encrypt_arm64_crypto_eor3_start_256 # AES_GCM_192 #ifndef NO_AES_192 cmp w14, #32 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_4 L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rev w24, w24 rev w23, w23 rev w22, w22 rev w21, w21 rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 mov v14.s[3], w24 mov v15.s[3], w23 mov v16.s[3], w22 mov v17.s[3], w21 mov v8.s[3], w20 mov v9.s[3], w19 mov v10.s[3], w17 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v20.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v21.16b}, [x0], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v0.16b}, [x0], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v1.16b}, [x0], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v2.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v3.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b ld1 {v13.2d}, [x12] st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_end_8 L_aes_gcm_encrypt_arm64_crypto_eor3_192_both_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w24, w24 rbit v19.16b, v19.16b rev w23, w23 rbit v20.16b, v20.16b rev w22, w22 rbit v21.16b, v21.16b rev w21, w21 rbit v0.16b, v0.16b rev w20, w20 rbit v1.16b, v1.16b rev w19, w19 rbit v2.16b, v2.16b rev w17, w17 rbit v3.16b, v3.16b rev w16, w15 mov v14.s[3], w24 mov v15.s[3], w23 mov v16.s[3], w22 mov v17.s[3], w21 mov v8.s[3], w20 mov v9.s[3], w19 mov v10.s[3], w17 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v31.16b, v3.16b, v3.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b ext v26.16b, v2.16b, v2.16b, #8 aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b ext v26.16b, v1.16b, v1.16b, #8 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b ext v26.16b, v0.16b, v0.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b ext v26.16b, v21.16b, v21.16b, #8 aese v9.16b, v13.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v8.16b, v12.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ext v26.16b, v19.16b, v19.16b, #8 aese v17.16b, v13.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull2 v29.1q, v29.2d, v27.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # Done GHASH aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v18.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v19.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x0], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x0], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x12] eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_encrypt_arm64_crypto_eor3_192_both_8 L_aes_gcm_encrypt_arm64_crypto_eor3_192_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 ld1 {v12.2d}, [x9] cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_done beq L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_1 cmp w14, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_2 add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w19 mov v16.s[3], w17 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b ld1 {v19.16b}, [x0], #16 aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x0], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x0], #16 aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w14, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_end_4 L_aes_gcm_encrypt_arm64_crypto_eor3_192_both_4: add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w20, w20 rev w19, w19 rev w17, w17 rbit v19.16b, v19.16b rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w19 mov v16.s[3], w17 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b rbit v20.16b, v20.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b rbit v21.16b, v21.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b eor v18.16b, v18.16b, v26.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b ext v31.16b, v21.16b, v21.16b, #8 aese v15.16b, v1.16b aesmc v15.16b, v15.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v31.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b ext v26.16b, v20.16b, v20.16b, #8 aese v16.16b, v2.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v17.16b, v2.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v16.16b, v3.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b ext v26.16b, v19.16b, v19.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v4.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v5.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v17.16b, v5.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v29.1q, v29.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v15.16b, v7.16b aesmc v15.16b, v15.16b mov v28.d[1], v31.d[0] aese v16.16b, v7.16b aesmc v16.16b, v16.16b eor v26.16b, v28.16b, v30.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b # Done GHASH aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w14, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 bge L_aes_gcm_encrypt_arm64_crypto_eor3_192_both_4 L_aes_gcm_encrypt_arm64_crypto_eor3_192_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w14, #1 beq L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_1 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_done L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_2: add w20, w15, #1 mov v14.16b, v13.16b add w15, w15, #2 mov v15.16b, v13.16b rev w20, w20 rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w14, w14, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x0], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b st1 {v18.16b, v19.16b}, [x1], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w14, L_aes_gcm_encrypt_arm64_crypto_eor3_192_done L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_1: add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b eor v18.16b, v18.16b, v14.16b st1 {v18.16b}, [x1], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_192_done: ands w14, w2, #15 beq L_aes_gcm_encrypt_arm64_crypto_eor3_192_partial_done eor v16.16b, v16.16b, v16.16b mov w19, w14 st1 {v16.2d}, [x11] cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_dw ldr x17, [x0], #8 sub x19, x19, #8 str x17, [x11], #8 L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_dw: cmp x19, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_sw ldr w17, [x0], #4 sub x19, x19, #4 str w17, [x11], #4 L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_sw: cmp x19, #2 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_byte ldrh w17, [x0], #2 sub x19, x19, #2 strh w17, [x11], #2 L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_byte: cbz x19, L_aes_gcm_encrypt_arm64_crypto_eor3_192_end_bytes ldrb w17, [x0], #1 subs x19, x19, #1 strb w17, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_192_end_bytes: sub x11, x11, x14 ld1 {v16.2d}, [x11] add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b eor v16.16b, v16.16b, v14.16b st1 {v16.2d}, [x11] mov w19, w14 cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_dw ldr x17, [x11], #8 sub x19, x19, #8 str x17, [x1], #8 L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_dw: cmp x19, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_sw ldr w17, [x11], #4 sub x19, x19, #4 str w17, [x1], #4 L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_sw: cmp x19, #2 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_byte ldrh w17, [x11], #2 sub x19, x19, #2 strh w17, [x1], #2 L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_byte: cbz x19, L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_end_bytes ldrb w17, [x11], #1 subs x19, x19, #1 strb w17, [x1], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_end_bytes: mov x17, #16 sub x17, x17, x14 L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_zero: subs x17, x17, #1 strb wzr, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_zero sub x11, x11, #16 ld1 {v14.2d}, [x11] rbit v14.16b, v14.16b eor v15.16b, v26.16b, v14.16b # X = C * H^1 pmull v28.1q, v15.1d, v22.1d pmull2 v29.1q, v15.2d, v22.2d ext v31.16b, v15.16b, v15.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_192_partial_done: ld1 {v14.2d}, [x12] lsl x8, x8, #3 rbit x8, x8 mov v28.d[0], x8 lsl x2, x2, #3 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b pmull v28.1q, v26.1d, v22.1d pmull2 v29.1q, v26.2d, v22.2d aese v14.16b, v0.16b aesmc v14.16b, v14.16b ext v31.16b, v26.16b, v26.16b, #8 aese v14.16b, v1.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v8.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b rbit v26.16b, v26.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b eor v26.16b, v26.16b, v14.16b cmp w6, #16 bne L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_partial st1 {v26.16b}, [x5] b L_aes_gcm_encrypt_arm64_crypto_eor3_done L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_partial: st1 {v26.16b}, [x11] cmp w6, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_dw ldr x16, [x11], #8 sub w6, w6, #8 str x16, [x5], #8 L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_dw: cmp w6, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_sw ldr w16, [x11], #4 sub w6, w6, #4 str w16, [x5], #4 L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_sw: cmp w6, #2 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_byte ldrh w16, [x11], #2 sub w6, w6, #2 strh w16, [x5], #2 L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_byte: cbz w6, L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_end_bytes ldrb w16, [x11], #1 subs w6, w6, #1 strb w16, [x5], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_end_bytes: #endif /* !NO_AES_192 */ b L_aes_gcm_encrypt_arm64_crypto_eor3_done # AES_GCM_256 L_aes_gcm_encrypt_arm64_crypto_eor3_start_256: #ifndef NO_AES_256 cmp w14, #32 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_4 L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rev w24, w24 rev w23, w23 rev w22, w22 rev w21, w21 rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 mov v14.s[3], w24 mov v15.s[3], w23 mov v16.s[3], w22 mov v17.s[3], w21 mov v8.s[3], w20 mov v9.s[3], w19 mov v10.s[3], w17 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v20.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v21.16b}, [x0], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v0.16b}, [x0], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v1.16b}, [x0], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v2.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v3.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b ld1 {v13.2d}, [x12] st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_end_8 L_aes_gcm_encrypt_arm64_crypto_eor3_256_both_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w24, w24 rbit v19.16b, v19.16b rev w23, w23 rbit v20.16b, v20.16b rev w22, w22 rbit v21.16b, v21.16b rev w21, w21 rbit v0.16b, v0.16b rev w20, w20 rbit v1.16b, v1.16b rev w19, w19 rbit v2.16b, v2.16b rev w17, w17 rbit v3.16b, v3.16b rev w16, w15 mov v14.s[3], w24 mov v15.s[3], w23 mov v16.s[3], w22 mov v17.s[3], w21 mov v8.s[3], w20 mov v9.s[3], w19 mov v10.s[3], w17 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v31.16b, v3.16b, v3.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b ext v26.16b, v2.16b, v2.16b, #8 aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b ext v26.16b, v1.16b, v1.16b, #8 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b ext v26.16b, v0.16b, v0.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b ext v26.16b, v21.16b, v21.16b, #8 aese v9.16b, v13.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v8.16b, v12.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ext v26.16b, v19.16b, v19.16b, #8 aese v17.16b, v13.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull2 v29.1q, v29.2d, v27.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # Done GHASH aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v18.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v19.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x0], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x0], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x12] eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_encrypt_arm64_crypto_eor3_256_both_8 L_aes_gcm_encrypt_arm64_crypto_eor3_256_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 ld1 {v12.2d}, [x9], #16 cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_done beq L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_1 cmp w14, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_2 add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w19 mov v16.s[3], w17 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b ld1 {v19.16b}, [x0], #16 aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x0], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x0], #16 aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w14, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_end_4 L_aes_gcm_encrypt_arm64_crypto_eor3_256_both_4: add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w20, w20 rev w19, w19 rev w17, w17 rbit v19.16b, v19.16b rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w19 mov v16.s[3], w17 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b rbit v20.16b, v20.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b rbit v21.16b, v21.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b eor v18.16b, v18.16b, v26.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b ext v31.16b, v21.16b, v21.16b, #8 aese v15.16b, v1.16b aesmc v15.16b, v15.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v31.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b ext v26.16b, v20.16b, v20.16b, #8 aese v16.16b, v2.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v17.16b, v2.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v16.16b, v3.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b ext v26.16b, v19.16b, v19.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v4.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v5.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v17.16b, v5.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v29.1q, v29.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v15.16b, v7.16b aesmc v15.16b, v15.16b mov v28.d[1], v31.d[0] aese v16.16b, v7.16b aesmc v16.16b, v16.16b eor v26.16b, v28.16b, v30.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b # Done GHASH aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w14, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 bge L_aes_gcm_encrypt_arm64_crypto_eor3_256_both_4 L_aes_gcm_encrypt_arm64_crypto_eor3_256_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w14, #1 beq L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_1 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_done L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_2: add w20, w15, #1 mov v14.16b, v13.16b add w15, w15, #2 mov v15.16b, v13.16b rev w20, w20 rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w14, w14, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x0], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b ld1 {v29.2d, v30.2d}, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b st1 {v18.16b, v19.16b}, [x1], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w14, L_aes_gcm_encrypt_arm64_crypto_eor3_256_done L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_1: add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b ldr q29, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ldr q30, [x9, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v18.16b, v18.16b, v14.16b st1 {v18.16b}, [x1], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_256_done: ands w14, w2, #15 beq L_aes_gcm_encrypt_arm64_crypto_eor3_256_partial_done eor v16.16b, v16.16b, v16.16b mov w19, w14 st1 {v16.2d}, [x11] cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_dw ldr x17, [x0], #8 sub x19, x19, #8 str x17, [x11], #8 L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_dw: cmp x19, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_sw ldr w17, [x0], #4 sub x19, x19, #4 str w17, [x11], #4 L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_sw: cmp x19, #2 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_byte ldrh w17, [x0], #2 sub x19, x19, #2 strh w17, [x11], #2 L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_byte: cbz x19, L_aes_gcm_encrypt_arm64_crypto_eor3_256_end_bytes ldrb w17, [x0], #1 subs x19, x19, #1 strb w17, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_256_end_bytes: sub x11, x11, x14 ld1 {v16.2d}, [x11] add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b ldr q29, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ldr q30, [x9, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v16.16b, v16.16b, v14.16b st1 {v16.2d}, [x11] mov w19, w14 cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_dw ldr x17, [x11], #8 sub x19, x19, #8 str x17, [x1], #8 L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_dw: cmp x19, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_sw ldr w17, [x11], #4 sub x19, x19, #4 str w17, [x1], #4 L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_sw: cmp x19, #2 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_byte ldrh w17, [x11], #2 sub x19, x19, #2 strh w17, [x1], #2 L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_byte: cbz x19, L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_end_bytes ldrb w17, [x11], #1 subs x19, x19, #1 strb w17, [x1], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_end_bytes: mov x17, #16 sub x17, x17, x14 L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_zero: subs x17, x17, #1 strb wzr, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_zero sub x11, x11, #16 ld1 {v14.2d}, [x11] rbit v14.16b, v14.16b eor v15.16b, v26.16b, v14.16b # X = C * H^1 pmull v28.1q, v15.1d, v22.1d pmull2 v29.1q, v15.2d, v22.2d ext v31.16b, v15.16b, v15.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_256_partial_done: ld1 {v14.2d}, [x12] lsl x8, x8, #3 rbit x8, x8 mov v28.d[0], x8 lsl x2, x2, #3 rbit x2, x2 mov v28.d[1], x2 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v28.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b pmull v28.1q, v26.1d, v22.1d pmull2 v29.1q, v26.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b ext v31.16b, v26.16b, v26.16b, #8 aese v14.16b, v3.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v9.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] ldr q11, [x9, #-32] aese v14.16b, v10.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b ldr q12, [x9, #-16] aese v14.16b, v11.16b aesmc v14.16b, v14.16b ldr q29, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b rbit v26.16b, v26.16b ldr q30, [x9, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v26.16b, v26.16b, v14.16b cmp w6, #16 bne L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_partial st1 {v26.16b}, [x5] b L_aes_gcm_encrypt_arm64_crypto_eor3_done L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_partial: st1 {v26.16b}, [x11] cmp w6, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_dw ldr x16, [x11], #8 sub w6, w6, #8 str x16, [x5], #8 L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_dw: cmp w6, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_sw ldr w16, [x11], #4 sub w6, w6, #4 str w16, [x5], #4 L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_sw: cmp w6, #2 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_byte ldrh w16, [x11], #2 sub w6, w6, #2 strh w16, [x5], #2 L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_byte: cbz w6, L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_end_bytes ldrb w16, [x11], #1 subs w6, w6, #1 strb w16, [x5], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_end_bytes: #endif /* !NO_AES_256 */ b L_aes_gcm_encrypt_arm64_crypto_eor3_done # AES_GCM_128 L_aes_gcm_encrypt_arm64_crypto_eor3_start_128: #ifndef NO_AES_128 cmp w14, #32 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_4 L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rev w24, w24 rev w23, w23 rev w22, w22 rev w21, w21 rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 mov v14.s[3], w24 mov v15.s[3], w23 mov v16.s[3], w22 mov v17.s[3], w21 mov v8.s[3], w20 mov v9.s[3], w19 mov v10.s[3], w17 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v20.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v21.16b}, [x0], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b ld1 {v0.16b}, [x0], #16 aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b ld1 {v1.16b}, [x0], #16 aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b ld1 {v2.16b}, [x0], #16 aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b ld1 {v3.16b}, [x0], #16 aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b ld1 {v13.2d}, [x12] st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_end_8 L_aes_gcm_encrypt_arm64_crypto_eor3_128_both_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w24, w24 rbit v19.16b, v19.16b rev w23, w23 rbit v20.16b, v20.16b rev w22, w22 rbit v21.16b, v21.16b rev w21, w21 rbit v0.16b, v0.16b rev w20, w20 rbit v1.16b, v1.16b rev w19, w19 rbit v2.16b, v2.16b rev w17, w17 rbit v3.16b, v3.16b rev w16, w15 mov v14.s[3], w24 mov v15.s[3], w23 mov v16.s[3], w22 mov v17.s[3], w21 mov v8.s[3], w20 mov v9.s[3], w19 mov v10.s[3], w17 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v31.16b, v3.16b, v3.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b ext v26.16b, v2.16b, v2.16b, #8 aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b ext v26.16b, v1.16b, v1.16b, #8 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b ext v26.16b, v0.16b, v0.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b ext v26.16b, v21.16b, v21.16b, #8 aese v9.16b, v13.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v8.16b, v12.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ext v26.16b, v19.16b, v19.16b, #8 aese v17.16b, v13.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull2 v29.1q, v29.2d, v27.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # Done GHASH aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v18.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v19.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x0], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x0], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x12] eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_encrypt_arm64_crypto_eor3_128_both_8 L_aes_gcm_encrypt_arm64_crypto_eor3_128_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 ld1 {v8.2d, v9.2d}, [x9], #32 ld1 {v10.2d}, [x9] cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_done beq L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_1 cmp w14, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_2 add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w19 mov v16.s[3], w17 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b ld1 {v19.16b}, [x0], #16 aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v20.16b}, [x0], #16 aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b ld1 {v21.16b}, [x0], #16 aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w14, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_end_4 L_aes_gcm_encrypt_arm64_crypto_eor3_128_both_4: add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w20, w20 rev w19, w19 rev w17, w17 rbit v19.16b, v19.16b rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w19 mov v16.s[3], w17 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b rbit v20.16b, v20.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b rbit v21.16b, v21.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b eor v18.16b, v18.16b, v26.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b ext v31.16b, v21.16b, v21.16b, #8 aese v15.16b, v1.16b aesmc v15.16b, v15.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v31.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b ext v26.16b, v20.16b, v20.16b, #8 aese v16.16b, v2.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v17.16b, v2.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v16.16b, v3.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b ext v26.16b, v19.16b, v19.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v4.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v5.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v17.16b, v5.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v29.1q, v29.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v15.16b, v7.16b aesmc v15.16b, v15.16b mov v28.d[1], v31.d[0] aese v16.16b, v7.16b aesmc v16.16b, v16.16b eor v26.16b, v28.16b, v30.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b # Done GHASH aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w14, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 bge L_aes_gcm_encrypt_arm64_crypto_eor3_128_both_4 L_aes_gcm_encrypt_arm64_crypto_eor3_128_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w14, #1 beq L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_1 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_done L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_2: add w20, w15, #1 mov v14.16b, v13.16b add w15, w15, #2 mov v15.16b, v13.16b rev w20, w20 rev w16, w15 mov v14.s[3], w20 mov v15.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w14, w14, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v19.16b}, [x0], #16 aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b st1 {v18.16b, v19.16b}, [x1], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w14, L_aes_gcm_encrypt_arm64_crypto_eor3_128_done L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_1: add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v18.16b}, [x0], #16 eor v18.16b, v18.16b, v14.16b st1 {v18.16b}, [x1], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_128_done: ands w14, w2, #15 beq L_aes_gcm_encrypt_arm64_crypto_eor3_128_partial_done eor v16.16b, v16.16b, v16.16b mov w19, w14 st1 {v16.2d}, [x11] cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_dw ldr x17, [x0], #8 sub x19, x19, #8 str x17, [x11], #8 L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_dw: cmp x19, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_sw ldr w17, [x0], #4 sub x19, x19, #4 str w17, [x11], #4 L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_sw: cmp x19, #2 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_byte ldrh w17, [x0], #2 sub x19, x19, #2 strh w17, [x11], #2 L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_byte: cbz x19, L_aes_gcm_encrypt_arm64_crypto_eor3_128_end_bytes ldrb w17, [x0], #1 subs x19, x19, #1 strb w17, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_128_end_bytes: sub x11, x11, x14 ld1 {v16.2d}, [x11] add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b eor v16.16b, v16.16b, v14.16b st1 {v16.2d}, [x11] mov w19, w14 cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_dw ldr x17, [x11], #8 sub x19, x19, #8 str x17, [x1], #8 L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_dw: cmp x19, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_sw ldr w17, [x11], #4 sub x19, x19, #4 str w17, [x1], #4 L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_sw: cmp x19, #2 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_byte ldrh w17, [x11], #2 sub x19, x19, #2 strh w17, [x1], #2 L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_byte: cbz x19, L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_end_bytes ldrb w17, [x11], #1 subs x19, x19, #1 strb w17, [x1], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_end_bytes: mov x17, #16 sub x17, x17, x14 L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_zero: subs x17, x17, #1 strb wzr, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_zero sub x11, x11, #16 ld1 {v14.2d}, [x11] rbit v14.16b, v14.16b eor v15.16b, v26.16b, v14.16b # X = C * H^1 pmull v28.1q, v15.1d, v22.1d pmull2 v29.1q, v15.2d, v22.2d ext v31.16b, v15.16b, v15.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_128_partial_done: ld1 {v14.2d}, [x12] lsl x8, x8, #3 rbit x8, x8 mov v28.d[0], x8 lsl x2, x2, #3 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b pmull v28.1q, v26.1d, v22.1d pmull2 v29.1q, v26.2d, v22.2d ext v31.16b, v26.16b, v26.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v6.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b rbit v26.16b, v26.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b eor v26.16b, v26.16b, v14.16b cmp w6, #16 bne L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_partial st1 {v26.16b}, [x5] b L_aes_gcm_encrypt_arm64_crypto_eor3_done L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_partial: st1 {v26.16b}, [x11] cmp w6, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_dw ldr x16, [x11], #8 sub w6, w6, #8 str x16, [x5], #8 L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_dw: cmp w6, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_sw ldr w16, [x11], #4 sub w6, w6, #4 str w16, [x5], #4 L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_sw: cmp w6, #2 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_byte ldrh w16, [x11], #2 sub w6, w6, #2 strh w16, [x5], #2 L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_byte: cbz w6, L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_end_bytes ldrb w16, [x11], #1 subs w6, w6, #1 strb w16, [x5], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_end_bytes: #endif /* !NO_AES_128 */ L_aes_gcm_encrypt_arm64_crypto_eor3_done: ldp x17, x19, [x29, #24] ldp x20, x21, [x29, #40] ldp x22, x23, [x29, #56] ldr x24, [x29, #72] ldp d8, d9, [x29, #80] ldp d10, d11, [x29, #96] ldp d12, d13, [x29, #112] ldp d14, d15, [x29, #128] ldp x29, x30, [sp], #0x90 ret #ifndef __APPLE__ .size AES_GCM_encrypt_AARCH64_EOR3,.-AES_GCM_encrypt_AARCH64_EOR3 #endif /* __APPLE__ */ #ifdef HAVE_AES_DECRYPT #ifndef __APPLE__ .text .globl AES_GCM_decrypt_AARCH64_EOR3 .type AES_GCM_decrypt_AARCH64_EOR3,@function .align 2 AES_GCM_decrypt_AARCH64_EOR3: #else .section __TEXT,__text .globl _AES_GCM_decrypt_AARCH64_EOR3 .p2align 2 _AES_GCM_decrypt_AARCH64_EOR3: #endif /* __APPLE__ */ stp x29, x30, [sp, #-144]! add x29, sp, #0 stp x17, x19, [x29, #24] stp x20, x21, [x29, #40] stp x22, x23, [x29, #56] str x24, [x29, #72] stp d8, d9, [x29, #80] stp d10, d11, [x29, #96] stp d12, d13, [x29, #112] stp d14, d15, [x29, #128] ldr w8, [x29, #144] ldr x9, [x29, #152] ldr x10, [x29, #160] ldr x11, [x29, #168] ldr x12, [x29, #176] ldr w13, [x29, #184] movi v27.16b, #0x87 eor v26.16b, v26.16b, v26.16b ushr v27.2d, v27.2d, #56 ld1 {v22.2d}, [x10] cmp w8, #0x40 csetm x16, lt cmp w2, #32 csetm x17, lt ands x16, x16, x17 bne L_aes_gcm_decrypt_arm64_crypto_eor3_h_done # Square H => H^2 pmull2 v31.1q, v22.2d, v22.2d pmull v30.1q, v22.1d, v22.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v23.16b, v30.16b, v31.16b cmp w8, #0x100 csetm x16, lt cmp w2, #0x40 csetm x17, lt ands x16, x16, x17 bne L_aes_gcm_decrypt_arm64_crypto_eor3_h_done # Multiply H and H^2 => H^3 pmull v28.1q, v22.1d, v23.1d pmull2 v29.1q, v22.2d, v23.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v23.1d pmull2 v31.1q, v31.2d, v23.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v24.16b, v28.16b, v30.16b # Square H^2 => H^4 pmull2 v31.1q, v23.2d, v23.2d pmull v30.1q, v23.1d, v23.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v25.16b, v30.16b, v31.16b # Done cmp w8, #0x400 csetm x16, lt cmp w2, #0x200 csetm x17, lt ands x16, x16, x17 bne L_aes_gcm_decrypt_arm64_crypto_eor3_h_done # Multiply H and H^4 => H^5 pmull v28.1q, v22.1d, v25.1d pmull2 v29.1q, v22.2d, v25.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v25.1d pmull2 v31.1q, v31.2d, v25.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v4.16b, v28.16b, v30.16b # Square H^3 => H^6 pmull2 v31.1q, v24.2d, v24.2d pmull v30.1q, v24.1d, v24.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v5.16b, v30.16b, v31.16b # Multiply H and H^6 => H^7 pmull v28.1q, v22.1d, v5.1d pmull2 v29.1q, v22.2d, v5.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v5.1d pmull2 v31.1q, v31.2d, v5.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v6.16b, v28.16b, v30.16b # Square H^4 => H^8 pmull2 v31.1q, v25.2d, v25.2d pmull v30.1q, v25.1d, v25.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v7.16b, v30.16b, v31.16b # Done L_aes_gcm_decrypt_arm64_crypto_eor3_h_done: lsr w14, w8, #4 cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_1 cmp w14, #16 blt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_2 cmp w14, #0x40 blt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_4 L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_8: ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x7], #0x40 rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH sub w14, w14, #8 cmp w14, #8 bge L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_8 cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_done beq L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_1 cmp w14, #16 blt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_2 L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_4: ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH sub w14, w14, #4 cmp w14, #4 bge L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_4 cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_done beq L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_1 L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_2: ld1 {v18.16b, v19.16b}, [x7], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH sub w14, w14, #2 cmp w14, #1 bgt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_2 blt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_done L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_1: cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_aad_done L_aes_gcm_decrypt_arm64_crypto_eor3_aad_both_1: ld1 {v18.16b}, [x7], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH subs w14, w14, #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_aad_both_1 L_aes_gcm_decrypt_arm64_crypto_eor3_aad_done: and w14, w8, #15 cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_aad_partial_done eor v28.16b, v28.16b, v28.16b mov w20, w14 st1 {v28.2d}, [x11] cmp w20, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_dw ldr x19, [x7], #8 sub w20, w20, #8 str x19, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_dw: cmp w20, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_sw ldr w19, [x7], #4 sub w20, w20, #4 str w19, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_sw: cmp w20, #2 blt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_byte ldrh w19, [x7], #2 sub w20, w20, #2 strh w19, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_byte: cbz w20, L_aes_gcm_decrypt_arm64_crypto_eor3_aad_end_bytes ldrb w19, [x7], #1 subs w20, w20, #1 strb w19, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_aad_end_bytes: sub x11, x11, x14 ld1 {v18.2d}, [x11] rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_aad_partial_done: # Load Nonce cmp w4, #12 bne L_aes_gcm_decrypt_arm64_crypto_eor3_ghash_nonce ldr x16, [x3] movi v13.4s, #1, lsl 24 ldr w17, [x3, #8] mov v13.d[0], x16 mov v13.s[2], w17 mov w15, #1 b L_aes_gcm_decrypt_arm64_crypto_eor3_done_nonce L_aes_gcm_decrypt_arm64_crypto_eor3_ghash_nonce: eor v13.16b, v13.16b, v13.16b lsr w14, w4, #4 cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_done L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_1: ld1 {v18.16b}, [x3], #16 rbit v18.16b, v18.16b eor v21.16b, v13.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v13.16b, v28.16b, v30.16b # Done GHASH subs w14, w14, #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_1 L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_done: and w24, w4, #15 cbz x24, L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_partial_done eor v28.16b, v28.16b, v28.16b mov w20, w24 st1 {v28.2d}, [x11] cmp w20, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_dw ldr x19, [x3], #8 sub w20, w20, #8 str x19, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_dw: cmp w20, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_sw ldr w19, [x3], #4 sub w20, w20, #4 str w19, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_sw: cmp w20, #2 blt L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_byte ldrh w19, [x3], #2 sub w20, w20, #2 strh w19, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_byte: cbz w20, L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_end_bytes ldrb w19, [x3], #1 subs w20, w20, #1 strb w19, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_end_bytes: sub x11, x11, x24 ld1 {v18.2d}, [x11] rbit v18.16b, v18.16b eor v21.16b, v13.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v13.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_partial_done: eor x14, x14, x14 lsl x24, x4, #3 mov v28.d[0], x14 mov v28.d[1], x24 rev64 v28.16b, v28.16b rbit v28.16b, v28.16b eor v13.16b, v13.16b, v28.16b pmull v28.1q, v13.1d, v22.1d pmull2 v29.1q, v13.2d, v22.2d ext v31.16b, v13.16b, v13.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v13.16b, v28.16b, v30.16b rbit v13.16b, v13.16b mov w15, v13.s[3] rev w15, w15 L_aes_gcm_decrypt_arm64_crypto_eor3_done_nonce: st1 {v13.2d}, [x12] lsr w14, w2, #4 cmp w13, #12 blt L_aes_gcm_decrypt_arm64_crypto_eor3_start_128 bgt L_aes_gcm_decrypt_arm64_crypto_eor3_start_256 # AES_GCM_192 #ifndef NO_AES_192 cmp w14, #32 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_4 L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rev w24, w24 mov v14.s[3], w24 rev w23, w23 mov v15.s[3], w23 rev w22, w22 mov v16.s[3], w22 rev w21, w21 mov v17.s[3], w21 rev w20, w20 mov v8.s[3], w20 rev w19, w19 mov v9.s[3], w19 rev w17, w17 mov v10.s[3], w17 rev w16, w15 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v18.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v19.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v20.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v21.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x0], #16 aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x0], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x0], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b ld1 {v13.2d}, [x12] st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_end_8 L_aes_gcm_decrypt_arm64_crypto_eor3_192_both_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w24, w24 rbit v19.16b, v19.16b mov v14.s[3], w24 rev w23, w23 rbit v20.16b, v20.16b mov v15.s[3], w23 rev w22, w22 rbit v21.16b, v21.16b mov v16.s[3], w22 rev w21, w21 rbit v0.16b, v0.16b mov v17.s[3], w21 rev w20, w20 rbit v1.16b, v1.16b mov v8.s[3], w20 rev w19, w19 rbit v2.16b, v2.16b mov v9.s[3], w19 rev w17, w17 rbit v3.16b, v3.16b mov v10.s[3], w17 rev w16, w15 eor v18.16b, v18.16b, v26.16b mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v31.16b, v3.16b, v3.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b ext v26.16b, v2.16b, v2.16b, #8 aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b ext v26.16b, v1.16b, v1.16b, #8 aese v10.16b, v13.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b ext v26.16b, v0.16b, v0.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ext v26.16b, v21.16b, v21.16b, #8 aese v8.16b, v13.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v26.16b, v20.16b, v20.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ext v26.16b, v18.16b, v18.16b, #8 aese v15.16b, v12.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull2 v29.1q, v29.2d, v27.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b mov v28.d[1], v31.d[0] aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v26.16b, v28.16b, v30.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # Done GHASH aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v18.16b}, [x0], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v19.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v20.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v21.16b}, [x0], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v0.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v1.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v2.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v3.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x12] eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_decrypt_arm64_crypto_eor3_192_both_8 L_aes_gcm_decrypt_arm64_crypto_eor3_192_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 ld1 {v12.2d}, [x9] cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_done beq L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_1 cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_2 add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rev w20, w20 mov v14.s[3], w20 rev w19, w19 mov v15.s[3], w19 rev w17, w17 mov v16.s[3], w17 rev w16, w15 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x0], #16 aese v17.16b, v7.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x0], #16 aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v20.16b}, [x0], #16 aese v17.16b, v8.16b aesmc v17.16b, v17.16b ld1 {v21.16b}, [x0], #16 aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w14, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_end_4 L_aes_gcm_decrypt_arm64_crypto_eor3_192_both_4: add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w20, w20 rbit v19.16b, v19.16b mov v14.s[3], w20 rev w19, w19 rbit v20.16b, v20.16b mov v15.s[3], w19 rev w17, w17 rbit v21.16b, v21.16b mov v16.s[3], w17 rev w16, w15 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v16.16b, v0.16b aesmc v16.16b, v16.16b ext v31.16b, v21.16b, v21.16b, #8 aese v17.16b, v0.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v2.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v3.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v17.16b, v3.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b ext v26.16b, v18.16b, v18.16b, #8 aese v14.16b, v5.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v15.16b, v5.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v15.16b, v6.16b aesmc v15.16b, v15.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v30.1q, v31.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 mov v28.d[1], v31.d[0] aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b # Done GHASH aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w14, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 bge L_aes_gcm_decrypt_arm64_crypto_eor3_192_both_4 L_aes_gcm_decrypt_arm64_crypto_eor3_192_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w14, #1 beq L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_1 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_done L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_2: add w20, w15, #1 mov v14.16b, v13.16b add w15, w15, #2 mov v15.16b, v13.16b rev w20, w20 mov v14.s[3], w20 rev w16, w15 mov v15.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w14, w14, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x0], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b st1 {v14.16b, v15.16b}, [x1], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_192_done L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_1: ld1 {v15.16b}, [x0], #16 add w15, w15, #1 mov v14.16b, v13.16b rbit v15.16b, v15.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v16.16b, v26.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v16.1d, v22.1d pmull2 v29.1q, v16.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b ext v31.16b, v16.16b, v16.16b, #8 aese v14.16b, v3.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v9.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v10.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b # Done GHASH rbit v15.16b, v15.16b eor v14.16b, v14.16b, v15.16b st1 {v14.16b}, [x1], #16 L_aes_gcm_decrypt_arm64_crypto_eor3_192_done: ands w14, w2, #15 beq L_aes_gcm_decrypt_arm64_crypto_eor3_192_partial_done eor v15.16b, v15.16b, v15.16b mov w19, w14 st1 {v15.2d}, [x11] cmp x19, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_dw ldr x17, [x0], #8 sub x19, x19, #8 str x17, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_dw: cmp x19, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_sw ldr w17, [x0], #4 sub x19, x19, #4 str w17, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_sw: cmp x19, #2 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_byte ldrh w17, [x0], #2 sub x19, x19, #2 strh w17, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_byte: cbz x19, L_aes_gcm_decrypt_arm64_crypto_eor3_192_end_bytes ldrb w17, [x0], #1 subs x19, x19, #1 strb w17, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_192_end_bytes: sub x11, x11, x14 ld1 {v15.2d}, [x11] add w15, w15, #1 mov v14.16b, v13.16b rbit v15.16b, v15.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v16.16b, v26.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v16.1d, v22.1d pmull2 v29.1q, v16.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b ext v31.16b, v16.16b, v16.16b, #8 aese v14.16b, v3.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v9.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v10.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b # Done GHASH rbit v15.16b, v15.16b eor v14.16b, v14.16b, v15.16b st1 {v14.2d}, [x11] cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_dw ldr x17, [x11], #8 sub w14, w14, #8 str x17, [x1], #8 L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_dw: cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_sw ldr w17, [x11], #4 sub w14, w14, #4 str w17, [x1], #4 L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_sw: cmp w14, #2 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_byte ldrh w17, [x11], #2 sub w14, w14, #2 strh w17, [x1], #2 L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_byte: cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_end_bytes ldrb w17, [x11], #1 subs w14, w14, #1 strb w17, [x1], #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_end_bytes: L_aes_gcm_decrypt_arm64_crypto_eor3_192_partial_done: ld1 {v14.2d}, [x12] lsl x8, x8, #3 rbit x8, x8 mov v28.d[0], x8 lsl x2, x2, #3 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b pmull v28.1q, v26.1d, v22.1d pmull2 v29.1q, v26.2d, v22.2d aese v14.16b, v0.16b aesmc v14.16b, v14.16b ext v31.16b, v26.16b, v26.16b, #8 aese v14.16b, v1.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v8.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b rbit v26.16b, v26.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b eor v26.16b, v26.16b, v14.16b cmp w6, #16 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_part_tag ld1 {v28.16b}, [x5] b L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_loaded L_aes_gcm_decrypt_arm64_crypto_eor3_192_part_tag: eor v28.16b, v28.16b, v28.16b mov x17, x6 st1 {v28.2d}, [x11] cmp x17, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_dw ldr x16, [x5], #8 sub x17, x17, #8 str x16, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_dw: cmp x17, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_sw ldr w16, [x5], #4 sub x17, x17, #4 str w16, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_sw: cmp x17, #2 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_byte ldrh w16, [x5], #2 sub x17, x17, #2 strh w16, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_byte: cbz x17, L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_end_bytes ldrb w16, [x5], #1 subs x17, x17, #1 strb w16, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_end_bytes: sub x11, x11, x6 ld1 {v28.2d}, [x11] mov x17, #16 st1 {v26.2d}, [x11] sub x17, x17, x6 add x11, x11, x6 L_aes_gcm_decrypt_arm64_crypto_eor3_192_calc_tag_byte: strb wzr, [x11], #1 subs x17, x17, #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_192_calc_tag_byte subs x11, x11, #16 ld1 {v26.2d}, [x11] L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_loaded: eor v28.16b, v28.16b, v26.16b mov x16, v28.d[0] mov x17, v28.d[1] mov w19, #-180 orr x16, x16, x17 cmp x16, #0 csetm x0, ne and x0, x0, x19 #endif /* !NO_AES_192 */ b L_aes_gcm_decrypt_arm64_crypto_eor3_done # AES_GCM_256 L_aes_gcm_decrypt_arm64_crypto_eor3_start_256: #ifndef NO_AES_256 cmp w14, #32 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_4 L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rev w24, w24 mov v14.s[3], w24 rev w23, w23 mov v15.s[3], w23 rev w22, w22 mov v16.s[3], w22 rev w21, w21 mov v17.s[3], w21 rev w20, w20 mov v8.s[3], w20 rev w19, w19 mov v9.s[3], w19 rev w17, w17 mov v10.s[3], w17 rev w16, w15 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v18.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v19.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v20.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v21.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x0], #16 aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x0], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x0], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b ld1 {v13.2d}, [x12] st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_end_8 L_aes_gcm_decrypt_arm64_crypto_eor3_256_both_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w24, w24 rbit v19.16b, v19.16b mov v14.s[3], w24 rev w23, w23 rbit v20.16b, v20.16b mov v15.s[3], w23 rev w22, w22 rbit v21.16b, v21.16b mov v16.s[3], w22 rev w21, w21 rbit v0.16b, v0.16b mov v17.s[3], w21 rev w20, w20 rbit v1.16b, v1.16b mov v8.s[3], w20 rev w19, w19 rbit v2.16b, v2.16b mov v9.s[3], w19 rev w17, w17 rbit v3.16b, v3.16b mov v10.s[3], w17 rev w16, w15 eor v18.16b, v18.16b, v26.16b mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v31.16b, v3.16b, v3.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b ext v26.16b, v2.16b, v2.16b, #8 aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b ext v26.16b, v1.16b, v1.16b, #8 aese v10.16b, v13.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b ext v26.16b, v0.16b, v0.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ext v26.16b, v21.16b, v21.16b, #8 aese v8.16b, v13.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v26.16b, v20.16b, v20.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ext v26.16b, v18.16b, v18.16b, #8 aese v15.16b, v12.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull2 v29.1q, v29.2d, v27.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b mov v28.d[1], v31.d[0] aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v26.16b, v28.16b, v30.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # Done GHASH aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v18.16b}, [x0], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v19.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v20.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v21.16b}, [x0], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v0.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v1.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v2.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v3.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x12] eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_decrypt_arm64_crypto_eor3_256_both_8 L_aes_gcm_decrypt_arm64_crypto_eor3_256_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 ld1 {v12.2d}, [x9], #16 cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_done beq L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_1 cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_2 add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rev w20, w20 mov v14.s[3], w20 rev w19, w19 mov v15.s[3], w19 rev w17, w17 mov v16.s[3], w17 rev w16, w15 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x0], #16 aese v17.16b, v7.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x0], #16 aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v20.16b}, [x0], #16 aese v17.16b, v8.16b aesmc v17.16b, v17.16b ld1 {v21.16b}, [x0], #16 aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w14, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_end_4 L_aes_gcm_decrypt_arm64_crypto_eor3_256_both_4: add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w20, w20 rbit v19.16b, v19.16b mov v14.s[3], w20 rev w19, w19 rbit v20.16b, v20.16b mov v15.s[3], w19 rev w17, w17 rbit v21.16b, v21.16b mov v16.s[3], w17 rev w16, w15 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v16.16b, v0.16b aesmc v16.16b, v16.16b ext v31.16b, v21.16b, v21.16b, #8 aese v17.16b, v0.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v2.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v3.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v17.16b, v3.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b ext v26.16b, v18.16b, v18.16b, #8 aese v14.16b, v5.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v15.16b, v5.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v15.16b, v6.16b aesmc v15.16b, v15.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v30.1q, v31.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 mov v28.d[1], v31.d[0] aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b # Done GHASH aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w14, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 bge L_aes_gcm_decrypt_arm64_crypto_eor3_256_both_4 L_aes_gcm_decrypt_arm64_crypto_eor3_256_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w14, #1 beq L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_1 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_done L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_2: add w20, w15, #1 mov v14.16b, v13.16b add w15, w15, #2 mov v15.16b, v13.16b rev w20, w20 mov v14.s[3], w20 rev w16, w15 mov v15.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w14, w14, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x0], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b ld1 {v29.2d, v30.2d}, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b st1 {v14.16b, v15.16b}, [x1], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_256_done L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_1: add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v14.16b, v11.16b aesmc v14.16b, v14.16b ldr q29, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ldr q30, [x9, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v14.16b, v14.16b, v18.16b st1 {v14.16b}, [x1], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_256_done: ands w14, w2, #15 beq L_aes_gcm_decrypt_arm64_crypto_eor3_256_partial_done eor v15.16b, v15.16b, v15.16b mov w19, w14 st1 {v15.2d}, [x11] cmp x19, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_dw ldr x17, [x0], #8 sub x19, x19, #8 str x17, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_dw: cmp x19, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_sw ldr w17, [x0], #4 sub x19, x19, #4 str w17, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_sw: cmp x19, #2 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_byte ldrh w17, [x0], #2 sub x19, x19, #2 strh w17, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_byte: cbz x19, L_aes_gcm_decrypt_arm64_crypto_eor3_256_end_bytes ldrb w17, [x0], #1 subs x19, x19, #1 strb w17, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_256_end_bytes: sub x11, x11, x14 ld1 {v15.2d}, [x11] add w15, w15, #1 mov v14.16b, v13.16b rbit v15.16b, v15.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v16.16b, v26.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v16.1d, v22.1d pmull2 v29.1q, v16.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b ext v31.16b, v16.16b, v16.16b, #8 aese v14.16b, v3.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v9.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v10.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b ldr q29, [x9] # Done GHASH aese v14.16b, v12.16b aesmc v14.16b, v14.16b ldr q30, [x9, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b rbit v15.16b, v15.16b eor v14.16b, v14.16b, v15.16b st1 {v14.2d}, [x11] cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_dw ldr x17, [x11], #8 sub w14, w14, #8 str x17, [x1], #8 L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_dw: cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_sw ldr w17, [x11], #4 sub w14, w14, #4 str w17, [x1], #4 L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_sw: cmp w14, #2 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_byte ldrh w17, [x11], #2 sub w14, w14, #2 strh w17, [x1], #2 L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_byte: cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_end_bytes ldrb w17, [x11], #1 subs w14, w14, #1 strb w17, [x1], #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_end_bytes: L_aes_gcm_decrypt_arm64_crypto_eor3_256_partial_done: ld1 {v14.2d}, [x12] lsl x8, x8, #3 rbit x8, x8 mov v28.d[0], x8 lsl x2, x2, #3 rbit x2, x2 mov v28.d[1], x2 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v28.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b pmull v28.1q, v26.1d, v22.1d pmull2 v29.1q, v26.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b ext v31.16b, v26.16b, v26.16b, #8 aese v14.16b, v3.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v9.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] ldr q11, [x9, #-32] aese v14.16b, v10.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b ldr q12, [x9, #-16] aese v14.16b, v11.16b aesmc v14.16b, v14.16b ldr q29, [x9] aese v14.16b, v12.16b aesmc v14.16b, v14.16b rbit v26.16b, v26.16b ldr q30, [x9, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v26.16b, v26.16b, v14.16b cmp w6, #16 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_part_tag ld1 {v28.16b}, [x5] b L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_loaded L_aes_gcm_decrypt_arm64_crypto_eor3_256_part_tag: eor v28.16b, v28.16b, v28.16b mov x17, x6 st1 {v28.2d}, [x11] cmp x17, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_dw ldr x16, [x5], #8 sub x17, x17, #8 str x16, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_dw: cmp x17, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_sw ldr w16, [x5], #4 sub x17, x17, #4 str w16, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_sw: cmp x17, #2 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_byte ldrh w16, [x5], #2 sub x17, x17, #2 strh w16, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_byte: cbz x17, L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_end_bytes ldrb w16, [x5], #1 subs x17, x17, #1 strb w16, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_end_bytes: sub x11, x11, x6 ld1 {v28.2d}, [x11] mov x17, #16 st1 {v26.2d}, [x11] sub x17, x17, x6 add x11, x11, x6 L_aes_gcm_decrypt_arm64_crypto_eor3_256_calc_tag_byte: strb wzr, [x11], #1 subs x17, x17, #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_256_calc_tag_byte subs x11, x11, #16 ld1 {v26.2d}, [x11] L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_loaded: eor v28.16b, v28.16b, v26.16b mov x16, v28.d[0] mov x17, v28.d[1] mov w19, #-180 orr x16, x16, x17 cmp x16, #0 csetm x0, ne and x0, x0, x19 #endif /* !NO_AES_256 */ b L_aes_gcm_decrypt_arm64_crypto_eor3_done # AES_GCM_128 L_aes_gcm_decrypt_arm64_crypto_eor3_start_128: #ifndef NO_AES_128 cmp w14, #32 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_4 L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rev w24, w24 mov v14.s[3], w24 rev w23, w23 mov v15.s[3], w23 rev w22, w22 mov v16.s[3], w22 rev w21, w21 mov v17.s[3], w21 rev w20, w20 mov v8.s[3], w20 rev w19, w19 mov v9.s[3], w19 rev w17, w17 mov v10.s[3], w17 rev w16, w15 mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v18.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v19.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v20.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v21.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b ld1 {v0.16b}, [x0], #16 aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b ld1 {v1.16b}, [x0], #16 aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b ld1 {v2.16b}, [x0], #16 aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b ld1 {v3.16b}, [x0], #16 aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b ld1 {v13.2d}, [x12] st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_end_8 L_aes_gcm_decrypt_arm64_crypto_eor3_128_both_8: ldr q12, [x9] add w24, w15, #1 mov v14.16b, v13.16b add w23, w15, #2 mov v15.16b, v13.16b add w22, w15, #3 mov v16.16b, v13.16b add w21, w15, #4 mov v17.16b, v13.16b add w20, w15, #5 mov v8.16b, v13.16b add w19, w15, #6 mov v9.16b, v13.16b add w17, w15, #7 mov v10.16b, v13.16b add w15, w15, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w24, w24 rbit v19.16b, v19.16b mov v14.s[3], w24 rev w23, w23 rbit v20.16b, v20.16b mov v15.s[3], w23 rev w22, w22 rbit v21.16b, v21.16b mov v16.s[3], w22 rev w21, w21 rbit v0.16b, v0.16b mov v17.s[3], w21 rev w20, w20 rbit v1.16b, v1.16b mov v8.s[3], w20 rev w19, w19 rbit v2.16b, v2.16b mov v9.s[3], w19 rev w17, w17 rbit v3.16b, v3.16b mov v10.s[3], w17 rev w16, w15 eor v18.16b, v18.16b, v26.16b mov v11.s[3], w16 ldr q13, [x9, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v31.16b, v3.16b, v3.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b ext v26.16b, v2.16b, v2.16b, #8 aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b ext v26.16b, v1.16b, v1.16b, #8 aese v10.16b, v13.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b ext v26.16b, v0.16b, v0.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ext v26.16b, v21.16b, v21.16b, #8 aese v8.16b, v13.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v26.16b, v20.16b, v20.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ext v26.16b, v18.16b, v18.16b, #8 aese v15.16b, v12.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull2 v29.1q, v29.2d, v27.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w14, w14, #8 ldr q12, [x9, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b mov v28.d[1], v31.d[0] aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v26.16b, v28.16b, v30.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # Done GHASH aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v18.16b}, [x0], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v19.16b}, [x0], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v20.16b}, [x0], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x9, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v21.16b}, [x0], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v0.16b}, [x0], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v1.16b}, [x0], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v2.16b}, [x0], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v3.16b}, [x0], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x9, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x12] eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_decrypt_arm64_crypto_eor3_128_both_8 L_aes_gcm_decrypt_arm64_crypto_eor3_128_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 ld1 {v8.2d, v9.2d}, [x9], #32 ld1 {v10.2d}, [x9] cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_done beq L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_1 cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_2 add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rev w20, w20 mov v14.s[3], w20 rev w19, w19 mov v15.s[3], w19 rev w17, w17 mov v16.s[3], w17 rev w16, w15 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x0], #16 aese v17.16b, v7.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x0], #16 aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v20.16b}, [x0], #16 aese v17.16b, v8.16b aesmc v17.16b, v17.16b ld1 {v21.16b}, [x0], #16 aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w14, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_end_4 L_aes_gcm_decrypt_arm64_crypto_eor3_128_both_4: add w20, w15, #1 mov v14.16b, v13.16b add w19, w15, #2 mov v15.16b, v13.16b add w17, w15, #3 mov v16.16b, v13.16b add w15, w15, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w20, w20 rbit v19.16b, v19.16b mov v14.s[3], w20 rev w19, w19 rbit v20.16b, v20.16b mov v15.s[3], w19 rev w17, w17 rbit v21.16b, v21.16b mov v16.s[3], w17 rev w16, w15 mov v17.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v16.16b, v0.16b aesmc v16.16b, v16.16b ext v31.16b, v21.16b, v21.16b, #8 aese v17.16b, v0.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v2.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v3.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v17.16b, v3.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b ext v26.16b, v18.16b, v18.16b, #8 aese v14.16b, v5.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v15.16b, v5.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v15.16b, v6.16b aesmc v15.16b, v15.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v30.1q, v31.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w14, w14, #4 mov v28.d[1], v31.d[0] aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b # Done GHASH aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w14, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 bge L_aes_gcm_decrypt_arm64_crypto_eor3_128_both_4 L_aes_gcm_decrypt_arm64_crypto_eor3_128_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w14, #1 beq L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_1 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_done L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_2: add w20, w15, #1 mov v14.16b, v13.16b add w15, w15, #2 mov v15.16b, v13.16b rev w20, w20 mov v14.s[3], w20 rev w16, w15 mov v15.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w14, w14, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x0], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v19.16b}, [x0], #16 aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b st1 {v14.16b, v15.16b}, [x1], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_128_done L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_1: add w15, w15, #1 mov v14.16b, v13.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v18.16b}, [x0], #16 eor v14.16b, v14.16b, v18.16b st1 {v14.16b}, [x1], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_128_done: ands w14, w2, #15 beq L_aes_gcm_decrypt_arm64_crypto_eor3_128_partial_done eor v15.16b, v15.16b, v15.16b mov w19, w14 st1 {v15.2d}, [x11] cmp x19, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_dw ldr x17, [x0], #8 sub x19, x19, #8 str x17, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_dw: cmp x19, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_sw ldr w17, [x0], #4 sub x19, x19, #4 str w17, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_sw: cmp x19, #2 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_byte ldrh w17, [x0], #2 sub x19, x19, #2 strh w17, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_byte: cbz x19, L_aes_gcm_decrypt_arm64_crypto_eor3_128_end_bytes ldrb w17, [x0], #1 subs x19, x19, #1 strb w17, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_128_end_bytes: sub x11, x11, x14 ld1 {v15.2d}, [x11] add w15, w15, #1 mov v14.16b, v13.16b rbit v15.16b, v15.16b rev w16, w15 mov v14.s[3], w16 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v16.16b, v26.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v16.1d, v22.1d pmull2 v29.1q, v16.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b ext v31.16b, v16.16b, v16.16b, #8 aese v14.16b, v3.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH rbit v15.16b, v15.16b eor v14.16b, v14.16b, v15.16b st1 {v14.2d}, [x11] cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_dw ldr x17, [x11], #8 sub w14, w14, #8 str x17, [x1], #8 L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_dw: cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_sw ldr w17, [x11], #4 sub w14, w14, #4 str w17, [x1], #4 L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_sw: cmp w14, #2 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_byte ldrh w17, [x11], #2 sub w14, w14, #2 strh w17, [x1], #2 L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_byte: cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_end_bytes ldrb w17, [x11], #1 subs w14, w14, #1 strb w17, [x1], #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_end_bytes: L_aes_gcm_decrypt_arm64_crypto_eor3_128_partial_done: ld1 {v14.2d}, [x12] lsl x8, x8, #3 rbit x8, x8 mov v28.d[0], x8 lsl x2, x2, #3 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b pmull v28.1q, v26.1d, v22.1d pmull2 v29.1q, v26.2d, v22.2d ext v31.16b, v26.16b, v26.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v6.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b rbit v26.16b, v26.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b eor v26.16b, v26.16b, v14.16b cmp w6, #16 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_part_tag ld1 {v28.16b}, [x5] b L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_loaded L_aes_gcm_decrypt_arm64_crypto_eor3_128_part_tag: eor v28.16b, v28.16b, v28.16b mov x17, x6 st1 {v28.2d}, [x11] cmp x17, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_dw ldr x16, [x5], #8 sub x17, x17, #8 str x16, [x11], #8 L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_dw: cmp x17, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_sw ldr w16, [x5], #4 sub x17, x17, #4 str w16, [x11], #4 L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_sw: cmp x17, #2 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_byte ldrh w16, [x5], #2 sub x17, x17, #2 strh w16, [x11], #2 L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_byte: cbz x17, L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_end_bytes ldrb w16, [x5], #1 subs x17, x17, #1 strb w16, [x11], #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_end_bytes: sub x11, x11, x6 ld1 {v28.2d}, [x11] mov x17, #16 st1 {v26.2d}, [x11] sub x17, x17, x6 add x11, x11, x6 L_aes_gcm_decrypt_arm64_crypto_eor3_128_calc_tag_byte: strb wzr, [x11], #1 subs x17, x17, #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_128_calc_tag_byte subs x11, x11, #16 ld1 {v26.2d}, [x11] L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_loaded: eor v28.16b, v28.16b, v26.16b mov x16, v28.d[0] mov x17, v28.d[1] mov w19, #-180 orr x16, x16, x17 cmp x16, #0 csetm x0, ne and x0, x0, x19 #endif /* !NO_AES_128 */ L_aes_gcm_decrypt_arm64_crypto_eor3_done: ldp x17, x19, [x29, #24] ldp x20, x21, [x29, #40] ldp x22, x23, [x29, #56] ldr x24, [x29, #72] ldp d8, d9, [x29, #80] ldp d10, d11, [x29, #96] ldp d12, d13, [x29, #112] ldp d14, d15, [x29, #128] ldp x29, x30, [sp], #0x90 ret #ifndef __APPLE__ .size AES_GCM_decrypt_AARCH64_EOR3,.-AES_GCM_decrypt_AARCH64_EOR3 #endif /* __APPLE__ */ #endif /* HAVE_AES_DECRYPT */ #endif /* !WOLFSSL_ARMASM_CRYPTO_SHA3 */ #ifdef WOLFSSL_AESGCM_STREAM #ifndef __APPLE__ .text .globl AES_GCM_init_AARCH64 .type AES_GCM_init_AARCH64,@function .align 2 AES_GCM_init_AARCH64: #else .section __TEXT,__text .globl _AES_GCM_init_AARCH64 .p2align 2 _AES_GCM_init_AARCH64: #endif /* __APPLE__ */ stp x29, x30, [sp, #-48]! add x29, sp, #0 stp d8, d9, [x29, #16] stp d10, d11, [x29, #32] movi v6.16b, #0x87 ld1 {v5.2d}, [x4] ushr v6.2d, v6.2d, #56 # Load Nonce cmp w3, #12 bne L_aes_gcm_init_arm64_crypto_ghash_nonce ldr x9, [x2] movi v4.4s, #1, lsl 24 ldr w10, [x2, #8] mov v4.d[0], x9 mov v4.s[2], w10 mov w8, #1 b L_aes_gcm_init_arm64_crypto_done_nonce L_aes_gcm_init_arm64_crypto_ghash_nonce: eor v4.16b, v4.16b, v4.16b lsr w7, w3, #4 cbz w7, L_aes_gcm_init_arm64_crypto_done L_aes_gcm_init_arm64_crypto_start_1: ld1 {v0.16b}, [x2], #16 rbit v0.16b, v0.16b eor v3.16b, v4.16b, v0.16b # X = C * H^1 pmull v7.1q, v3.1d, v5.1d pmull2 v8.1q, v3.2d, v5.2d ext v10.16b, v3.16b, v3.16b, #8 pmull v9.1q, v10.1d, v5.1d pmull2 v10.1q, v10.2d, v5.2d eor v9.16b, v9.16b, v10.16b # Reduce ext v10.16b, v7.16b, v8.16b, #8 pmull2 v8.1q, v8.2d, v6.2d eor v10.16b, v10.16b, v8.16b eor v10.16b, v10.16b, v9.16b pmull2 v9.1q, v10.2d, v6.2d mov v7.d[1], v10.d[0] eor v4.16b, v7.16b, v9.16b # Done GHASH subs w7, w7, #1 bne L_aes_gcm_init_arm64_crypto_start_1 L_aes_gcm_init_arm64_crypto_done: and w13, w3, #15 cbz x13, L_aes_gcm_init_arm64_crypto_partial_done eor v7.16b, v7.16b, v7.16b mov w12, w13 st1 {v7.2d}, [x6] cmp w12, #8 blt L_aes_gcm_init_arm64_crypto_start_dw ldr x11, [x2], #8 sub w12, w12, #8 str x11, [x6], #8 L_aes_gcm_init_arm64_crypto_start_dw: cmp w12, #4 blt L_aes_gcm_init_arm64_crypto_start_sw ldr w11, [x2], #4 sub w12, w12, #4 str w11, [x6], #4 L_aes_gcm_init_arm64_crypto_start_sw: cmp w12, #2 blt L_aes_gcm_init_arm64_crypto_start_byte ldrh w11, [x2], #2 sub w12, w12, #2 strh w11, [x6], #2 L_aes_gcm_init_arm64_crypto_start_byte: cbz w12, L_aes_gcm_init_arm64_crypto_end_bytes ldrb w11, [x2], #1 subs w12, w12, #1 strb w11, [x6], #1 bne L_aes_gcm_init_arm64_crypto_start_byte L_aes_gcm_init_arm64_crypto_end_bytes: sub x6, x6, x13 ld1 {v0.2d}, [x6] rbit v0.16b, v0.16b eor v3.16b, v4.16b, v0.16b # X = C * H^1 pmull v7.1q, v3.1d, v5.1d pmull2 v8.1q, v3.2d, v5.2d ext v10.16b, v3.16b, v3.16b, #8 pmull v9.1q, v10.1d, v5.1d pmull2 v10.1q, v10.2d, v5.2d eor v9.16b, v9.16b, v10.16b # Reduce ext v10.16b, v7.16b, v8.16b, #8 pmull2 v8.1q, v8.2d, v6.2d eor v10.16b, v10.16b, v8.16b eor v10.16b, v10.16b, v9.16b pmull2 v9.1q, v10.2d, v6.2d mov v7.d[1], v10.d[0] eor v4.16b, v7.16b, v9.16b # Done GHASH L_aes_gcm_init_arm64_crypto_partial_done: eor x7, x7, x7 lsl x13, x3, #3 mov v7.d[0], x7 mov v7.d[1], x13 rev64 v7.16b, v7.16b rbit v7.16b, v7.16b eor v4.16b, v4.16b, v7.16b pmull v7.1q, v4.1d, v5.1d pmull2 v8.1q, v4.2d, v5.2d ext v10.16b, v4.16b, v4.16b, #8 pmull v9.1q, v10.1d, v5.1d pmull2 v10.1q, v10.2d, v5.2d eor v9.16b, v9.16b, v10.16b ext v10.16b, v7.16b, v8.16b, #8 pmull2 v8.1q, v8.2d, v6.2d eor v10.16b, v10.16b, v8.16b eor v10.16b, v10.16b, v9.16b pmull2 v9.1q, v10.2d, v6.2d mov v7.d[1], v10.d[0] eor v4.16b, v7.16b, v9.16b rbit v4.16b, v4.16b mov w8, v4.s[3] rev w8, w8 L_aes_gcm_init_arm64_crypto_done_nonce: st1 {v4.2d}, [x5] ld1 {v7.2d, v8.2d, v9.2d, v10.2d}, [x0], #0x40 aese v4.16b, v7.16b aesmc v4.16b, v4.16b aese v4.16b, v8.16b aesmc v4.16b, v4.16b aese v4.16b, v9.16b aesmc v4.16b, v4.16b aese v4.16b, v10.16b aesmc v4.16b, v4.16b ld1 {v7.2d, v8.2d, v9.2d, v10.2d}, [x0], #0x40 aese v4.16b, v7.16b aesmc v4.16b, v4.16b aese v4.16b, v8.16b aesmc v4.16b, v4.16b aese v4.16b, v9.16b aesmc v4.16b, v4.16b aese v4.16b, v10.16b aesmc v4.16b, v4.16b subs w1, w1, #10 ld1 {v7.2d, v8.2d}, [x0], #32 aese v4.16b, v7.16b aesmc v4.16b, v4.16b aese v4.16b, v8.16b beq L_aes_gcm_init_arm64_crypto_round_done ld1 {v7.2d, v8.2d}, [x0], #32 subs w1, w1, #2 aesmc v4.16b, v4.16b aese v4.16b, v7.16b aesmc v4.16b, v4.16b aese v4.16b, v8.16b beq L_aes_gcm_init_arm64_crypto_round_done ld1 {v7.2d, v8.2d}, [x0], #32 aesmc v4.16b, v4.16b aese v4.16b, v7.16b aesmc v4.16b, v4.16b aese v4.16b, v8.16b L_aes_gcm_init_arm64_crypto_round_done: ld1 {v7.2d}, [x0] eor v4.16b, v4.16b, v7.16b st1 {v4.2d}, [x6] ldp d8, d9, [x29, #16] ldp d10, d11, [x29, #32] ldp x29, x30, [sp], #48 ret #ifndef __APPLE__ .size AES_GCM_init_AARCH64,.-AES_GCM_init_AARCH64 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_ghash_block_AARCH64 .type AES_GCM_ghash_block_AARCH64,@function .align 2 AES_GCM_ghash_block_AARCH64: #else .section __TEXT,__text .globl _AES_GCM_ghash_block_AARCH64 .p2align 2 _AES_GCM_ghash_block_AARCH64: #endif /* __APPLE__ */ stp x29, x30, [sp, #-32]! add x29, sp, #0 stp d8, d9, [x29, #16] ld1 {v6.2d}, [x1] movi v7.16b, #0x87 ld1 {v5.2d}, [x2] ushr v7.2d, v7.2d, #56 ld1 {v4.2d}, [x0] rbit v4.16b, v4.16b eor v8.16b, v6.16b, v4.16b # X = C * H^1 pmull v0.1q, v8.1d, v5.1d pmull2 v1.1q, v8.2d, v5.2d ext v3.16b, v8.16b, v8.16b, #8 pmull v2.1q, v3.1d, v5.1d pmull2 v3.1q, v3.2d, v5.2d eor v2.16b, v2.16b, v3.16b # Reduce ext v3.16b, v0.16b, v1.16b, #8 pmull2 v1.1q, v1.2d, v7.2d eor v3.16b, v3.16b, v1.16b eor v3.16b, v3.16b, v2.16b pmull2 v2.1q, v3.2d, v7.2d mov v0.d[1], v3.d[0] eor v6.16b, v0.16b, v2.16b # Done GHASH st1 {v6.2d}, [x1] ldp d8, d9, [x29, #16] ldp x29, x30, [sp], #32 ret #ifndef __APPLE__ .size AES_GCM_ghash_block_AARCH64,.-AES_GCM_ghash_block_AARCH64 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_aad_update_AARCH64 .type AES_GCM_aad_update_AARCH64,@function .align 2 AES_GCM_aad_update_AARCH64: #else .section __TEXT,__text .globl _AES_GCM_aad_update_AARCH64 .p2align 2 _AES_GCM_aad_update_AARCH64: #endif /* __APPLE__ */ stp x29, x30, [sp, #-80]! add x29, sp, #0 stp d8, d9, [x29, #16] stp d10, d11, [x29, #32] stp d12, d13, [x29, #48] stp d14, d15, [x29, #64] ld1 {v20.2d}, [x2] movi v21.16b, #0x87 ld1 {v12.2d}, [x3] ushr v21.2d, v21.2d, #56 cmp x1, #0x40 blt L_aes_gcm_aad_update_arm64_crypto_h_done # Square H => H^2 pmull2 v11.1q, v12.2d, v12.2d pmull v10.1q, v12.1d, v12.1d pmull2 v8.1q, v11.2d, v21.2d ext v9.16b, v10.16b, v11.16b, #8 eor v9.16b, v9.16b, v8.16b pmull2 v11.1q, v9.2d, v21.2d mov v10.d[1], v9.d[0] eor v13.16b, v10.16b, v11.16b cmp x1, #0x100 blt L_aes_gcm_aad_update_arm64_crypto_h_done # Multiply H and H^2 => H^3 pmull v8.1q, v12.1d, v13.1d pmull2 v9.1q, v12.2d, v13.2d ext v11.16b, v12.16b, v12.16b, #8 pmull v10.1q, v11.1d, v13.1d pmull2 v11.1q, v11.2d, v13.2d eor v10.16b, v10.16b, v11.16b # Reduce ext v11.16b, v8.16b, v9.16b, #8 pmull2 v9.1q, v9.2d, v21.2d eor v11.16b, v11.16b, v9.16b eor v11.16b, v11.16b, v10.16b pmull2 v10.1q, v11.2d, v21.2d mov v8.d[1], v11.d[0] eor v14.16b, v8.16b, v10.16b # Square H^2 => H^4 pmull2 v11.1q, v13.2d, v13.2d pmull v10.1q, v13.1d, v13.1d pmull2 v8.1q, v11.2d, v21.2d ext v9.16b, v10.16b, v11.16b, #8 eor v9.16b, v9.16b, v8.16b pmull2 v11.1q, v9.2d, v21.2d mov v10.d[1], v9.d[0] eor v15.16b, v10.16b, v11.16b # Done cmp x1, #0x400 blt L_aes_gcm_aad_update_arm64_crypto_h_done # Multiply H and H^4 => H^5 pmull v8.1q, v12.1d, v15.1d pmull2 v9.1q, v12.2d, v15.2d ext v11.16b, v12.16b, v12.16b, #8 pmull v10.1q, v11.1d, v15.1d pmull2 v11.1q, v11.2d, v15.2d eor v10.16b, v10.16b, v11.16b # Reduce ext v11.16b, v8.16b, v9.16b, #8 pmull2 v9.1q, v9.2d, v21.2d eor v11.16b, v11.16b, v9.16b eor v11.16b, v11.16b, v10.16b pmull2 v10.1q, v11.2d, v21.2d mov v8.d[1], v11.d[0] eor v16.16b, v8.16b, v10.16b # Square H^3 => H^6 pmull2 v11.1q, v14.2d, v14.2d pmull v10.1q, v14.1d, v14.1d pmull2 v8.1q, v11.2d, v21.2d ext v9.16b, v10.16b, v11.16b, #8 eor v9.16b, v9.16b, v8.16b pmull2 v11.1q, v9.2d, v21.2d mov v10.d[1], v9.d[0] eor v17.16b, v10.16b, v11.16b # Multiply H and H^6 => H^7 pmull v8.1q, v12.1d, v17.1d pmull2 v9.1q, v12.2d, v17.2d ext v11.16b, v12.16b, v12.16b, #8 pmull v10.1q, v11.1d, v17.1d pmull2 v11.1q, v11.2d, v17.2d eor v10.16b, v10.16b, v11.16b # Reduce ext v11.16b, v8.16b, v9.16b, #8 pmull2 v9.1q, v9.2d, v21.2d eor v11.16b, v11.16b, v9.16b eor v11.16b, v11.16b, v10.16b pmull2 v10.1q, v11.2d, v21.2d mov v8.d[1], v11.d[0] eor v18.16b, v8.16b, v10.16b # Square H^4 => H^8 pmull2 v11.1q, v15.2d, v15.2d pmull v10.1q, v15.1d, v15.1d pmull2 v8.1q, v11.2d, v21.2d ext v9.16b, v10.16b, v11.16b, #8 eor v9.16b, v9.16b, v8.16b pmull2 v11.1q, v9.2d, v21.2d mov v10.d[1], v9.d[0] eor v19.16b, v10.16b, v11.16b # Done L_aes_gcm_aad_update_arm64_crypto_h_done: lsr x1, x1, #4 cmp x1, #4 blt L_aes_gcm_aad_update_arm64_crypto_start_1 cmp x1, #16 blt L_aes_gcm_aad_update_arm64_crypto_start_2 cmp x1, #0x40 blt L_aes_gcm_aad_update_arm64_crypto_start_4 L_aes_gcm_aad_update_arm64_crypto_start_8: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b rbit v4.16b, v4.16b rbit v5.16b, v5.16b rbit v6.16b, v6.16b rbit v7.16b, v7.16b eor v0.16b, v0.16b, v20.16b # X = C * H^1 pmull v8.1q, v7.1d, v12.1d pmull2 v9.1q, v7.2d, v12.2d ext v11.16b, v7.16b, v7.16b, #8 pmull v10.1q, v11.1d, v12.1d pmull2 v11.1q, v11.2d, v12.2d eor v10.16b, v10.16b, v11.16b # X += C * H^2 pmull v11.1q, v13.1d, v6.1d pmull2 v20.1q, v13.2d, v6.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v6.16b, v6.16b, #8 pmull v11.1q, v20.1d, v13.1d pmull2 v20.1q, v20.2d, v13.2d eor v20.16b, v20.16b, v11.16b eor v10.16b, v10.16b, v20.16b # X += C * H^3 pmull v11.1q, v14.1d, v5.1d pmull2 v20.1q, v14.2d, v5.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v5.16b, v5.16b, #8 pmull v11.1q, v20.1d, v14.1d pmull2 v20.1q, v20.2d, v14.2d eor v20.16b, v20.16b, v11.16b eor v10.16b, v10.16b, v20.16b # X += C * H^4 pmull v11.1q, v15.1d, v4.1d pmull2 v20.1q, v15.2d, v4.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v4.16b, v4.16b, #8 pmull v11.1q, v20.1d, v15.1d pmull2 v20.1q, v20.2d, v15.2d eor v20.16b, v20.16b, v11.16b eor v10.16b, v10.16b, v20.16b # X += C * H^5 pmull v11.1q, v16.1d, v3.1d pmull2 v20.1q, v16.2d, v3.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v3.16b, v3.16b, #8 pmull v11.1q, v20.1d, v16.1d pmull2 v20.1q, v20.2d, v16.2d eor v20.16b, v20.16b, v11.16b eor v10.16b, v10.16b, v20.16b # X += C * H^6 pmull v11.1q, v17.1d, v2.1d pmull2 v20.1q, v17.2d, v2.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v2.16b, v2.16b, #8 pmull v11.1q, v20.1d, v17.1d pmull2 v20.1q, v20.2d, v17.2d eor v20.16b, v20.16b, v11.16b eor v10.16b, v10.16b, v20.16b # X += C * H^7 pmull v11.1q, v18.1d, v1.1d pmull2 v20.1q, v18.2d, v1.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v1.16b, v1.16b, #8 pmull v11.1q, v20.1d, v18.1d pmull2 v20.1q, v20.2d, v18.2d eor v20.16b, v20.16b, v11.16b eor v10.16b, v10.16b, v20.16b # X += C * H^8 pmull v11.1q, v19.1d, v0.1d pmull2 v20.1q, v19.2d, v0.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v0.16b, v0.16b, #8 pmull v11.1q, v20.1d, v19.1d pmull2 v20.1q, v20.2d, v19.2d eor v20.16b, v20.16b, v11.16b eor v10.16b, v10.16b, v20.16b # Reduce ext v11.16b, v8.16b, v9.16b, #8 pmull2 v9.1q, v9.2d, v21.2d eor v11.16b, v11.16b, v9.16b eor v11.16b, v11.16b, v10.16b pmull2 v10.1q, v11.2d, v21.2d mov v8.d[1], v11.d[0] eor v20.16b, v8.16b, v10.16b # Done GHASH sub x1, x1, #8 cmp x1, #8 bge L_aes_gcm_aad_update_arm64_crypto_start_8 cmp x1, #1 blt L_aes_gcm_aad_update_arm64_crypto_done beq L_aes_gcm_aad_update_arm64_crypto_start_1 cmp x1, #16 blt L_aes_gcm_aad_update_arm64_crypto_start_2 L_aes_gcm_aad_update_arm64_crypto_start_4: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v0.16b, v0.16b, v20.16b # X = C * H^1 pmull v8.1q, v3.1d, v12.1d pmull2 v9.1q, v3.2d, v12.2d ext v11.16b, v3.16b, v3.16b, #8 pmull v10.1q, v11.1d, v12.1d pmull2 v11.1q, v11.2d, v12.2d eor v10.16b, v10.16b, v11.16b # X += C * H^2 pmull v11.1q, v13.1d, v2.1d pmull2 v20.1q, v13.2d, v2.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v2.16b, v2.16b, #8 pmull v11.1q, v20.1d, v13.1d pmull2 v20.1q, v20.2d, v13.2d eor v20.16b, v20.16b, v11.16b eor v10.16b, v10.16b, v20.16b # X += C * H^3 pmull v11.1q, v14.1d, v1.1d pmull2 v20.1q, v14.2d, v1.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v1.16b, v1.16b, #8 pmull v11.1q, v20.1d, v14.1d pmull2 v20.1q, v20.2d, v14.2d eor v20.16b, v20.16b, v11.16b eor v10.16b, v10.16b, v20.16b # X += C * H^4 pmull v11.1q, v15.1d, v0.1d pmull2 v20.1q, v15.2d, v0.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v0.16b, v0.16b, #8 pmull v11.1q, v20.1d, v15.1d pmull2 v20.1q, v20.2d, v15.2d eor v20.16b, v20.16b, v11.16b eor v10.16b, v10.16b, v20.16b # Reduce ext v11.16b, v8.16b, v9.16b, #8 pmull2 v9.1q, v9.2d, v21.2d eor v11.16b, v11.16b, v9.16b eor v11.16b, v11.16b, v10.16b pmull2 v10.1q, v11.2d, v21.2d mov v8.d[1], v11.d[0] eor v20.16b, v8.16b, v10.16b # Done GHASH sub x1, x1, #4 cmp x1, #4 bge L_aes_gcm_aad_update_arm64_crypto_start_4 cmp x1, #1 blt L_aes_gcm_aad_update_arm64_crypto_done beq L_aes_gcm_aad_update_arm64_crypto_start_1 L_aes_gcm_aad_update_arm64_crypto_start_2: ld1 {v0.16b, v1.16b}, [x0], #32 rbit v0.16b, v0.16b rbit v1.16b, v1.16b eor v3.16b, v20.16b, v0.16b # X = C * H^1 pmull v8.1q, v1.1d, v12.1d pmull2 v9.1q, v1.2d, v12.2d ext v11.16b, v1.16b, v1.16b, #8 pmull v10.1q, v11.1d, v12.1d pmull2 v11.1q, v11.2d, v12.2d eor v10.16b, v10.16b, v11.16b # X += C * H^2 pmull v11.1q, v13.1d, v3.1d pmull2 v20.1q, v13.2d, v3.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v3.16b, v3.16b, #8 pmull v11.1q, v20.1d, v13.1d pmull2 v20.1q, v20.2d, v13.2d eor v20.16b, v20.16b, v11.16b eor v10.16b, v10.16b, v20.16b # Reduce ext v11.16b, v8.16b, v9.16b, #8 pmull2 v9.1q, v9.2d, v21.2d eor v11.16b, v11.16b, v9.16b eor v11.16b, v11.16b, v10.16b pmull2 v10.1q, v11.2d, v21.2d mov v8.d[1], v11.d[0] eor v20.16b, v8.16b, v10.16b # Done GHASH sub x1, x1, #2 cmp x1, #1 bgt L_aes_gcm_aad_update_arm64_crypto_start_2 blt L_aes_gcm_aad_update_arm64_crypto_done L_aes_gcm_aad_update_arm64_crypto_start_1: cbz x1, L_aes_gcm_aad_update_arm64_crypto_done L_aes_gcm_aad_update_arm64_crypto_both_1: ld1 {v0.16b}, [x0], #16 rbit v0.16b, v0.16b eor v3.16b, v20.16b, v0.16b # X = C * H^1 pmull v8.1q, v3.1d, v12.1d pmull2 v9.1q, v3.2d, v12.2d ext v11.16b, v3.16b, v3.16b, #8 pmull v10.1q, v11.1d, v12.1d pmull2 v11.1q, v11.2d, v12.2d eor v10.16b, v10.16b, v11.16b # Reduce ext v11.16b, v8.16b, v9.16b, #8 pmull2 v9.1q, v9.2d, v21.2d eor v11.16b, v11.16b, v9.16b eor v11.16b, v11.16b, v10.16b pmull2 v10.1q, v11.2d, v21.2d mov v8.d[1], v11.d[0] eor v20.16b, v8.16b, v10.16b # Done GHASH subs x1, x1, #1 bne L_aes_gcm_aad_update_arm64_crypto_both_1 L_aes_gcm_aad_update_arm64_crypto_done: st1 {v20.2d}, [x2] ldp d8, d9, [x29, #16] ldp d10, d11, [x29, #32] ldp d12, d13, [x29, #48] ldp d14, d15, [x29, #64] ldp x29, x30, [sp], #0x50 ret #ifndef __APPLE__ .size AES_GCM_aad_update_AARCH64,.-AES_GCM_aad_update_AARCH64 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_encrypt_block_AARCH64 .type AES_GCM_encrypt_block_AARCH64,@function .align 2 AES_GCM_encrypt_block_AARCH64: #else .section __TEXT,__text .globl _AES_GCM_encrypt_block_AARCH64 .p2align 2 _AES_GCM_encrypt_block_AARCH64: #endif /* __APPLE__ */ ld1 {v5.2d}, [x4] ld1 {v4.2d}, [x3] mov w5, v5.s[3] rev w5, w5 add w5, w5, #1 rev w5, w5 mov v5.s[3], w5 st1 {v5.2d}, [x4] ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 aese v5.16b, v0.16b aesmc v5.16b, v5.16b aese v5.16b, v1.16b aesmc v5.16b, v5.16b aese v5.16b, v2.16b aesmc v5.16b, v5.16b aese v5.16b, v3.16b aesmc v5.16b, v5.16b ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 aese v5.16b, v0.16b aesmc v5.16b, v5.16b aese v5.16b, v1.16b aesmc v5.16b, v5.16b aese v5.16b, v2.16b aesmc v5.16b, v5.16b aese v5.16b, v3.16b aesmc v5.16b, v5.16b subs w1, w1, #10 ld1 {v0.2d, v1.2d}, [x0], #32 aese v5.16b, v0.16b aesmc v5.16b, v5.16b aese v5.16b, v1.16b beq L_aes_gcm_encrypt_block_arm64_crypto_round_done ld1 {v0.2d, v1.2d}, [x0], #32 subs w1, w1, #2 aesmc v5.16b, v5.16b aese v5.16b, v0.16b aesmc v5.16b, v5.16b aese v5.16b, v1.16b beq L_aes_gcm_encrypt_block_arm64_crypto_round_done ld1 {v0.2d, v1.2d}, [x0], #32 aesmc v5.16b, v5.16b aese v5.16b, v0.16b aesmc v5.16b, v5.16b aese v5.16b, v1.16b L_aes_gcm_encrypt_block_arm64_crypto_round_done: ld1 {v0.2d}, [x0] eor v5.16b, v5.16b, v0.16b eor v4.16b, v4.16b, v5.16b st1 {v4.2d}, [x2] ret #ifndef __APPLE__ .size AES_GCM_encrypt_block_AARCH64,.-AES_GCM_encrypt_block_AARCH64 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_encrypt_update_AARCH64 .type AES_GCM_encrypt_update_AARCH64,@function .align 2 AES_GCM_encrypt_update_AARCH64: #else .section __TEXT,__text .globl _AES_GCM_encrypt_update_AARCH64 .p2align 2 _AES_GCM_encrypt_update_AARCH64: #endif /* __APPLE__ */ stp x29, x30, [sp, #-96]! add x29, sp, #0 str x17, [x29, #24] stp d8, d9, [x29, #32] stp d10, d11, [x29, #48] stp d12, d13, [x29, #64] stp d14, d15, [x29, #80] ld1 {v13.2d}, [x7] movi v27.16b, #0x87 ld1 {v26.2d}, [x5] ushr v27.2d, v27.2d, #56 ld1 {v22.2d}, [x6] mov w9, v13.s[3] rev w9, w9 cmp w4, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_h_done # Square H => H^2 pmull2 v31.1q, v22.2d, v22.2d pmull v30.1q, v22.1d, v22.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v23.16b, v30.16b, v31.16b cmp w4, #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_h_done # Multiply H and H^2 => H^3 pmull v28.1q, v22.1d, v23.1d pmull2 v29.1q, v22.2d, v23.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v23.1d pmull2 v31.1q, v31.2d, v23.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v24.16b, v28.16b, v30.16b # Square H^2 => H^4 pmull2 v31.1q, v23.2d, v23.2d pmull v30.1q, v23.1d, v23.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v25.16b, v30.16b, v31.16b # Done cmp w4, #0x200 blt L_aes_gcm_encrypt_update_arm64_crypto_h_done # Multiply H and H^4 => H^5 pmull v28.1q, v22.1d, v25.1d pmull2 v29.1q, v22.2d, v25.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v25.1d pmull2 v31.1q, v31.2d, v25.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v4.16b, v28.16b, v30.16b # Square H^3 => H^6 pmull2 v31.1q, v24.2d, v24.2d pmull v30.1q, v24.1d, v24.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v5.16b, v30.16b, v31.16b # Multiply H and H^6 => H^7 pmull v28.1q, v22.1d, v5.1d pmull2 v29.1q, v22.2d, v5.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v5.1d pmull2 v31.1q, v31.2d, v5.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v6.16b, v28.16b, v30.16b # Square H^4 => H^8 pmull2 v31.1q, v25.2d, v25.2d pmull v30.1q, v25.1d, v25.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v7.16b, v30.16b, v31.16b # Done L_aes_gcm_encrypt_update_arm64_crypto_h_done: lsr w8, w4, #4 cmp x1, #12 blt L_aes_gcm_encrypt_update_arm64_crypto_start_128 bgt L_aes_gcm_encrypt_update_arm64_crypto_start_256 # AES_GCM_192 #ifndef NO_AES_192 cmp w8, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_192_start_4 L_aes_gcm_encrypt_update_arm64_crypto_192_start_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rev w17, w17 rev w16, w16 rev w15, w15 rev w14, w14 rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 mov v14.s[3], w17 mov v15.s[3], w16 mov v16.s[3], w15 mov v17.s[3], w14 mov v8.s[3], w13 mov v9.s[3], w12 mov v10.s[3], w11 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v20.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v21.16b}, [x3], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v0.16b}, [x3], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v1.16b}, [x3], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v2.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v3.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b ld1 {v13.2d}, [x7] st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_encrypt_update_arm64_crypto_192_end_8 L_aes_gcm_encrypt_update_arm64_crypto_192_both_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w17, w17 rbit v19.16b, v19.16b rev w16, w16 rbit v20.16b, v20.16b rev w15, w15 rbit v21.16b, v21.16b rev w14, w14 rbit v0.16b, v0.16b rev w13, w13 rbit v1.16b, v1.16b rev w12, w12 rbit v2.16b, v2.16b rev w11, w11 rbit v3.16b, v3.16b rev w10, w9 mov v14.s[3], w17 mov v15.s[3], w16 mov v16.s[3], w15 mov v17.s[3], w14 mov v8.s[3], w13 mov v9.s[3], w12 mov v10.s[3], w11 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v31.16b, v3.16b, v3.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b ext v26.16b, v2.16b, v2.16b, #8 aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v26.16b, v26.16b, v31.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b ext v26.16b, v1.16b, v1.16b, #8 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b ext v26.16b, v0.16b, v0.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v31.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v30.16b, v30.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b ext v26.16b, v21.16b, v21.16b, #8 aese v9.16b, v13.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v26.16b, v26.16b, v31.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v8.16b, v12.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v26.16b, v26.16b, v31.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v30.16b, v30.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ext v26.16b, v19.16b, v19.16b, #8 aese v17.16b, v13.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v26.16b, v26.16b, v31.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v30.16b, v30.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull2 v29.1q, v29.2d, v27.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v31.16b, v31.16b, v30.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull2 v30.1q, v31.2d, v27.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b mov v28.d[1], v31.d[0] aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v18.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v19.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x3], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x3], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x7] eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_encrypt_update_arm64_crypto_192_both_8 L_aes_gcm_encrypt_update_arm64_crypto_192_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_192_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 ld1 {v12.2d}, [x0] cmp w8, #1 blt L_aes_gcm_encrypt_update_arm64_crypto_192_done beq L_aes_gcm_encrypt_update_arm64_crypto_192_start_1 cmp w8, #4 blt L_aes_gcm_encrypt_update_arm64_crypto_192_start_2 add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w12 mov v16.s[3], w11 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b ld1 {v19.16b}, [x3], #16 aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x3], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x3], #16 aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w8, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_192_end_4 L_aes_gcm_encrypt_update_arm64_crypto_192_both_4: add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w13, w13 rev w12, w12 rev w11, w11 rbit v19.16b, v19.16b rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w12 mov v16.s[3], w11 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b rbit v20.16b, v20.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b rbit v21.16b, v21.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b eor v18.16b, v18.16b, v26.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b ext v31.16b, v21.16b, v21.16b, #8 aese v15.16b, v1.16b aesmc v15.16b, v15.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v31.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b ext v26.16b, v20.16b, v20.16b, #8 aese v16.16b, v2.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v17.16b, v2.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v16.16b, v3.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b ext v26.16b, v19.16b, v19.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v4.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v5.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v17.16b, v5.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v29.1q, v29.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 eor v31.16b, v31.16b, v29.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v7.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v7.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w8, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 bge L_aes_gcm_encrypt_update_arm64_crypto_192_both_4 L_aes_gcm_encrypt_update_arm64_crypto_192_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w8, #1 beq L_aes_gcm_encrypt_update_arm64_crypto_192_start_1 blt L_aes_gcm_encrypt_update_arm64_crypto_192_done L_aes_gcm_encrypt_update_arm64_crypto_192_start_2: add w13, w9, #1 mov v14.16b, v13.16b add w9, w9, #2 mov v15.16b, v13.16b rev w13, w13 rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w8, w8, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x3], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b st1 {v18.16b, v19.16b}, [x2], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w8, L_aes_gcm_encrypt_update_arm64_crypto_192_done L_aes_gcm_encrypt_update_arm64_crypto_192_start_1: add w9, w9, #1 mov v14.16b, v13.16b rev w10, w9 mov v14.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b eor v18.16b, v18.16b, v14.16b st1 {v18.16b}, [x2], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_192_done: #endif /* !NO_AES_192 */ b L_aes_gcm_encrypt_update_arm64_crypto_done # AES_GCM_256 L_aes_gcm_encrypt_update_arm64_crypto_start_256: #ifndef NO_AES_256 cmp w8, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_256_start_4 L_aes_gcm_encrypt_update_arm64_crypto_256_start_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rev w17, w17 rev w16, w16 rev w15, w15 rev w14, w14 rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 mov v14.s[3], w17 mov v15.s[3], w16 mov v16.s[3], w15 mov v17.s[3], w14 mov v8.s[3], w13 mov v9.s[3], w12 mov v10.s[3], w11 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v20.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v21.16b}, [x3], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v0.16b}, [x3], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v1.16b}, [x3], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v2.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v3.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b ld1 {v13.2d}, [x7] st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_encrypt_update_arm64_crypto_256_end_8 L_aes_gcm_encrypt_update_arm64_crypto_256_both_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w17, w17 rbit v19.16b, v19.16b rev w16, w16 rbit v20.16b, v20.16b rev w15, w15 rbit v21.16b, v21.16b rev w14, w14 rbit v0.16b, v0.16b rev w13, w13 rbit v1.16b, v1.16b rev w12, w12 rbit v2.16b, v2.16b rev w11, w11 rbit v3.16b, v3.16b rev w10, w9 mov v14.s[3], w17 mov v15.s[3], w16 mov v16.s[3], w15 mov v17.s[3], w14 mov v8.s[3], w13 mov v9.s[3], w12 mov v10.s[3], w11 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v31.16b, v3.16b, v3.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b ext v26.16b, v2.16b, v2.16b, #8 aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v26.16b, v26.16b, v31.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b ext v26.16b, v1.16b, v1.16b, #8 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b ext v26.16b, v0.16b, v0.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v31.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v30.16b, v30.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b ext v26.16b, v21.16b, v21.16b, #8 aese v9.16b, v13.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v26.16b, v26.16b, v31.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v8.16b, v12.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v26.16b, v26.16b, v31.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v30.16b, v30.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ext v26.16b, v19.16b, v19.16b, #8 aese v17.16b, v13.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v26.16b, v26.16b, v31.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v30.16b, v30.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull2 v29.1q, v29.2d, v27.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v31.16b, v31.16b, v30.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull2 v30.1q, v31.2d, v27.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b mov v28.d[1], v31.d[0] aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v18.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v19.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x3], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x3], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x7] eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_encrypt_update_arm64_crypto_256_both_8 L_aes_gcm_encrypt_update_arm64_crypto_256_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_256_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 ld1 {v12.2d}, [x0], #16 cmp w8, #1 blt L_aes_gcm_encrypt_update_arm64_crypto_256_done beq L_aes_gcm_encrypt_update_arm64_crypto_256_start_1 cmp w8, #4 blt L_aes_gcm_encrypt_update_arm64_crypto_256_start_2 add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w12 mov v16.s[3], w11 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b ld1 {v19.16b}, [x3], #16 aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x3], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x3], #16 aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w8, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_256_end_4 L_aes_gcm_encrypt_update_arm64_crypto_256_both_4: add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w13, w13 rev w12, w12 rev w11, w11 rbit v19.16b, v19.16b rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w12 mov v16.s[3], w11 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b rbit v20.16b, v20.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b rbit v21.16b, v21.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b eor v18.16b, v18.16b, v26.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b ext v31.16b, v21.16b, v21.16b, #8 aese v15.16b, v1.16b aesmc v15.16b, v15.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v31.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b ext v26.16b, v20.16b, v20.16b, #8 aese v16.16b, v2.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v17.16b, v2.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v16.16b, v3.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b ext v26.16b, v19.16b, v19.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v4.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v5.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v17.16b, v5.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v29.1q, v29.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 eor v31.16b, v31.16b, v29.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v7.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v7.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w8, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 bge L_aes_gcm_encrypt_update_arm64_crypto_256_both_4 L_aes_gcm_encrypt_update_arm64_crypto_256_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w8, #1 beq L_aes_gcm_encrypt_update_arm64_crypto_256_start_1 blt L_aes_gcm_encrypt_update_arm64_crypto_256_done L_aes_gcm_encrypt_update_arm64_crypto_256_start_2: add w13, w9, #1 mov v14.16b, v13.16b add w9, w9, #2 mov v15.16b, v13.16b rev w13, w13 rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w8, w8, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x3], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b ld1 {v29.2d, v30.2d}, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b st1 {v18.16b, v19.16b}, [x2], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w8, L_aes_gcm_encrypt_update_arm64_crypto_256_done L_aes_gcm_encrypt_update_arm64_crypto_256_start_1: add w9, w9, #1 mov v14.16b, v13.16b rev w10, w9 mov v14.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b ldr q29, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ldr q30, [x0, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v18.16b, v18.16b, v14.16b st1 {v18.16b}, [x2], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_256_done: #endif /* !NO_AES_256 */ b L_aes_gcm_encrypt_update_arm64_crypto_done # AES_GCM_128 L_aes_gcm_encrypt_update_arm64_crypto_start_128: #ifndef NO_AES_128 cmp w8, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_128_start_4 L_aes_gcm_encrypt_update_arm64_crypto_128_start_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rev w17, w17 rev w16, w16 rev w15, w15 rev w14, w14 rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 mov v14.s[3], w17 mov v15.s[3], w16 mov v16.s[3], w15 mov v17.s[3], w14 mov v8.s[3], w13 mov v9.s[3], w12 mov v10.s[3], w11 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v20.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v21.16b}, [x3], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b ld1 {v0.16b}, [x3], #16 aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b ld1 {v1.16b}, [x3], #16 aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b ld1 {v2.16b}, [x3], #16 aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b ld1 {v3.16b}, [x3], #16 aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b ld1 {v13.2d}, [x7] st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_encrypt_update_arm64_crypto_128_end_8 L_aes_gcm_encrypt_update_arm64_crypto_128_both_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w17, w17 rbit v19.16b, v19.16b rev w16, w16 rbit v20.16b, v20.16b rev w15, w15 rbit v21.16b, v21.16b rev w14, w14 rbit v0.16b, v0.16b rev w13, w13 rbit v1.16b, v1.16b rev w12, w12 rbit v2.16b, v2.16b rev w11, w11 rbit v3.16b, v3.16b rev w10, w9 mov v14.s[3], w17 mov v15.s[3], w16 mov v16.s[3], w15 mov v17.s[3], w14 mov v8.s[3], w13 mov v9.s[3], w12 mov v10.s[3], w11 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v31.16b, v3.16b, v3.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b ext v26.16b, v2.16b, v2.16b, #8 aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v26.16b, v26.16b, v31.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b ext v26.16b, v1.16b, v1.16b, #8 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b ext v26.16b, v0.16b, v0.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v31.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v30.16b, v30.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b ext v26.16b, v21.16b, v21.16b, #8 aese v9.16b, v13.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v26.16b, v26.16b, v31.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v8.16b, v12.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v26.16b, v26.16b, v31.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v30.16b, v30.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ext v26.16b, v19.16b, v19.16b, #8 aese v17.16b, v13.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v26.16b, v26.16b, v31.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v30.16b, v30.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull2 v29.1q, v29.2d, v27.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v31.16b, v31.16b, v30.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull2 v30.1q, v31.2d, v27.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b mov v28.d[1], v31.d[0] aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v18.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v19.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x3], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x3], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x7] eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_encrypt_update_arm64_crypto_128_both_8 L_aes_gcm_encrypt_update_arm64_crypto_128_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_128_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 ld1 {v8.2d, v9.2d}, [x0], #32 ld1 {v10.2d}, [x0] cmp w8, #1 blt L_aes_gcm_encrypt_update_arm64_crypto_128_done beq L_aes_gcm_encrypt_update_arm64_crypto_128_start_1 cmp w8, #4 blt L_aes_gcm_encrypt_update_arm64_crypto_128_start_2 add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w12 mov v16.s[3], w11 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b ld1 {v19.16b}, [x3], #16 aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v20.16b}, [x3], #16 aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b ld1 {v21.16b}, [x3], #16 aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w8, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_128_end_4 L_aes_gcm_encrypt_update_arm64_crypto_128_both_4: add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w13, w13 rev w12, w12 rev w11, w11 rbit v19.16b, v19.16b rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w12 mov v16.s[3], w11 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b rbit v20.16b, v20.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b rbit v21.16b, v21.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b eor v18.16b, v18.16b, v26.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b ext v31.16b, v21.16b, v21.16b, #8 aese v15.16b, v1.16b aesmc v15.16b, v15.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v31.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b ext v26.16b, v20.16b, v20.16b, #8 aese v16.16b, v2.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v17.16b, v2.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v16.16b, v3.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b ext v26.16b, v19.16b, v19.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v4.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v5.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v17.16b, v5.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v29.1q, v29.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 eor v31.16b, v31.16b, v29.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v7.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v7.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w8, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 bge L_aes_gcm_encrypt_update_arm64_crypto_128_both_4 L_aes_gcm_encrypt_update_arm64_crypto_128_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w8, #1 beq L_aes_gcm_encrypt_update_arm64_crypto_128_start_1 blt L_aes_gcm_encrypt_update_arm64_crypto_128_done L_aes_gcm_encrypt_update_arm64_crypto_128_start_2: add w13, w9, #1 mov v14.16b, v13.16b add w9, w9, #2 mov v15.16b, v13.16b rev w13, w13 rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w8, w8, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v19.16b}, [x3], #16 aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b st1 {v18.16b, v19.16b}, [x2], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w8, L_aes_gcm_encrypt_update_arm64_crypto_128_done L_aes_gcm_encrypt_update_arm64_crypto_128_start_1: add w9, w9, #1 mov v14.16b, v13.16b rev w10, w9 mov v14.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v18.16b}, [x3], #16 eor v18.16b, v18.16b, v14.16b st1 {v18.16b}, [x2], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_128_done: #endif /* !NO_AES_128 */ L_aes_gcm_encrypt_update_arm64_crypto_done: rev w9, w9 mov v13.s[3], w9 st1 {v26.2d}, [x5] st1 {v13.2d}, [x7] ldr x17, [x29, #24] ldp d8, d9, [x29, #32] ldp d10, d11, [x29, #48] ldp d12, d13, [x29, #64] ldp d14, d15, [x29, #80] ldp x29, x30, [sp], #0x60 ret #ifndef __APPLE__ .size AES_GCM_encrypt_update_AARCH64,.-AES_GCM_encrypt_update_AARCH64 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_encrypt_final_AARCH64 .type AES_GCM_encrypt_final_AARCH64,@function .align 2 AES_GCM_encrypt_final_AARCH64: #else .section __TEXT,__text .globl _AES_GCM_encrypt_final_AARCH64 .p2align 2 _AES_GCM_encrypt_final_AARCH64: #endif /* __APPLE__ */ ld1 {v5.2d}, [x0] movi v6.16b, #0x87 ld1 {v4.2d}, [x5] ushr v6.2d, v6.2d, #56 ld1 {v7.2d}, [x6] lsl x4, x4, #3 rbit x4, x4 mov v0.d[0], x4 lsl x3, x3, #3 rbit x3, x3 mov v0.d[1], x3 eor v5.16b, v5.16b, v0.16b pmull v0.1q, v5.1d, v4.1d pmull2 v1.1q, v5.2d, v4.2d ext v3.16b, v5.16b, v5.16b, #8 pmull v2.1q, v3.1d, v4.1d pmull2 v3.1q, v3.2d, v4.2d eor v2.16b, v2.16b, v3.16b # Reduce ext v3.16b, v0.16b, v1.16b, #8 pmull2 v1.1q, v1.2d, v6.2d eor v3.16b, v3.16b, v1.16b eor v3.16b, v3.16b, v2.16b pmull2 v2.1q, v3.2d, v6.2d mov v0.d[1], v3.d[0] eor v5.16b, v0.16b, v2.16b rbit v5.16b, v5.16b eor v5.16b, v5.16b, v7.16b cmp w2, #16 bne L_aes_gcm_encrypt_final_arm64_crypto_tag_partial st1 {v5.16b}, [x1] b L_aes_gcm_encrypt_final_arm64_crypto_done L_aes_gcm_encrypt_final_arm64_crypto_tag_partial: st1 {v5.16b}, [x0] cmp w2, #8 blt L_aes_gcm_encrypt_final_arm64_crypto_tag_start_dw ldr x8, [x0], #8 sub w2, w2, #8 str x8, [x1], #8 L_aes_gcm_encrypt_final_arm64_crypto_tag_start_dw: cmp w2, #4 blt L_aes_gcm_encrypt_final_arm64_crypto_tag_start_sw ldr w8, [x0], #4 sub w2, w2, #4 str w8, [x1], #4 L_aes_gcm_encrypt_final_arm64_crypto_tag_start_sw: cmp w2, #2 blt L_aes_gcm_encrypt_final_arm64_crypto_tag_start_byte ldrh w8, [x0], #2 sub w2, w2, #2 strh w8, [x1], #2 L_aes_gcm_encrypt_final_arm64_crypto_tag_start_byte: cbz w2, L_aes_gcm_encrypt_final_arm64_crypto_tag_end_bytes ldrb w8, [x0], #1 subs w2, w2, #1 strb w8, [x1], #1 bne L_aes_gcm_encrypt_final_arm64_crypto_tag_start_byte L_aes_gcm_encrypt_final_arm64_crypto_tag_end_bytes: L_aes_gcm_encrypt_final_arm64_crypto_done: ret #ifndef __APPLE__ .size AES_GCM_encrypt_final_AARCH64,.-AES_GCM_encrypt_final_AARCH64 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_decrypt_update_AARCH64 .type AES_GCM_decrypt_update_AARCH64,@function .align 2 AES_GCM_decrypt_update_AARCH64: #else .section __TEXT,__text .globl _AES_GCM_decrypt_update_AARCH64 .p2align 2 _AES_GCM_decrypt_update_AARCH64: #endif /* __APPLE__ */ stp x29, x30, [sp, #-96]! add x29, sp, #0 str x17, [x29, #24] stp d8, d9, [x29, #32] stp d10, d11, [x29, #48] stp d12, d13, [x29, #64] stp d14, d15, [x29, #80] ld1 {v13.2d}, [x7] movi v27.16b, #0x87 ld1 {v26.2d}, [x5] ushr v27.2d, v27.2d, #56 ld1 {v22.2d}, [x6] mov w9, v13.s[3] rev w9, w9 cmp w4, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_h_done # Square H => H^2 pmull2 v31.1q, v22.2d, v22.2d pmull v30.1q, v22.1d, v22.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v23.16b, v30.16b, v31.16b cmp w4, #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_h_done # Multiply H and H^2 => H^3 pmull v28.1q, v22.1d, v23.1d pmull2 v29.1q, v22.2d, v23.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v23.1d pmull2 v31.1q, v31.2d, v23.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v24.16b, v28.16b, v30.16b # Square H^2 => H^4 pmull2 v31.1q, v23.2d, v23.2d pmull v30.1q, v23.1d, v23.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v25.16b, v30.16b, v31.16b # Done cmp w4, #0x200 blt L_aes_gcm_decrypt_update_arm64_crypto_h_done # Multiply H and H^4 => H^5 pmull v28.1q, v22.1d, v25.1d pmull2 v29.1q, v22.2d, v25.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v25.1d pmull2 v31.1q, v31.2d, v25.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v4.16b, v28.16b, v30.16b # Square H^3 => H^6 pmull2 v31.1q, v24.2d, v24.2d pmull v30.1q, v24.1d, v24.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v5.16b, v30.16b, v31.16b # Multiply H and H^6 => H^7 pmull v28.1q, v22.1d, v5.1d pmull2 v29.1q, v22.2d, v5.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v5.1d pmull2 v31.1q, v31.2d, v5.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v6.16b, v28.16b, v30.16b # Square H^4 => H^8 pmull2 v31.1q, v25.2d, v25.2d pmull v30.1q, v25.1d, v25.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v7.16b, v30.16b, v31.16b # Done L_aes_gcm_decrypt_update_arm64_crypto_h_done: lsr w8, w4, #4 cmp x1, #12 blt L_aes_gcm_decrypt_update_arm64_crypto_start_128 bgt L_aes_gcm_decrypt_update_arm64_crypto_start_256 # AES_GCM_192 #ifndef NO_AES_192 cmp w8, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_192_start_4 L_aes_gcm_decrypt_update_arm64_crypto_192_start_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rev w17, w17 mov v14.s[3], w17 rev w16, w16 mov v15.s[3], w16 rev w15, w15 mov v16.s[3], w15 rev w14, w14 mov v17.s[3], w14 rev w13, w13 mov v8.s[3], w13 rev w12, w12 mov v9.s[3], w12 rev w11, w11 mov v10.s[3], w11 rev w10, w9 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v18.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v19.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v20.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v21.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x3], #16 aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x3], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x3], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b ld1 {v13.2d}, [x7] st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_decrypt_update_arm64_crypto_192_end_8 L_aes_gcm_decrypt_update_arm64_crypto_192_both_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w17, w17 rbit v19.16b, v19.16b mov v14.s[3], w17 rev w16, w16 rbit v20.16b, v20.16b mov v15.s[3], w16 rev w15, w15 rbit v21.16b, v21.16b mov v16.s[3], w15 rev w14, w14 rbit v0.16b, v0.16b mov v17.s[3], w14 rev w13, w13 rbit v1.16b, v1.16b mov v8.s[3], w13 rev w12, w12 rbit v2.16b, v2.16b mov v9.s[3], w12 rev w11, w11 rbit v3.16b, v3.16b mov v10.s[3], w11 rev w10, w9 eor v18.16b, v18.16b, v26.16b mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v31.16b, v3.16b, v3.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b ext v26.16b, v2.16b, v2.16b, #8 aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b ext v26.16b, v1.16b, v1.16b, #8 aese v10.16b, v13.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v31.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v30.16b, v30.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b ext v26.16b, v0.16b, v0.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v26.16b, v26.16b, v31.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ext v26.16b, v21.16b, v21.16b, #8 aese v8.16b, v13.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v26.16b, v26.16b, v31.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v30.16b, v30.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v26.16b, v20.16b, v20.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v26.16b, v26.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v30.16b, v30.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ext v26.16b, v18.16b, v18.16b, #8 aese v15.16b, v12.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v26.16b, v26.16b, v31.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull2 v29.1q, v29.2d, v27.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v31.16b, v31.16b, v29.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v18.16b}, [x3], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v19.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v20.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v21.16b}, [x3], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v0.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v1.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v2.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v3.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x7] eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_decrypt_update_arm64_crypto_192_both_8 L_aes_gcm_decrypt_update_arm64_crypto_192_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_192_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 ld1 {v12.2d}, [x0] cmp w8, #1 blt L_aes_gcm_decrypt_update_arm64_crypto_192_done beq L_aes_gcm_decrypt_update_arm64_crypto_192_start_1 cmp w8, #4 blt L_aes_gcm_decrypt_update_arm64_crypto_192_start_2 add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rev w13, w13 mov v14.s[3], w13 rev w12, w12 mov v15.s[3], w12 rev w11, w11 mov v16.s[3], w11 rev w10, w9 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x3], #16 aese v17.16b, v7.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x3], #16 aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v20.16b}, [x3], #16 aese v17.16b, v8.16b aesmc v17.16b, v17.16b ld1 {v21.16b}, [x3], #16 aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w8, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_192_end_4 L_aes_gcm_decrypt_update_arm64_crypto_192_both_4: add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w13, w13 rbit v19.16b, v19.16b mov v14.s[3], w13 rev w12, w12 rbit v20.16b, v20.16b mov v15.s[3], w12 rev w11, w11 rbit v21.16b, v21.16b mov v16.s[3], w11 rev w10, w9 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v16.16b, v0.16b aesmc v16.16b, v16.16b ext v31.16b, v21.16b, v21.16b, #8 aese v17.16b, v0.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v2.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v3.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v17.16b, v3.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b ext v26.16b, v18.16b, v18.16b, #8 aese v14.16b, v5.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v15.16b, v5.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v15.16b, v6.16b aesmc v15.16b, v15.16b eor v31.16b, v31.16b, v29.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b eor v31.16b, v31.16b, v30.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v15.16b, v7.16b aesmc v15.16b, v15.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w8, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 bge L_aes_gcm_decrypt_update_arm64_crypto_192_both_4 L_aes_gcm_decrypt_update_arm64_crypto_192_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w8, #1 beq L_aes_gcm_decrypt_update_arm64_crypto_192_start_1 blt L_aes_gcm_decrypt_update_arm64_crypto_192_done L_aes_gcm_decrypt_update_arm64_crypto_192_start_2: add w13, w9, #1 mov v14.16b, v13.16b add w9, w9, #2 mov v15.16b, v13.16b rev w13, w13 mov v14.s[3], w13 rev w10, w9 mov v15.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w8, w8, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x3], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b st1 {v14.16b, v15.16b}, [x2], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w8, L_aes_gcm_decrypt_update_arm64_crypto_192_done L_aes_gcm_decrypt_update_arm64_crypto_192_start_1: ld1 {v15.16b}, [x3], #16 add w9, w9, #1 mov v14.16b, v13.16b rbit v15.16b, v15.16b rev w10, w9 mov v14.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v16.16b, v26.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v16.1d, v22.1d pmull2 v29.1q, v16.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b ext v31.16b, v16.16b, v16.16b, #8 aese v14.16b, v3.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v29.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v10.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b eor v26.16b, v28.16b, v30.16b # Done GHASH rbit v15.16b, v15.16b eor v14.16b, v14.16b, v15.16b st1 {v14.16b}, [x2], #16 L_aes_gcm_decrypt_update_arm64_crypto_192_done: #endif /* !NO_AES_192 */ b L_aes_gcm_decrypt_update_arm64_crypto_done # AES_GCM_256 L_aes_gcm_decrypt_update_arm64_crypto_start_256: #ifndef NO_AES_256 cmp w8, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_256_start_4 L_aes_gcm_decrypt_update_arm64_crypto_256_start_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rev w17, w17 mov v14.s[3], w17 rev w16, w16 mov v15.s[3], w16 rev w15, w15 mov v16.s[3], w15 rev w14, w14 mov v17.s[3], w14 rev w13, w13 mov v8.s[3], w13 rev w12, w12 mov v9.s[3], w12 rev w11, w11 mov v10.s[3], w11 rev w10, w9 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v18.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v19.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v20.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v21.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x3], #16 aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x3], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x3], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b ld1 {v13.2d}, [x7] st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_decrypt_update_arm64_crypto_256_end_8 L_aes_gcm_decrypt_update_arm64_crypto_256_both_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w17, w17 rbit v19.16b, v19.16b mov v14.s[3], w17 rev w16, w16 rbit v20.16b, v20.16b mov v15.s[3], w16 rev w15, w15 rbit v21.16b, v21.16b mov v16.s[3], w15 rev w14, w14 rbit v0.16b, v0.16b mov v17.s[3], w14 rev w13, w13 rbit v1.16b, v1.16b mov v8.s[3], w13 rev w12, w12 rbit v2.16b, v2.16b mov v9.s[3], w12 rev w11, w11 rbit v3.16b, v3.16b mov v10.s[3], w11 rev w10, w9 eor v18.16b, v18.16b, v26.16b mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v31.16b, v3.16b, v3.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b ext v26.16b, v2.16b, v2.16b, #8 aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b ext v26.16b, v1.16b, v1.16b, #8 aese v10.16b, v13.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v31.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v30.16b, v30.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b ext v26.16b, v0.16b, v0.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v26.16b, v26.16b, v31.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ext v26.16b, v21.16b, v21.16b, #8 aese v8.16b, v13.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v26.16b, v26.16b, v31.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v30.16b, v30.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v26.16b, v20.16b, v20.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v26.16b, v26.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v30.16b, v30.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ext v26.16b, v18.16b, v18.16b, #8 aese v15.16b, v12.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v26.16b, v26.16b, v31.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull2 v29.1q, v29.2d, v27.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v31.16b, v31.16b, v29.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v18.16b}, [x3], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v19.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v20.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v21.16b}, [x3], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v0.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v1.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v2.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v3.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x7] eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_decrypt_update_arm64_crypto_256_both_8 L_aes_gcm_decrypt_update_arm64_crypto_256_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_256_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 ld1 {v12.2d}, [x0], #16 cmp w8, #1 blt L_aes_gcm_decrypt_update_arm64_crypto_256_done beq L_aes_gcm_decrypt_update_arm64_crypto_256_start_1 cmp w8, #4 blt L_aes_gcm_decrypt_update_arm64_crypto_256_start_2 add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rev w13, w13 mov v14.s[3], w13 rev w12, w12 mov v15.s[3], w12 rev w11, w11 mov v16.s[3], w11 rev w10, w9 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x3], #16 aese v17.16b, v7.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x3], #16 aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v20.16b}, [x3], #16 aese v17.16b, v8.16b aesmc v17.16b, v17.16b ld1 {v21.16b}, [x3], #16 aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w8, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_256_end_4 L_aes_gcm_decrypt_update_arm64_crypto_256_both_4: add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w13, w13 rbit v19.16b, v19.16b mov v14.s[3], w13 rev w12, w12 rbit v20.16b, v20.16b mov v15.s[3], w12 rev w11, w11 rbit v21.16b, v21.16b mov v16.s[3], w11 rev w10, w9 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v16.16b, v0.16b aesmc v16.16b, v16.16b ext v31.16b, v21.16b, v21.16b, #8 aese v17.16b, v0.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v2.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v3.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v17.16b, v3.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b ext v26.16b, v18.16b, v18.16b, #8 aese v14.16b, v5.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v15.16b, v5.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v15.16b, v6.16b aesmc v15.16b, v15.16b eor v31.16b, v31.16b, v29.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b eor v31.16b, v31.16b, v30.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v15.16b, v7.16b aesmc v15.16b, v15.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w8, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 bge L_aes_gcm_decrypt_update_arm64_crypto_256_both_4 L_aes_gcm_decrypt_update_arm64_crypto_256_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w8, #1 beq L_aes_gcm_decrypt_update_arm64_crypto_256_start_1 blt L_aes_gcm_decrypt_update_arm64_crypto_256_done L_aes_gcm_decrypt_update_arm64_crypto_256_start_2: add w13, w9, #1 mov v14.16b, v13.16b add w9, w9, #2 mov v15.16b, v13.16b rev w13, w13 mov v14.s[3], w13 rev w10, w9 mov v15.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w8, w8, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x3], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b ld1 {v29.2d, v30.2d}, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b st1 {v14.16b, v15.16b}, [x2], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w8, L_aes_gcm_decrypt_update_arm64_crypto_256_done L_aes_gcm_decrypt_update_arm64_crypto_256_start_1: add w9, w9, #1 mov v14.16b, v13.16b rev w10, w9 mov v14.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v14.16b, v11.16b aesmc v14.16b, v14.16b ldr q29, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ldr q30, [x0, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v14.16b, v14.16b, v18.16b st1 {v14.16b}, [x2], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_256_done: #endif /* !NO_AES_256 */ b L_aes_gcm_decrypt_update_arm64_crypto_done # AES_GCM_128 L_aes_gcm_decrypt_update_arm64_crypto_start_128: #ifndef NO_AES_128 cmp w8, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_128_start_4 L_aes_gcm_decrypt_update_arm64_crypto_128_start_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rev w17, w17 mov v14.s[3], w17 rev w16, w16 mov v15.s[3], w16 rev w15, w15 mov v16.s[3], w15 rev w14, w14 mov v17.s[3], w14 rev w13, w13 mov v8.s[3], w13 rev w12, w12 mov v9.s[3], w12 rev w11, w11 mov v10.s[3], w11 rev w10, w9 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v18.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v19.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v20.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v21.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b ld1 {v0.16b}, [x3], #16 aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b ld1 {v1.16b}, [x3], #16 aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b ld1 {v2.16b}, [x3], #16 aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b ld1 {v3.16b}, [x3], #16 aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b ld1 {v13.2d}, [x7] st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_decrypt_update_arm64_crypto_128_end_8 L_aes_gcm_decrypt_update_arm64_crypto_128_both_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w17, w17 rbit v19.16b, v19.16b mov v14.s[3], w17 rev w16, w16 rbit v20.16b, v20.16b mov v15.s[3], w16 rev w15, w15 rbit v21.16b, v21.16b mov v16.s[3], w15 rev w14, w14 rbit v0.16b, v0.16b mov v17.s[3], w14 rev w13, w13 rbit v1.16b, v1.16b mov v8.s[3], w13 rev w12, w12 rbit v2.16b, v2.16b mov v9.s[3], w12 rev w11, w11 rbit v3.16b, v3.16b mov v10.s[3], w11 rev w10, w9 eor v18.16b, v18.16b, v26.16b mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v31.16b, v3.16b, v3.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b ext v26.16b, v2.16b, v2.16b, #8 aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b ext v26.16b, v1.16b, v1.16b, #8 aese v10.16b, v13.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v26.16b, v26.16b, v31.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v30.16b, v30.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b ext v26.16b, v0.16b, v0.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v26.16b, v26.16b, v31.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ext v26.16b, v21.16b, v21.16b, #8 aese v8.16b, v13.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v26.16b, v26.16b, v31.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v30.16b, v30.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v26.16b, v20.16b, v20.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v26.16b, v26.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v30.16b, v30.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ext v26.16b, v18.16b, v18.16b, #8 aese v15.16b, v12.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v26.16b, v26.16b, v31.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull2 v29.1q, v29.2d, v27.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v31.16b, v31.16b, v29.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v31.16b, v31.16b, v30.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v18.16b}, [x3], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v19.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v20.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v21.16b}, [x3], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v0.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v1.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v2.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v3.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x7] eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_decrypt_update_arm64_crypto_128_both_8 L_aes_gcm_decrypt_update_arm64_crypto_128_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_128_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 ld1 {v8.2d, v9.2d}, [x0], #32 ld1 {v10.2d}, [x0] cmp w8, #1 blt L_aes_gcm_decrypt_update_arm64_crypto_128_done beq L_aes_gcm_decrypt_update_arm64_crypto_128_start_1 cmp w8, #4 blt L_aes_gcm_decrypt_update_arm64_crypto_128_start_2 add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rev w13, w13 mov v14.s[3], w13 rev w12, w12 mov v15.s[3], w12 rev w11, w11 mov v16.s[3], w11 rev w10, w9 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x3], #16 aese v17.16b, v7.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x3], #16 aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v20.16b}, [x3], #16 aese v17.16b, v8.16b aesmc v17.16b, v17.16b ld1 {v21.16b}, [x3], #16 aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w8, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_128_end_4 L_aes_gcm_decrypt_update_arm64_crypto_128_both_4: add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w13, w13 rbit v19.16b, v19.16b mov v14.s[3], w13 rev w12, w12 rbit v20.16b, v20.16b mov v15.s[3], w12 rev w11, w11 rbit v21.16b, v21.16b mov v16.s[3], w11 rev w10, w9 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v16.16b, v0.16b aesmc v16.16b, v16.16b ext v31.16b, v21.16b, v21.16b, #8 aese v17.16b, v0.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v2.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v3.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v17.16b, v3.16b aesmc v17.16b, v17.16b eor v26.16b, v26.16b, v31.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v26.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b ext v26.16b, v18.16b, v18.16b, #8 aese v14.16b, v5.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v15.16b, v5.16b aesmc v15.16b, v15.16b eor v26.16b, v26.16b, v31.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v26.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v15.16b, v6.16b aesmc v15.16b, v15.16b eor v31.16b, v31.16b, v29.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b eor v31.16b, v31.16b, v30.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v15.16b, v7.16b aesmc v15.16b, v15.16b eor v26.16b, v28.16b, v30.16b # Done GHASH aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w8, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 bge L_aes_gcm_decrypt_update_arm64_crypto_128_both_4 L_aes_gcm_decrypt_update_arm64_crypto_128_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w8, #1 beq L_aes_gcm_decrypt_update_arm64_crypto_128_start_1 blt L_aes_gcm_decrypt_update_arm64_crypto_128_done L_aes_gcm_decrypt_update_arm64_crypto_128_start_2: add w13, w9, #1 mov v14.16b, v13.16b add w9, w9, #2 mov v15.16b, v13.16b rev w13, w13 mov v14.s[3], w13 rev w10, w9 mov v15.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w8, w8, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v19.16b}, [x3], #16 aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b st1 {v14.16b, v15.16b}, [x2], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor v26.16b, v26.16b, v31.16b eor v30.16b, v30.16b, v26.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w8, L_aes_gcm_decrypt_update_arm64_crypto_128_done L_aes_gcm_decrypt_update_arm64_crypto_128_start_1: add w9, w9, #1 mov v14.16b, v13.16b rev w10, w9 mov v14.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v18.16b}, [x3], #16 eor v14.16b, v14.16b, v18.16b st1 {v14.16b}, [x2], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor v31.16b, v31.16b, v29.16b eor v31.16b, v31.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_128_done: #endif /* !NO_AES_128 */ L_aes_gcm_decrypt_update_arm64_crypto_done: rev w9, w9 mov v13.s[3], w9 st1 {v26.2d}, [x5] st1 {v13.2d}, [x7] ldr x17, [x29, #24] ldp d8, d9, [x29, #32] ldp d10, d11, [x29, #48] ldp d12, d13, [x29, #64] ldp d14, d15, [x29, #80] ldp x29, x30, [sp], #0x60 ret #ifndef __APPLE__ .size AES_GCM_decrypt_update_AARCH64,.-AES_GCM_decrypt_update_AARCH64 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_decrypt_final_AARCH64 .type AES_GCM_decrypt_final_AARCH64,@function .align 2 AES_GCM_decrypt_final_AARCH64: #else .section __TEXT,__text .globl _AES_GCM_decrypt_final_AARCH64 .p2align 2 _AES_GCM_decrypt_final_AARCH64: #endif /* __APPLE__ */ ld1 {v5.2d}, [x0] movi v6.16b, #0x87 ld1 {v4.2d}, [x5] ushr v6.2d, v6.2d, #56 ld1 {v7.2d}, [x6] lsl x4, x4, #3 rbit x4, x4 mov v0.d[0], x4 lsl x3, x3, #3 rbit x3, x3 mov v0.d[1], x3 eor v5.16b, v5.16b, v0.16b pmull v0.1q, v5.1d, v4.1d pmull2 v1.1q, v5.2d, v4.2d ext v3.16b, v5.16b, v5.16b, #8 pmull v2.1q, v3.1d, v4.1d pmull2 v3.1q, v3.2d, v4.2d eor v2.16b, v2.16b, v3.16b # Reduce ext v3.16b, v0.16b, v1.16b, #8 pmull2 v1.1q, v1.2d, v6.2d eor v3.16b, v3.16b, v1.16b eor v3.16b, v3.16b, v2.16b pmull2 v2.1q, v3.2d, v6.2d mov v0.d[1], v3.d[0] eor v5.16b, v0.16b, v2.16b rbit v5.16b, v5.16b eor v5.16b, v5.16b, v7.16b cmp w2, #16 blt L_aes_gcm_decrypt_final_arm64_crypto_part_tag ld1 {v0.16b}, [x1] b L_aes_gcm_decrypt_final_arm64_crypto_tag_loaded L_aes_gcm_decrypt_final_arm64_crypto_part_tag: eor v0.16b, v0.16b, v0.16b mov x10, x2 st1 {v0.2d}, [x0] cmp x10, #8 blt L_aes_gcm_decrypt_final_arm64_crypto_tag_start_dw ldr x9, [x1], #8 sub x10, x10, #8 str x9, [x0], #8 L_aes_gcm_decrypt_final_arm64_crypto_tag_start_dw: cmp x10, #4 blt L_aes_gcm_decrypt_final_arm64_crypto_tag_start_sw ldr w9, [x1], #4 sub x10, x10, #4 str w9, [x0], #4 L_aes_gcm_decrypt_final_arm64_crypto_tag_start_sw: cmp x10, #2 blt L_aes_gcm_decrypt_final_arm64_crypto_tag_start_byte ldrh w9, [x1], #2 sub x10, x10, #2 strh w9, [x0], #2 L_aes_gcm_decrypt_final_arm64_crypto_tag_start_byte: cbz x10, L_aes_gcm_decrypt_final_arm64_crypto_tag_end_bytes ldrb w9, [x1], #1 subs x10, x10, #1 strb w9, [x0], #1 bne L_aes_gcm_decrypt_final_arm64_crypto_tag_start_byte L_aes_gcm_decrypt_final_arm64_crypto_tag_end_bytes: sub x0, x0, x2 ld1 {v0.2d}, [x0] mov x10, #16 st1 {v5.2d}, [x0] sub x10, x10, x2 add x0, x0, x2 L_aes_gcm_decrypt_final_arm64_crypto_calc_tag_byte: strb wzr, [x0], #1 subs x10, x10, #1 bne L_aes_gcm_decrypt_final_arm64_crypto_calc_tag_byte subs x0, x0, #16 ld1 {v5.2d}, [x0] L_aes_gcm_decrypt_final_arm64_crypto_tag_loaded: eor v0.16b, v0.16b, v5.16b mov x9, v0.d[0] mov x10, v0.d[1] mov w11, #-180 orr x9, x9, x10 cmp x9, #0 csetm x8, ne and x8, x8, x11 add w8, w8, #0xb4 str w8, [x7] ret #ifndef __APPLE__ .size AES_GCM_decrypt_final_AARCH64,.-AES_GCM_decrypt_final_AARCH64 #endif /* __APPLE__ */ #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 #ifndef __APPLE__ .text .globl AES_GCM_init_AARCH64_EOR3 .type AES_GCM_init_AARCH64_EOR3,@function .align 2 AES_GCM_init_AARCH64_EOR3: #else .section __TEXT,__text .globl _AES_GCM_init_AARCH64_EOR3 .p2align 2 _AES_GCM_init_AARCH64_EOR3: #endif /* __APPLE__ */ stp x29, x30, [sp, #-48]! add x29, sp, #0 stp d8, d9, [x29, #16] stp d10, d11, [x29, #32] movi v6.16b, #0x87 ld1 {v5.2d}, [x4] ushr v6.2d, v6.2d, #56 # Load Nonce cmp w3, #12 bne L_aes_gcm_init_arm64_crypto_eor3_ghash_nonce ldr x9, [x2] movi v4.4s, #1, lsl 24 ldr w10, [x2, #8] mov v4.d[0], x9 mov v4.s[2], w10 mov w8, #1 b L_aes_gcm_init_arm64_crypto_eor3_done_nonce L_aes_gcm_init_arm64_crypto_eor3_ghash_nonce: eor v4.16b, v4.16b, v4.16b lsr w7, w3, #4 cbz w7, L_aes_gcm_init_arm64_crypto_eor3_done L_aes_gcm_init_arm64_crypto_eor3_start_1: ld1 {v0.16b}, [x2], #16 rbit v0.16b, v0.16b eor v3.16b, v4.16b, v0.16b # X = C * H^1 pmull v7.1q, v3.1d, v5.1d pmull2 v8.1q, v3.2d, v5.2d ext v10.16b, v3.16b, v3.16b, #8 pmull v9.1q, v10.1d, v5.1d pmull2 v10.1q, v10.2d, v5.2d eor v9.16b, v9.16b, v10.16b # Reduce ext v10.16b, v7.16b, v8.16b, #8 pmull2 v8.1q, v8.2d, v6.2d eor3 v10.16b, v10.16b, v8.16b, v9.16b pmull2 v9.1q, v10.2d, v6.2d mov v7.d[1], v10.d[0] eor v4.16b, v7.16b, v9.16b # Done GHASH subs w7, w7, #1 bne L_aes_gcm_init_arm64_crypto_eor3_start_1 L_aes_gcm_init_arm64_crypto_eor3_done: and w13, w3, #15 cbz x13, L_aes_gcm_init_arm64_crypto_eor3_partial_done eor v7.16b, v7.16b, v7.16b mov w12, w13 st1 {v7.2d}, [x6] cmp w12, #8 blt L_aes_gcm_init_arm64_crypto_eor3_start_dw ldr x11, [x2], #8 sub w12, w12, #8 str x11, [x6], #8 L_aes_gcm_init_arm64_crypto_eor3_start_dw: cmp w12, #4 blt L_aes_gcm_init_arm64_crypto_eor3_start_sw ldr w11, [x2], #4 sub w12, w12, #4 str w11, [x6], #4 L_aes_gcm_init_arm64_crypto_eor3_start_sw: cmp w12, #2 blt L_aes_gcm_init_arm64_crypto_eor3_start_byte ldrh w11, [x2], #2 sub w12, w12, #2 strh w11, [x6], #2 L_aes_gcm_init_arm64_crypto_eor3_start_byte: cbz w12, L_aes_gcm_init_arm64_crypto_eor3_end_bytes ldrb w11, [x2], #1 subs w12, w12, #1 strb w11, [x6], #1 bne L_aes_gcm_init_arm64_crypto_eor3_start_byte L_aes_gcm_init_arm64_crypto_eor3_end_bytes: sub x6, x6, x13 ld1 {v0.2d}, [x6] rbit v0.16b, v0.16b eor v3.16b, v4.16b, v0.16b # X = C * H^1 pmull v7.1q, v3.1d, v5.1d pmull2 v8.1q, v3.2d, v5.2d ext v10.16b, v3.16b, v3.16b, #8 pmull v9.1q, v10.1d, v5.1d pmull2 v10.1q, v10.2d, v5.2d eor v9.16b, v9.16b, v10.16b # Reduce ext v10.16b, v7.16b, v8.16b, #8 pmull2 v8.1q, v8.2d, v6.2d eor3 v10.16b, v10.16b, v8.16b, v9.16b pmull2 v9.1q, v10.2d, v6.2d mov v7.d[1], v10.d[0] eor v4.16b, v7.16b, v9.16b # Done GHASH L_aes_gcm_init_arm64_crypto_eor3_partial_done: eor x7, x7, x7 lsl x13, x3, #3 mov v7.d[0], x7 mov v7.d[1], x13 rev64 v7.16b, v7.16b rbit v7.16b, v7.16b eor v4.16b, v4.16b, v7.16b pmull v7.1q, v4.1d, v5.1d pmull2 v8.1q, v4.2d, v5.2d ext v10.16b, v4.16b, v4.16b, #8 pmull v9.1q, v10.1d, v5.1d pmull2 v10.1q, v10.2d, v5.2d eor v9.16b, v9.16b, v10.16b ext v10.16b, v7.16b, v8.16b, #8 pmull2 v8.1q, v8.2d, v6.2d eor3 v10.16b, v10.16b, v8.16b, v9.16b pmull2 v9.1q, v10.2d, v6.2d mov v7.d[1], v10.d[0] eor v4.16b, v7.16b, v9.16b rbit v4.16b, v4.16b mov w8, v4.s[3] rev w8, w8 L_aes_gcm_init_arm64_crypto_eor3_done_nonce: st1 {v4.2d}, [x5] ld1 {v7.2d, v8.2d, v9.2d, v10.2d}, [x0], #0x40 aese v4.16b, v7.16b aesmc v4.16b, v4.16b aese v4.16b, v8.16b aesmc v4.16b, v4.16b aese v4.16b, v9.16b aesmc v4.16b, v4.16b aese v4.16b, v10.16b aesmc v4.16b, v4.16b ld1 {v7.2d, v8.2d, v9.2d, v10.2d}, [x0], #0x40 aese v4.16b, v7.16b aesmc v4.16b, v4.16b aese v4.16b, v8.16b aesmc v4.16b, v4.16b aese v4.16b, v9.16b aesmc v4.16b, v4.16b aese v4.16b, v10.16b aesmc v4.16b, v4.16b subs w1, w1, #10 ld1 {v7.2d, v8.2d}, [x0], #32 aese v4.16b, v7.16b aesmc v4.16b, v4.16b aese v4.16b, v8.16b beq L_aes_gcm_init_arm64_crypto_eor3_round_done ld1 {v7.2d, v8.2d}, [x0], #32 subs w1, w1, #2 aesmc v4.16b, v4.16b aese v4.16b, v7.16b aesmc v4.16b, v4.16b aese v4.16b, v8.16b beq L_aes_gcm_init_arm64_crypto_eor3_round_done ld1 {v7.2d, v8.2d}, [x0], #32 aesmc v4.16b, v4.16b aese v4.16b, v7.16b aesmc v4.16b, v4.16b aese v4.16b, v8.16b L_aes_gcm_init_arm64_crypto_eor3_round_done: ld1 {v7.2d}, [x0] eor v4.16b, v4.16b, v7.16b st1 {v4.2d}, [x6] ldp d8, d9, [x29, #16] ldp d10, d11, [x29, #32] ldp x29, x30, [sp], #48 ret #ifndef __APPLE__ .size AES_GCM_init_AARCH64_EOR3,.-AES_GCM_init_AARCH64_EOR3 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_ghash_block_AARCH64_EOR3 .type AES_GCM_ghash_block_AARCH64_EOR3,@function .align 2 AES_GCM_ghash_block_AARCH64_EOR3: #else .section __TEXT,__text .globl _AES_GCM_ghash_block_AARCH64_EOR3 .p2align 2 _AES_GCM_ghash_block_AARCH64_EOR3: #endif /* __APPLE__ */ stp x29, x30, [sp, #-32]! add x29, sp, #0 stp d8, d9, [x29, #16] ld1 {v6.2d}, [x1] movi v7.16b, #0x87 ld1 {v5.2d}, [x2] ushr v7.2d, v7.2d, #56 ld1 {v4.2d}, [x0] rbit v4.16b, v4.16b eor v8.16b, v6.16b, v4.16b # X = C * H^1 pmull v0.1q, v8.1d, v5.1d pmull2 v1.1q, v8.2d, v5.2d ext v3.16b, v8.16b, v8.16b, #8 pmull v2.1q, v3.1d, v5.1d pmull2 v3.1q, v3.2d, v5.2d eor v2.16b, v2.16b, v3.16b # Reduce ext v3.16b, v0.16b, v1.16b, #8 pmull2 v1.1q, v1.2d, v7.2d eor3 v3.16b, v3.16b, v1.16b, v2.16b pmull2 v2.1q, v3.2d, v7.2d mov v0.d[1], v3.d[0] eor v6.16b, v0.16b, v2.16b # Done GHASH st1 {v6.2d}, [x1] ldp d8, d9, [x29, #16] ldp x29, x30, [sp], #32 ret #ifndef __APPLE__ .size AES_GCM_ghash_block_AARCH64_EOR3,.-AES_GCM_ghash_block_AARCH64_EOR3 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_aad_update_AARCH64_EOR3 .type AES_GCM_aad_update_AARCH64_EOR3,@function .align 2 AES_GCM_aad_update_AARCH64_EOR3: #else .section __TEXT,__text .globl _AES_GCM_aad_update_AARCH64_EOR3 .p2align 2 _AES_GCM_aad_update_AARCH64_EOR3: #endif /* __APPLE__ */ stp x29, x30, [sp, #-80]! add x29, sp, #0 stp d8, d9, [x29, #16] stp d10, d11, [x29, #32] stp d12, d13, [x29, #48] stp d14, d15, [x29, #64] ld1 {v20.2d}, [x2] movi v21.16b, #0x87 ld1 {v12.2d}, [x3] ushr v21.2d, v21.2d, #56 cmp x1, #0x40 blt L_aes_gcm_aad_update_arm64_crypto_eor3_h_done # Square H => H^2 pmull2 v11.1q, v12.2d, v12.2d pmull v10.1q, v12.1d, v12.1d pmull2 v8.1q, v11.2d, v21.2d ext v9.16b, v10.16b, v11.16b, #8 eor v9.16b, v9.16b, v8.16b pmull2 v11.1q, v9.2d, v21.2d mov v10.d[1], v9.d[0] eor v13.16b, v10.16b, v11.16b cmp x1, #0x100 blt L_aes_gcm_aad_update_arm64_crypto_eor3_h_done # Multiply H and H^2 => H^3 pmull v8.1q, v12.1d, v13.1d pmull2 v9.1q, v12.2d, v13.2d ext v11.16b, v12.16b, v12.16b, #8 pmull v10.1q, v11.1d, v13.1d pmull2 v11.1q, v11.2d, v13.2d eor v10.16b, v10.16b, v11.16b # Reduce ext v11.16b, v8.16b, v9.16b, #8 pmull2 v9.1q, v9.2d, v21.2d eor3 v11.16b, v11.16b, v9.16b, v10.16b pmull2 v10.1q, v11.2d, v21.2d mov v8.d[1], v11.d[0] eor v14.16b, v8.16b, v10.16b # Square H^2 => H^4 pmull2 v11.1q, v13.2d, v13.2d pmull v10.1q, v13.1d, v13.1d pmull2 v8.1q, v11.2d, v21.2d ext v9.16b, v10.16b, v11.16b, #8 eor v9.16b, v9.16b, v8.16b pmull2 v11.1q, v9.2d, v21.2d mov v10.d[1], v9.d[0] eor v15.16b, v10.16b, v11.16b # Done cmp x1, #0x400 blt L_aes_gcm_aad_update_arm64_crypto_eor3_h_done # Multiply H and H^4 => H^5 pmull v8.1q, v12.1d, v15.1d pmull2 v9.1q, v12.2d, v15.2d ext v11.16b, v12.16b, v12.16b, #8 pmull v10.1q, v11.1d, v15.1d pmull2 v11.1q, v11.2d, v15.2d eor v10.16b, v10.16b, v11.16b # Reduce ext v11.16b, v8.16b, v9.16b, #8 pmull2 v9.1q, v9.2d, v21.2d eor3 v11.16b, v11.16b, v9.16b, v10.16b pmull2 v10.1q, v11.2d, v21.2d mov v8.d[1], v11.d[0] eor v16.16b, v8.16b, v10.16b # Square H^3 => H^6 pmull2 v11.1q, v14.2d, v14.2d pmull v10.1q, v14.1d, v14.1d pmull2 v8.1q, v11.2d, v21.2d ext v9.16b, v10.16b, v11.16b, #8 eor v9.16b, v9.16b, v8.16b pmull2 v11.1q, v9.2d, v21.2d mov v10.d[1], v9.d[0] eor v17.16b, v10.16b, v11.16b # Multiply H and H^6 => H^7 pmull v8.1q, v12.1d, v17.1d pmull2 v9.1q, v12.2d, v17.2d ext v11.16b, v12.16b, v12.16b, #8 pmull v10.1q, v11.1d, v17.1d pmull2 v11.1q, v11.2d, v17.2d eor v10.16b, v10.16b, v11.16b # Reduce ext v11.16b, v8.16b, v9.16b, #8 pmull2 v9.1q, v9.2d, v21.2d eor3 v11.16b, v11.16b, v9.16b, v10.16b pmull2 v10.1q, v11.2d, v21.2d mov v8.d[1], v11.d[0] eor v18.16b, v8.16b, v10.16b # Square H^4 => H^8 pmull2 v11.1q, v15.2d, v15.2d pmull v10.1q, v15.1d, v15.1d pmull2 v8.1q, v11.2d, v21.2d ext v9.16b, v10.16b, v11.16b, #8 eor v9.16b, v9.16b, v8.16b pmull2 v11.1q, v9.2d, v21.2d mov v10.d[1], v9.d[0] eor v19.16b, v10.16b, v11.16b # Done L_aes_gcm_aad_update_arm64_crypto_eor3_h_done: lsr x1, x1, #4 cmp x1, #4 blt L_aes_gcm_aad_update_arm64_crypto_eor3_start_1 cmp x1, #16 blt L_aes_gcm_aad_update_arm64_crypto_eor3_start_2 cmp x1, #0x40 blt L_aes_gcm_aad_update_arm64_crypto_eor3_start_4 L_aes_gcm_aad_update_arm64_crypto_eor3_start_8: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b rbit v4.16b, v4.16b rbit v5.16b, v5.16b rbit v6.16b, v6.16b rbit v7.16b, v7.16b eor v0.16b, v0.16b, v20.16b # X = C * H^1 pmull v8.1q, v7.1d, v12.1d pmull2 v9.1q, v7.2d, v12.2d ext v11.16b, v7.16b, v7.16b, #8 pmull v10.1q, v11.1d, v12.1d pmull2 v11.1q, v11.2d, v12.2d eor v10.16b, v10.16b, v11.16b # X += C * H^2 pmull v11.1q, v13.1d, v6.1d pmull2 v20.1q, v13.2d, v6.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v6.16b, v6.16b, #8 pmull v11.1q, v20.1d, v13.1d pmull2 v20.1q, v20.2d, v13.2d eor3 v10.16b, v10.16b, v20.16b, v11.16b # X += C * H^3 pmull v11.1q, v14.1d, v5.1d pmull2 v20.1q, v14.2d, v5.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v5.16b, v5.16b, #8 pmull v11.1q, v20.1d, v14.1d pmull2 v20.1q, v20.2d, v14.2d eor3 v10.16b, v10.16b, v20.16b, v11.16b # X += C * H^4 pmull v11.1q, v15.1d, v4.1d pmull2 v20.1q, v15.2d, v4.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v4.16b, v4.16b, #8 pmull v11.1q, v20.1d, v15.1d pmull2 v20.1q, v20.2d, v15.2d eor3 v10.16b, v10.16b, v20.16b, v11.16b # X += C * H^5 pmull v11.1q, v16.1d, v3.1d pmull2 v20.1q, v16.2d, v3.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v3.16b, v3.16b, #8 pmull v11.1q, v20.1d, v16.1d pmull2 v20.1q, v20.2d, v16.2d eor3 v10.16b, v10.16b, v20.16b, v11.16b # X += C * H^6 pmull v11.1q, v17.1d, v2.1d pmull2 v20.1q, v17.2d, v2.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v2.16b, v2.16b, #8 pmull v11.1q, v20.1d, v17.1d pmull2 v20.1q, v20.2d, v17.2d eor3 v10.16b, v10.16b, v20.16b, v11.16b # X += C * H^7 pmull v11.1q, v18.1d, v1.1d pmull2 v20.1q, v18.2d, v1.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v1.16b, v1.16b, #8 pmull v11.1q, v20.1d, v18.1d pmull2 v20.1q, v20.2d, v18.2d eor3 v10.16b, v10.16b, v20.16b, v11.16b # X += C * H^8 pmull v11.1q, v19.1d, v0.1d pmull2 v20.1q, v19.2d, v0.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v0.16b, v0.16b, #8 pmull v11.1q, v20.1d, v19.1d pmull2 v20.1q, v20.2d, v19.2d eor3 v10.16b, v10.16b, v20.16b, v11.16b # Reduce ext v11.16b, v8.16b, v9.16b, #8 pmull2 v9.1q, v9.2d, v21.2d eor3 v11.16b, v11.16b, v9.16b, v10.16b pmull2 v10.1q, v11.2d, v21.2d mov v8.d[1], v11.d[0] eor v20.16b, v8.16b, v10.16b # Done GHASH sub x1, x1, #8 cmp x1, #8 bge L_aes_gcm_aad_update_arm64_crypto_eor3_start_8 cmp x1, #1 blt L_aes_gcm_aad_update_arm64_crypto_eor3_done beq L_aes_gcm_aad_update_arm64_crypto_eor3_start_1 cmp x1, #16 blt L_aes_gcm_aad_update_arm64_crypto_eor3_start_2 L_aes_gcm_aad_update_arm64_crypto_eor3_start_4: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v0.16b, v0.16b, v20.16b # X = C * H^1 pmull v8.1q, v3.1d, v12.1d pmull2 v9.1q, v3.2d, v12.2d ext v11.16b, v3.16b, v3.16b, #8 pmull v10.1q, v11.1d, v12.1d pmull2 v11.1q, v11.2d, v12.2d eor v10.16b, v10.16b, v11.16b # X += C * H^2 pmull v11.1q, v13.1d, v2.1d pmull2 v20.1q, v13.2d, v2.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v2.16b, v2.16b, #8 pmull v11.1q, v20.1d, v13.1d pmull2 v20.1q, v20.2d, v13.2d eor3 v10.16b, v10.16b, v20.16b, v11.16b # X += C * H^3 pmull v11.1q, v14.1d, v1.1d pmull2 v20.1q, v14.2d, v1.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v1.16b, v1.16b, #8 pmull v11.1q, v20.1d, v14.1d pmull2 v20.1q, v20.2d, v14.2d eor3 v10.16b, v10.16b, v20.16b, v11.16b # X += C * H^4 pmull v11.1q, v15.1d, v0.1d pmull2 v20.1q, v15.2d, v0.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v0.16b, v0.16b, #8 pmull v11.1q, v20.1d, v15.1d pmull2 v20.1q, v20.2d, v15.2d eor3 v10.16b, v10.16b, v20.16b, v11.16b # Reduce ext v11.16b, v8.16b, v9.16b, #8 pmull2 v9.1q, v9.2d, v21.2d eor3 v11.16b, v11.16b, v9.16b, v10.16b pmull2 v10.1q, v11.2d, v21.2d mov v8.d[1], v11.d[0] eor v20.16b, v8.16b, v10.16b # Done GHASH sub x1, x1, #4 cmp x1, #4 bge L_aes_gcm_aad_update_arm64_crypto_eor3_start_4 cmp x1, #1 blt L_aes_gcm_aad_update_arm64_crypto_eor3_done beq L_aes_gcm_aad_update_arm64_crypto_eor3_start_1 L_aes_gcm_aad_update_arm64_crypto_eor3_start_2: ld1 {v0.16b, v1.16b}, [x0], #32 rbit v0.16b, v0.16b rbit v1.16b, v1.16b eor v3.16b, v20.16b, v0.16b # X = C * H^1 pmull v8.1q, v1.1d, v12.1d pmull2 v9.1q, v1.2d, v12.2d ext v11.16b, v1.16b, v1.16b, #8 pmull v10.1q, v11.1d, v12.1d pmull2 v11.1q, v11.2d, v12.2d eor v10.16b, v10.16b, v11.16b # X += C * H^2 pmull v11.1q, v13.1d, v3.1d pmull2 v20.1q, v13.2d, v3.2d eor v8.16b, v8.16b, v11.16b eor v9.16b, v9.16b, v20.16b ext v20.16b, v3.16b, v3.16b, #8 pmull v11.1q, v20.1d, v13.1d pmull2 v20.1q, v20.2d, v13.2d eor3 v10.16b, v10.16b, v20.16b, v11.16b # Reduce ext v11.16b, v8.16b, v9.16b, #8 pmull2 v9.1q, v9.2d, v21.2d eor3 v11.16b, v11.16b, v9.16b, v10.16b pmull2 v10.1q, v11.2d, v21.2d mov v8.d[1], v11.d[0] eor v20.16b, v8.16b, v10.16b # Done GHASH sub x1, x1, #2 cmp x1, #1 bgt L_aes_gcm_aad_update_arm64_crypto_eor3_start_2 blt L_aes_gcm_aad_update_arm64_crypto_eor3_done L_aes_gcm_aad_update_arm64_crypto_eor3_start_1: cbz x1, L_aes_gcm_aad_update_arm64_crypto_eor3_done L_aes_gcm_aad_update_arm64_crypto_eor3_both_1: ld1 {v0.16b}, [x0], #16 rbit v0.16b, v0.16b eor v3.16b, v20.16b, v0.16b # X = C * H^1 pmull v8.1q, v3.1d, v12.1d pmull2 v9.1q, v3.2d, v12.2d ext v11.16b, v3.16b, v3.16b, #8 pmull v10.1q, v11.1d, v12.1d pmull2 v11.1q, v11.2d, v12.2d eor v10.16b, v10.16b, v11.16b # Reduce ext v11.16b, v8.16b, v9.16b, #8 pmull2 v9.1q, v9.2d, v21.2d eor3 v11.16b, v11.16b, v9.16b, v10.16b pmull2 v10.1q, v11.2d, v21.2d mov v8.d[1], v11.d[0] eor v20.16b, v8.16b, v10.16b # Done GHASH subs x1, x1, #1 bne L_aes_gcm_aad_update_arm64_crypto_eor3_both_1 L_aes_gcm_aad_update_arm64_crypto_eor3_done: st1 {v20.2d}, [x2] ldp d8, d9, [x29, #16] ldp d10, d11, [x29, #32] ldp d12, d13, [x29, #48] ldp d14, d15, [x29, #64] ldp x29, x30, [sp], #0x50 ret #ifndef __APPLE__ .size AES_GCM_aad_update_AARCH64_EOR3,.-AES_GCM_aad_update_AARCH64_EOR3 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_encrypt_block_AARCH64_EOR3 .type AES_GCM_encrypt_block_AARCH64_EOR3,@function .align 2 AES_GCM_encrypt_block_AARCH64_EOR3: #else .section __TEXT,__text .globl _AES_GCM_encrypt_block_AARCH64_EOR3 .p2align 2 _AES_GCM_encrypt_block_AARCH64_EOR3: #endif /* __APPLE__ */ ld1 {v5.2d}, [x4] ld1 {v4.2d}, [x3] mov w5, v5.s[3] rev w5, w5 add w5, w5, #1 rev w5, w5 mov v5.s[3], w5 st1 {v5.2d}, [x4] ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 aese v5.16b, v0.16b aesmc v5.16b, v5.16b aese v5.16b, v1.16b aesmc v5.16b, v5.16b aese v5.16b, v2.16b aesmc v5.16b, v5.16b aese v5.16b, v3.16b aesmc v5.16b, v5.16b ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 aese v5.16b, v0.16b aesmc v5.16b, v5.16b aese v5.16b, v1.16b aesmc v5.16b, v5.16b aese v5.16b, v2.16b aesmc v5.16b, v5.16b aese v5.16b, v3.16b aesmc v5.16b, v5.16b subs w1, w1, #10 ld1 {v0.2d, v1.2d}, [x0], #32 aese v5.16b, v0.16b aesmc v5.16b, v5.16b aese v5.16b, v1.16b beq L_aes_gcm_encrypt_block_arm64_crypto_eor3_round_done ld1 {v0.2d, v1.2d}, [x0], #32 subs w1, w1, #2 aesmc v5.16b, v5.16b aese v5.16b, v0.16b aesmc v5.16b, v5.16b aese v5.16b, v1.16b beq L_aes_gcm_encrypt_block_arm64_crypto_eor3_round_done ld1 {v0.2d, v1.2d}, [x0], #32 aesmc v5.16b, v5.16b aese v5.16b, v0.16b aesmc v5.16b, v5.16b aese v5.16b, v1.16b L_aes_gcm_encrypt_block_arm64_crypto_eor3_round_done: ld1 {v0.2d}, [x0] eor v5.16b, v5.16b, v0.16b eor v4.16b, v4.16b, v5.16b st1 {v4.2d}, [x2] ret #ifndef __APPLE__ .size AES_GCM_encrypt_block_AARCH64_EOR3,.-AES_GCM_encrypt_block_AARCH64_EOR3 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_encrypt_update_AARCH64_EOR3 .type AES_GCM_encrypt_update_AARCH64_EOR3,@function .align 2 AES_GCM_encrypt_update_AARCH64_EOR3: #else .section __TEXT,__text .globl _AES_GCM_encrypt_update_AARCH64_EOR3 .p2align 2 _AES_GCM_encrypt_update_AARCH64_EOR3: #endif /* __APPLE__ */ stp x29, x30, [sp, #-96]! add x29, sp, #0 str x17, [x29, #24] stp d8, d9, [x29, #32] stp d10, d11, [x29, #48] stp d12, d13, [x29, #64] stp d14, d15, [x29, #80] ld1 {v13.2d}, [x7] movi v27.16b, #0x87 ld1 {v26.2d}, [x5] ushr v27.2d, v27.2d, #56 ld1 {v22.2d}, [x6] mov w9, v13.s[3] rev w9, w9 cmp w4, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done # Square H => H^2 pmull2 v31.1q, v22.2d, v22.2d pmull v30.1q, v22.1d, v22.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v23.16b, v30.16b, v31.16b cmp w4, #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done # Multiply H and H^2 => H^3 pmull v28.1q, v22.1d, v23.1d pmull2 v29.1q, v22.2d, v23.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v23.1d pmull2 v31.1q, v31.2d, v23.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v24.16b, v28.16b, v30.16b # Square H^2 => H^4 pmull2 v31.1q, v23.2d, v23.2d pmull v30.1q, v23.1d, v23.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v25.16b, v30.16b, v31.16b # Done cmp w4, #0x200 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done # Multiply H and H^4 => H^5 pmull v28.1q, v22.1d, v25.1d pmull2 v29.1q, v22.2d, v25.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v25.1d pmull2 v31.1q, v31.2d, v25.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v4.16b, v28.16b, v30.16b # Square H^3 => H^6 pmull2 v31.1q, v24.2d, v24.2d pmull v30.1q, v24.1d, v24.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v5.16b, v30.16b, v31.16b # Multiply H and H^6 => H^7 pmull v28.1q, v22.1d, v5.1d pmull2 v29.1q, v22.2d, v5.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v5.1d pmull2 v31.1q, v31.2d, v5.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v6.16b, v28.16b, v30.16b # Square H^4 => H^8 pmull2 v31.1q, v25.2d, v25.2d pmull v30.1q, v25.1d, v25.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v7.16b, v30.16b, v31.16b # Done L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done: lsr w8, w4, #4 cmp x1, #12 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_start_128 bgt L_aes_gcm_encrypt_update_arm64_crypto_eor3_start_256 # AES_GCM_192 #ifndef NO_AES_192 cmp w8, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rev w17, w17 rev w16, w16 rev w15, w15 rev w14, w14 rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 mov v14.s[3], w17 mov v15.s[3], w16 mov v16.s[3], w15 mov v17.s[3], w14 mov v8.s[3], w13 mov v9.s[3], w12 mov v10.s[3], w11 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v20.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v21.16b}, [x3], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v0.16b}, [x3], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v1.16b}, [x3], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v2.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v3.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b ld1 {v13.2d}, [x7] st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_end_8 L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_both_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w17, w17 rbit v19.16b, v19.16b rev w16, w16 rbit v20.16b, v20.16b rev w15, w15 rbit v21.16b, v21.16b rev w14, w14 rbit v0.16b, v0.16b rev w13, w13 rbit v1.16b, v1.16b rev w12, w12 rbit v2.16b, v2.16b rev w11, w11 rbit v3.16b, v3.16b rev w10, w9 mov v14.s[3], w17 mov v15.s[3], w16 mov v16.s[3], w15 mov v17.s[3], w14 mov v8.s[3], w13 mov v9.s[3], w12 mov v10.s[3], w11 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v31.16b, v3.16b, v3.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b ext v26.16b, v2.16b, v2.16b, #8 aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b ext v26.16b, v1.16b, v1.16b, #8 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b ext v26.16b, v0.16b, v0.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b ext v26.16b, v21.16b, v21.16b, #8 aese v9.16b, v13.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v8.16b, v12.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ext v26.16b, v19.16b, v19.16b, #8 aese v17.16b, v13.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull2 v29.1q, v29.2d, v27.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # Done GHASH aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v18.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v19.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x3], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x3], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x7] eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_both_8 L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 ld1 {v12.2d}, [x0] cmp w8, #1 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_done beq L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_1 cmp w8, #4 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_2 add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w12 mov v16.s[3], w11 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b ld1 {v19.16b}, [x3], #16 aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x3], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x3], #16 aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w8, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_end_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_both_4: add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w13, w13 rev w12, w12 rev w11, w11 rbit v19.16b, v19.16b rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w12 mov v16.s[3], w11 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b rbit v20.16b, v20.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b rbit v21.16b, v21.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b eor v18.16b, v18.16b, v26.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b ext v31.16b, v21.16b, v21.16b, #8 aese v15.16b, v1.16b aesmc v15.16b, v15.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v31.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b ext v26.16b, v20.16b, v20.16b, #8 aese v16.16b, v2.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v17.16b, v2.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v16.16b, v3.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b ext v26.16b, v19.16b, v19.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v4.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v5.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v17.16b, v5.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v29.1q, v29.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v15.16b, v7.16b aesmc v15.16b, v15.16b mov v28.d[1], v31.d[0] aese v16.16b, v7.16b aesmc v16.16b, v16.16b eor v26.16b, v28.16b, v30.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b # Done GHASH aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w8, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 bge L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_both_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w8, #1 beq L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_1 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_done L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_2: add w13, w9, #1 mov v14.16b, v13.16b add w9, w9, #2 mov v15.16b, v13.16b rev w13, w13 rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w8, w8, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x3], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b st1 {v18.16b, v19.16b}, [x2], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w8, L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_done L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_1: add w9, w9, #1 mov v14.16b, v13.16b rev w10, w9 mov v14.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b eor v18.16b, v18.16b, v14.16b st1 {v18.16b}, [x2], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_done: #endif /* !NO_AES_192 */ b L_aes_gcm_encrypt_update_arm64_crypto_eor3_done # AES_GCM_256 L_aes_gcm_encrypt_update_arm64_crypto_eor3_start_256: #ifndef NO_AES_256 cmp w8, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rev w17, w17 rev w16, w16 rev w15, w15 rev w14, w14 rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 mov v14.s[3], w17 mov v15.s[3], w16 mov v16.s[3], w15 mov v17.s[3], w14 mov v8.s[3], w13 mov v9.s[3], w12 mov v10.s[3], w11 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v20.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v21.16b}, [x3], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v0.16b}, [x3], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v1.16b}, [x3], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v2.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v3.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b ld1 {v13.2d}, [x7] st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_end_8 L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_both_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w17, w17 rbit v19.16b, v19.16b rev w16, w16 rbit v20.16b, v20.16b rev w15, w15 rbit v21.16b, v21.16b rev w14, w14 rbit v0.16b, v0.16b rev w13, w13 rbit v1.16b, v1.16b rev w12, w12 rbit v2.16b, v2.16b rev w11, w11 rbit v3.16b, v3.16b rev w10, w9 mov v14.s[3], w17 mov v15.s[3], w16 mov v16.s[3], w15 mov v17.s[3], w14 mov v8.s[3], w13 mov v9.s[3], w12 mov v10.s[3], w11 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v31.16b, v3.16b, v3.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b ext v26.16b, v2.16b, v2.16b, #8 aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b ext v26.16b, v1.16b, v1.16b, #8 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b ext v26.16b, v0.16b, v0.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b ext v26.16b, v21.16b, v21.16b, #8 aese v9.16b, v13.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v8.16b, v12.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ext v26.16b, v19.16b, v19.16b, #8 aese v17.16b, v13.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull2 v29.1q, v29.2d, v27.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # Done GHASH aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v18.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v19.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x3], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x3], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x7] eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_both_8 L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 ld1 {v12.2d}, [x0], #16 cmp w8, #1 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_done beq L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_1 cmp w8, #4 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_2 add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w12 mov v16.s[3], w11 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b ld1 {v19.16b}, [x3], #16 aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x3], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x3], #16 aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w8, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_end_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_both_4: add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w13, w13 rev w12, w12 rev w11, w11 rbit v19.16b, v19.16b rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w12 mov v16.s[3], w11 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b rbit v20.16b, v20.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b rbit v21.16b, v21.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b eor v18.16b, v18.16b, v26.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b ext v31.16b, v21.16b, v21.16b, #8 aese v15.16b, v1.16b aesmc v15.16b, v15.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v31.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b ext v26.16b, v20.16b, v20.16b, #8 aese v16.16b, v2.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v17.16b, v2.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v16.16b, v3.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b ext v26.16b, v19.16b, v19.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v4.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v5.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v17.16b, v5.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v29.1q, v29.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v15.16b, v7.16b aesmc v15.16b, v15.16b mov v28.d[1], v31.d[0] aese v16.16b, v7.16b aesmc v16.16b, v16.16b eor v26.16b, v28.16b, v30.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b # Done GHASH aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w8, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 bge L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_both_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w8, #1 beq L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_1 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_done L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_2: add w13, w9, #1 mov v14.16b, v13.16b add w9, w9, #2 mov v15.16b, v13.16b rev w13, w13 rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w8, w8, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x3], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b ld1 {v29.2d, v30.2d}, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b st1 {v18.16b, v19.16b}, [x2], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w8, L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_done L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_1: add w9, w9, #1 mov v14.16b, v13.16b rev w10, w9 mov v14.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b ldr q29, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ldr q30, [x0, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v18.16b, v18.16b, v14.16b st1 {v18.16b}, [x2], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_done: #endif /* !NO_AES_256 */ b L_aes_gcm_encrypt_update_arm64_crypto_eor3_done # AES_GCM_128 L_aes_gcm_encrypt_update_arm64_crypto_eor3_start_128: #ifndef NO_AES_128 cmp w8, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rev w17, w17 rev w16, w16 rev w15, w15 rev w14, w14 rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 mov v14.s[3], w17 mov v15.s[3], w16 mov v16.s[3], w15 mov v17.s[3], w14 mov v8.s[3], w13 mov v9.s[3], w12 mov v10.s[3], w11 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v20.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v21.16b}, [x3], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b ld1 {v0.16b}, [x3], #16 aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b ld1 {v1.16b}, [x3], #16 aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b ld1 {v2.16b}, [x3], #16 aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b ld1 {v3.16b}, [x3], #16 aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b ld1 {v13.2d}, [x7] st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_end_8 L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_both_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w17, w17 rbit v19.16b, v19.16b rev w16, w16 rbit v20.16b, v20.16b rev w15, w15 rbit v21.16b, v21.16b rev w14, w14 rbit v0.16b, v0.16b rev w13, w13 rbit v1.16b, v1.16b rev w12, w12 rbit v2.16b, v2.16b rev w11, w11 rbit v3.16b, v3.16b rev w10, w9 mov v14.s[3], w17 mov v15.s[3], w16 mov v16.s[3], w15 mov v17.s[3], w14 mov v8.s[3], w13 mov v9.s[3], w12 mov v10.s[3], w11 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v31.16b, v3.16b, v3.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v30.16b, v30.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b ext v26.16b, v2.16b, v2.16b, #8 aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b ext v26.16b, v1.16b, v1.16b, #8 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b ext v26.16b, v0.16b, v0.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b ext v26.16b, v21.16b, v21.16b, #8 aese v9.16b, v13.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v8.16b, v12.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ext v26.16b, v19.16b, v19.16b, #8 aese v17.16b, v13.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v10.16b, v12.16b aesmc v10.16b, v10.16b pmull2 v29.1q, v29.2d, v27.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b pmull2 v30.1q, v31.2d, v27.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b mov v28.d[1], v31.d[0] aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor v26.16b, v28.16b, v30.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b # Done GHASH aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v18.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v19.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v20.16b}, [x3], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v21.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x3], #16 aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x7] eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_both_8 L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 ld1 {v8.2d, v9.2d}, [x0], #32 ld1 {v10.2d}, [x0] cmp w8, #1 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_done beq L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_1 cmp w8, #4 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_2 add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w12 mov v16.s[3], w11 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b ld1 {v19.16b}, [x3], #16 aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v20.16b}, [x3], #16 aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b ld1 {v21.16b}, [x3], #16 aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w8, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_end_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_both_4: add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w13, w13 rev w12, w12 rev w11, w11 rbit v19.16b, v19.16b rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w12 mov v16.s[3], w11 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b rbit v20.16b, v20.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b rbit v21.16b, v21.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b eor v18.16b, v18.16b, v26.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b ext v31.16b, v21.16b, v21.16b, #8 aese v15.16b, v1.16b aesmc v15.16b, v15.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v30.16b, v30.16b, v31.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b ext v26.16b, v20.16b, v20.16b, #8 aese v16.16b, v2.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v17.16b, v2.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v16.16b, v3.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b ext v26.16b, v19.16b, v19.16b, #8 aese v14.16b, v4.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v15.16b, v4.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v14.16b, v5.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b ext v26.16b, v18.16b, v18.16b, #8 aese v16.16b, v5.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v17.16b, v5.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v29.1q, v29.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v15.16b, v7.16b aesmc v15.16b, v15.16b mov v28.d[1], v31.d[0] aese v16.16b, v7.16b aesmc v16.16b, v16.16b eor v26.16b, v28.16b, v30.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b # Done GHASH aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b eor v20.16b, v20.16b, v16.16b eor v21.16b, v21.16b, v17.16b cmp w8, #4 st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 bge L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_both_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w8, #1 beq L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_1 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_done L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_2: add w13, w9, #1 mov v14.16b, v13.16b add w9, w9, #2 mov v15.16b, v13.16b rev w13, w13 rev w10, w9 mov v14.s[3], w13 mov v15.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w8, w8, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v19.16b}, [x3], #16 aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b eor v18.16b, v18.16b, v14.16b eor v19.16b, v19.16b, v15.16b st1 {v18.16b, v19.16b}, [x2], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w8, L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_done L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_1: add w9, w9, #1 mov v14.16b, v13.16b rev w10, w9 mov v14.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v18.16b}, [x3], #16 eor v18.16b, v18.16b, v14.16b st1 {v18.16b}, [x2], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_done: #endif /* !NO_AES_128 */ L_aes_gcm_encrypt_update_arm64_crypto_eor3_done: rev w9, w9 mov v13.s[3], w9 st1 {v26.2d}, [x5] st1 {v13.2d}, [x7] ldr x17, [x29, #24] ldp d8, d9, [x29, #32] ldp d10, d11, [x29, #48] ldp d12, d13, [x29, #64] ldp d14, d15, [x29, #80] ldp x29, x30, [sp], #0x60 ret #ifndef __APPLE__ .size AES_GCM_encrypt_update_AARCH64_EOR3,.-AES_GCM_encrypt_update_AARCH64_EOR3 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_encrypt_final_AARCH64_EOR3 .type AES_GCM_encrypt_final_AARCH64_EOR3,@function .align 2 AES_GCM_encrypt_final_AARCH64_EOR3: #else .section __TEXT,__text .globl _AES_GCM_encrypt_final_AARCH64_EOR3 .p2align 2 _AES_GCM_encrypt_final_AARCH64_EOR3: #endif /* __APPLE__ */ ld1 {v5.2d}, [x0] movi v6.16b, #0x87 ld1 {v4.2d}, [x5] ushr v6.2d, v6.2d, #56 ld1 {v7.2d}, [x6] lsl x4, x4, #3 rbit x4, x4 mov v0.d[0], x4 lsl x3, x3, #3 rbit x3, x3 mov v0.d[1], x3 eor v5.16b, v5.16b, v0.16b pmull v0.1q, v5.1d, v4.1d pmull2 v1.1q, v5.2d, v4.2d ext v3.16b, v5.16b, v5.16b, #8 pmull v2.1q, v3.1d, v4.1d pmull2 v3.1q, v3.2d, v4.2d eor v2.16b, v2.16b, v3.16b # Reduce ext v3.16b, v0.16b, v1.16b, #8 pmull2 v1.1q, v1.2d, v6.2d eor3 v3.16b, v3.16b, v1.16b, v2.16b pmull2 v2.1q, v3.2d, v6.2d mov v0.d[1], v3.d[0] eor v5.16b, v0.16b, v2.16b rbit v5.16b, v5.16b eor v5.16b, v5.16b, v7.16b cmp w2, #16 bne L_aes_gcm_encrypt_final_arm64_crypto_eor3_tag_partial st1 {v5.16b}, [x1] b L_aes_gcm_encrypt_final_arm64_crypto_eor3_done L_aes_gcm_encrypt_final_arm64_crypto_eor3_tag_partial: st1 {v5.16b}, [x0] cmp w2, #8 blt L_aes_gcm_encrypt_final_arm64_crypto_eor3_tag_start_dw ldr x8, [x0], #8 sub w2, w2, #8 str x8, [x1], #8 L_aes_gcm_encrypt_final_arm64_crypto_eor3_tag_start_dw: cmp w2, #4 blt L_aes_gcm_encrypt_final_arm64_crypto_eor3_tag_start_sw ldr w8, [x0], #4 sub w2, w2, #4 str w8, [x1], #4 L_aes_gcm_encrypt_final_arm64_crypto_eor3_tag_start_sw: cmp w2, #2 blt L_aes_gcm_encrypt_final_arm64_crypto_eor3_tag_start_byte ldrh w8, [x0], #2 sub w2, w2, #2 strh w8, [x1], #2 L_aes_gcm_encrypt_final_arm64_crypto_eor3_tag_start_byte: cbz w2, L_aes_gcm_encrypt_final_arm64_crypto_eor3_tag_end_bytes ldrb w8, [x0], #1 subs w2, w2, #1 strb w8, [x1], #1 bne L_aes_gcm_encrypt_final_arm64_crypto_eor3_tag_start_byte L_aes_gcm_encrypt_final_arm64_crypto_eor3_tag_end_bytes: L_aes_gcm_encrypt_final_arm64_crypto_eor3_done: ret #ifndef __APPLE__ .size AES_GCM_encrypt_final_AARCH64_EOR3,.-AES_GCM_encrypt_final_AARCH64_EOR3 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_decrypt_update_AARCH64_EOR3 .type AES_GCM_decrypt_update_AARCH64_EOR3,@function .align 2 AES_GCM_decrypt_update_AARCH64_EOR3: #else .section __TEXT,__text .globl _AES_GCM_decrypt_update_AARCH64_EOR3 .p2align 2 _AES_GCM_decrypt_update_AARCH64_EOR3: #endif /* __APPLE__ */ stp x29, x30, [sp, #-96]! add x29, sp, #0 str x17, [x29, #24] stp d8, d9, [x29, #32] stp d10, d11, [x29, #48] stp d12, d13, [x29, #64] stp d14, d15, [x29, #80] ld1 {v13.2d}, [x7] movi v27.16b, #0x87 ld1 {v26.2d}, [x5] ushr v27.2d, v27.2d, #56 ld1 {v22.2d}, [x6] mov w9, v13.s[3] rev w9, w9 cmp w4, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done # Square H => H^2 pmull2 v31.1q, v22.2d, v22.2d pmull v30.1q, v22.1d, v22.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v23.16b, v30.16b, v31.16b cmp w4, #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done # Multiply H and H^2 => H^3 pmull v28.1q, v22.1d, v23.1d pmull2 v29.1q, v22.2d, v23.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v23.1d pmull2 v31.1q, v31.2d, v23.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v24.16b, v28.16b, v30.16b # Square H^2 => H^4 pmull2 v31.1q, v23.2d, v23.2d pmull v30.1q, v23.1d, v23.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v25.16b, v30.16b, v31.16b # Done cmp w4, #0x200 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done # Multiply H and H^4 => H^5 pmull v28.1q, v22.1d, v25.1d pmull2 v29.1q, v22.2d, v25.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v25.1d pmull2 v31.1q, v31.2d, v25.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v4.16b, v28.16b, v30.16b # Square H^3 => H^6 pmull2 v31.1q, v24.2d, v24.2d pmull v30.1q, v24.1d, v24.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v5.16b, v30.16b, v31.16b # Multiply H and H^6 => H^7 pmull v28.1q, v22.1d, v5.1d pmull2 v29.1q, v22.2d, v5.2d ext v31.16b, v22.16b, v22.16b, #8 pmull v30.1q, v31.1d, v5.1d pmull2 v31.1q, v31.2d, v5.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v6.16b, v28.16b, v30.16b # Square H^4 => H^8 pmull2 v31.1q, v25.2d, v25.2d pmull v30.1q, v25.1d, v25.1d pmull2 v28.1q, v31.2d, v27.2d ext v29.16b, v30.16b, v31.16b, #8 eor v29.16b, v29.16b, v28.16b pmull2 v31.1q, v29.2d, v27.2d mov v30.d[1], v29.d[0] eor v7.16b, v30.16b, v31.16b # Done L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done: lsr w8, w4, #4 cmp x1, #12 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_start_128 bgt L_aes_gcm_decrypt_update_arm64_crypto_eor3_start_256 # AES_GCM_192 #ifndef NO_AES_192 cmp w8, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rev w17, w17 mov v14.s[3], w17 rev w16, w16 mov v15.s[3], w16 rev w15, w15 mov v16.s[3], w15 rev w14, w14 mov v17.s[3], w14 rev w13, w13 mov v8.s[3], w13 rev w12, w12 mov v9.s[3], w12 rev w11, w11 mov v10.s[3], w11 rev w10, w9 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v18.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v19.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v20.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v21.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x3], #16 aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x3], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x3], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b ld1 {v13.2d}, [x7] st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_end_8 L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_both_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w17, w17 rbit v19.16b, v19.16b mov v14.s[3], w17 rev w16, w16 rbit v20.16b, v20.16b mov v15.s[3], w16 rev w15, w15 rbit v21.16b, v21.16b mov v16.s[3], w15 rev w14, w14 rbit v0.16b, v0.16b mov v17.s[3], w14 rev w13, w13 rbit v1.16b, v1.16b mov v8.s[3], w13 rev w12, w12 rbit v2.16b, v2.16b mov v9.s[3], w12 rev w11, w11 rbit v3.16b, v3.16b mov v10.s[3], w11 rev w10, w9 eor v18.16b, v18.16b, v26.16b mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v31.16b, v3.16b, v3.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b ext v26.16b, v2.16b, v2.16b, #8 aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b ext v26.16b, v1.16b, v1.16b, #8 aese v10.16b, v13.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b ext v26.16b, v0.16b, v0.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ext v26.16b, v21.16b, v21.16b, #8 aese v8.16b, v13.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v26.16b, v20.16b, v20.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ext v26.16b, v18.16b, v18.16b, #8 aese v15.16b, v12.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull2 v29.1q, v29.2d, v27.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b mov v28.d[1], v31.d[0] aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v26.16b, v28.16b, v30.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # Done GHASH aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v18.16b}, [x3], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v19.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v20.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v21.16b}, [x3], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v0.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v1.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v2.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v3.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x7] eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_both_8 L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 ld1 {v12.2d}, [x0] cmp w8, #1 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_done beq L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_1 cmp w8, #4 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_2 add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rev w13, w13 mov v14.s[3], w13 rev w12, w12 mov v15.s[3], w12 rev w11, w11 mov v16.s[3], w11 rev w10, w9 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x3], #16 aese v17.16b, v7.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x3], #16 aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v20.16b}, [x3], #16 aese v17.16b, v8.16b aesmc v17.16b, v17.16b ld1 {v21.16b}, [x3], #16 aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w8, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_end_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_both_4: add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w13, w13 rbit v19.16b, v19.16b mov v14.s[3], w13 rev w12, w12 rbit v20.16b, v20.16b mov v15.s[3], w12 rev w11, w11 rbit v21.16b, v21.16b mov v16.s[3], w11 rev w10, w9 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v16.16b, v0.16b aesmc v16.16b, v16.16b ext v31.16b, v21.16b, v21.16b, #8 aese v17.16b, v0.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v2.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v3.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v17.16b, v3.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b ext v26.16b, v18.16b, v18.16b, #8 aese v14.16b, v5.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v15.16b, v5.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v15.16b, v6.16b aesmc v15.16b, v15.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v30.1q, v31.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 mov v28.d[1], v31.d[0] aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b # Done GHASH aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v11.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v11.16b eor v17.16b, v17.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w8, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 bge L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_both_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w8, #1 beq L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_1 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_done L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_2: add w13, w9, #1 mov v14.16b, v13.16b add w9, w9, #2 mov v15.16b, v13.16b rev w13, w13 mov v14.s[3], w13 rev w10, w9 mov v15.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w8, w8, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x3], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v11.16b eor v15.16b, v15.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b st1 {v14.16b, v15.16b}, [x2], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w8, L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_done L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_1: ld1 {v15.16b}, [x3], #16 add w9, w9, #1 mov v14.16b, v13.16b rbit v15.16b, v15.16b rev w10, w9 mov v14.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v16.16b, v26.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v16.1d, v22.1d pmull2 v29.1q, v16.2d, v22.2d aese v14.16b, v2.16b aesmc v14.16b, v14.16b ext v31.16b, v16.16b, v16.16b, #8 aese v14.16b, v3.16b aesmc v14.16b, v14.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v4.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v14.16b, v9.16b aesmc v14.16b, v14.16b mov v28.d[1], v31.d[0] aese v14.16b, v10.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v14.16b, v11.16b eor v14.16b, v14.16b, v12.16b # Done GHASH rbit v15.16b, v15.16b eor v14.16b, v14.16b, v15.16b st1 {v14.16b}, [x2], #16 L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_done: #endif /* !NO_AES_192 */ b L_aes_gcm_decrypt_update_arm64_crypto_eor3_done # AES_GCM_256 L_aes_gcm_decrypt_update_arm64_crypto_eor3_start_256: #ifndef NO_AES_256 cmp w8, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rev w17, w17 mov v14.s[3], w17 rev w16, w16 mov v15.s[3], w16 rev w15, w15 mov v16.s[3], w15 rev w14, w14 mov v17.s[3], w14 rev w13, w13 mov v8.s[3], w13 rev w12, w12 mov v9.s[3], w12 rev w11, w11 mov v10.s[3], w11 rev w10, w9 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v18.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v19.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v20.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v21.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b ld1 {v0.16b}, [x3], #16 aese v17.16b, v13.16b aesmc v17.16b, v17.16b ld1 {v1.16b}, [x3], #16 aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v2.16b}, [x3], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v3.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b ld1 {v13.2d}, [x7] st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_end_8 L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_both_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w17, w17 rbit v19.16b, v19.16b mov v14.s[3], w17 rev w16, w16 rbit v20.16b, v20.16b mov v15.s[3], w16 rev w15, w15 rbit v21.16b, v21.16b mov v16.s[3], w15 rev w14, w14 rbit v0.16b, v0.16b mov v17.s[3], w14 rev w13, w13 rbit v1.16b, v1.16b mov v8.s[3], w13 rev w12, w12 rbit v2.16b, v2.16b mov v9.s[3], w12 rev w11, w11 rbit v3.16b, v3.16b mov v10.s[3], w11 rev w10, w9 eor v18.16b, v18.16b, v26.16b mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v31.16b, v3.16b, v3.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b ext v26.16b, v2.16b, v2.16b, #8 aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b ext v26.16b, v1.16b, v1.16b, #8 aese v10.16b, v13.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b ext v26.16b, v0.16b, v0.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ext v26.16b, v21.16b, v21.16b, #8 aese v8.16b, v13.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v26.16b, v20.16b, v20.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ext v26.16b, v18.16b, v18.16b, #8 aese v15.16b, v12.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull2 v29.1q, v29.2d, v27.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b mov v28.d[1], v31.d[0] aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v26.16b, v28.16b, v30.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # Done GHASH aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v18.16b}, [x3], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v19.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v20.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v21.16b}, [x3], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v0.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v1.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v2.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v3.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #176] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #192] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #208] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #224] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x7] eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_both_8 L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 ld1 {v12.2d}, [x0], #16 cmp w8, #1 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_done beq L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_1 cmp w8, #4 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_2 add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rev w13, w13 mov v14.s[3], w13 rev w12, w12 mov v15.s[3], w12 rev w11, w11 mov v16.s[3], w11 rev w10, w9 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x3], #16 aese v17.16b, v7.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x3], #16 aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v20.16b}, [x3], #16 aese v17.16b, v8.16b aesmc v17.16b, v17.16b ld1 {v21.16b}, [x3], #16 aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w8, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_end_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_both_4: add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w13, w13 rbit v19.16b, v19.16b mov v14.s[3], w13 rev w12, w12 rbit v20.16b, v20.16b mov v15.s[3], w12 rev w11, w11 rbit v21.16b, v21.16b mov v16.s[3], w11 rev w10, w9 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v16.16b, v0.16b aesmc v16.16b, v16.16b ext v31.16b, v21.16b, v21.16b, #8 aese v17.16b, v0.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v2.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v3.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v17.16b, v3.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b ext v26.16b, v18.16b, v18.16b, #8 aese v14.16b, v5.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v15.16b, v5.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v15.16b, v6.16b aesmc v15.16b, v15.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v30.1q, v31.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 mov v28.d[1], v31.d[0] aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b # Done GHASH aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v16.16b, v9.16b aesmc v16.16b, v16.16b aese v17.16b, v9.16b aesmc v17.16b, v17.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v16.16b, v10.16b aesmc v16.16b, v16.16b aese v17.16b, v10.16b aesmc v17.16b, v17.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b aese v16.16b, v11.16b aesmc v16.16b, v16.16b aese v17.16b, v11.16b aesmc v17.16b, v17.16b ld1 {v29.2d, v30.2d}, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b aese v16.16b, v29.16b eor v16.16b, v16.16b, v30.16b aese v17.16b, v29.16b eor v17.16b, v17.16b, v30.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w8, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 bge L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_both_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w8, #1 beq L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_1 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_done L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_2: add w13, w9, #1 mov v14.16b, v13.16b add w9, w9, #2 mov v15.16b, v13.16b rev w13, w13 mov v14.s[3], w13 rev w10, w9 mov v15.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w8, w8, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b ld1 {v19.16b}, [x3], #16 aese v15.16b, v9.16b aesmc v15.16b, v15.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b aese v15.16b, v10.16b aesmc v15.16b, v15.16b aese v14.16b, v11.16b aesmc v14.16b, v14.16b aese v15.16b, v11.16b aesmc v15.16b, v15.16b ld1 {v29.2d, v30.2d}, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b aese v15.16b, v29.16b eor v15.16b, v15.16b, v30.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b st1 {v14.16b, v15.16b}, [x2], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w8, L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_done L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_1: add w9, w9, #1 mov v14.16b, v13.16b rev w10, w9 mov v14.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b aesmc v14.16b, v14.16b aese v14.16b, v10.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v14.16b, v11.16b aesmc v14.16b, v14.16b ldr q29, [x0] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ldr q30, [x0, #16] aese v14.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v14.16b, v14.16b, v18.16b st1 {v14.16b}, [x2], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_done: #endif /* !NO_AES_256 */ b L_aes_gcm_decrypt_update_arm64_crypto_eor3_done # AES_GCM_128 L_aes_gcm_decrypt_update_arm64_crypto_eor3_start_128: #ifndef NO_AES_128 cmp w8, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rev w17, w17 mov v14.s[3], w17 rev w16, w16 mov v15.s[3], w16 rev w15, w15 mov v16.s[3], w15 rev w14, w14 mov v17.s[3], w14 rev w13, w13 mov v8.s[3], w13 rev w12, w12 mov v9.s[3], w12 rev w11, w11 mov v10.s[3], w11 rev w10, w9 mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v18.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v19.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v20.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v21.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b ld1 {v0.16b}, [x3], #16 aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b ld1 {v1.16b}, [x3], #16 aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b ld1 {v2.16b}, [x3], #16 aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b ld1 {v3.16b}, [x3], #16 aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b ld1 {v13.2d}, [x7] st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_end_8 L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_both_8: ldr q12, [x0] add w17, w9, #1 mov v14.16b, v13.16b add w16, w9, #2 mov v15.16b, v13.16b add w15, w9, #3 mov v16.16b, v13.16b add w14, w9, #4 mov v17.16b, v13.16b add w13, w9, #5 mov v8.16b, v13.16b add w12, w9, #6 mov v9.16b, v13.16b add w11, w9, #7 mov v10.16b, v13.16b add w9, w9, #8 mov v11.16b, v13.16b rbit v18.16b, v18.16b rev w17, w17 rbit v19.16b, v19.16b mov v14.s[3], w17 rev w16, w16 rbit v20.16b, v20.16b mov v15.s[3], w16 rev w15, w15 rbit v21.16b, v21.16b mov v16.s[3], w15 rev w14, w14 rbit v0.16b, v0.16b mov v17.s[3], w14 rev w13, w13 rbit v1.16b, v1.16b mov v8.s[3], w13 rev w12, w12 rbit v2.16b, v2.16b mov v9.s[3], w12 rev w11, w11 rbit v3.16b, v3.16b mov v10.s[3], w11 rev w10, w9 eor v18.16b, v18.16b, v26.16b mov v11.s[3], w10 ldr q13, [x0, #16] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b ext v31.16b, v3.16b, v3.16b, #8 aese v16.16b, v12.16b aesmc v16.16b, v16.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v30.16b, v30.16b, v31.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d aese v9.16b, v12.16b aesmc v9.16b, v9.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b ext v26.16b, v2.16b, v2.16b, #8 aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #32] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v13.16b aesmc v16.16b, v16.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d aese v8.16b, v13.16b aesmc v8.16b, v8.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b ext v26.16b, v1.16b, v1.16b, #8 aese v10.16b, v13.16b aesmc v10.16b, v10.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #48] aese v14.16b, v12.16b aesmc v14.16b, v14.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v15.16b, v12.16b aesmc v15.16b, v15.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d aese v17.16b, v12.16b aesmc v17.16b, v17.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b ext v26.16b, v0.16b, v0.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #64] aese v14.16b, v13.16b aesmc v14.16b, v14.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b ext v26.16b, v21.16b, v21.16b, #8 aese v8.16b, v13.16b aesmc v8.16b, v8.16b pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d aese v9.16b, v13.16b aesmc v9.16b, v9.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v10.16b, v13.16b aesmc v10.16b, v10.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #80] aese v14.16b, v12.16b aesmc v14.16b, v14.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d aese v15.16b, v12.16b aesmc v15.16b, v15.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v16.16b, v12.16b aesmc v16.16b, v16.16b ext v26.16b, v20.16b, v20.16b, #8 aese v17.16b, v12.16b aesmc v17.16b, v17.16b pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d aese v8.16b, v12.16b aesmc v8.16b, v8.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #96] aese v14.16b, v13.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v13.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v13.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d aese v17.16b, v13.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v8.16b, v13.16b aesmc v8.16b, v8.16b aese v9.16b, v13.16b aesmc v9.16b, v9.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d aese v10.16b, v13.16b aesmc v10.16b, v10.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #112] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ext v26.16b, v18.16b, v18.16b, #8 aese v15.16b, v12.16b aesmc v15.16b, v15.16b pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d aese v16.16b, v12.16b aesmc v16.16b, v16.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v17.16b, v12.16b aesmc v17.16b, v17.16b aese v8.16b, v12.16b aesmc v8.16b, v8.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v9.16b, v12.16b aesmc v9.16b, v9.16b pmull2 v29.1q, v29.2d, v27.2d aese v10.16b, v12.16b aesmc v10.16b, v10.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b subs w8, w8, #8 ldr q12, [x0, #128] aese v14.16b, v13.16b aesmc v14.16b, v14.16b pmull2 v30.1q, v31.2d, v27.2d aese v15.16b, v13.16b aesmc v15.16b, v15.16b mov v28.d[1], v31.d[0] aese v16.16b, v13.16b aesmc v16.16b, v16.16b eor v26.16b, v28.16b, v30.16b aese v17.16b, v13.16b aesmc v17.16b, v17.16b # Done GHASH aese v8.16b, v13.16b aesmc v8.16b, v8.16b ld1 {v18.16b}, [x3], #16 aese v9.16b, v13.16b aesmc v9.16b, v9.16b ld1 {v19.16b}, [x3], #16 aese v10.16b, v13.16b aesmc v10.16b, v10.16b ld1 {v20.16b}, [x3], #16 aese v11.16b, v13.16b aesmc v11.16b, v11.16b ldr q13, [x0, #144] aese v14.16b, v12.16b aesmc v14.16b, v14.16b ld1 {v21.16b}, [x3], #16 aese v15.16b, v12.16b aesmc v15.16b, v15.16b ld1 {v0.16b}, [x3], #16 aese v16.16b, v12.16b aesmc v16.16b, v16.16b ld1 {v1.16b}, [x3], #16 aese v17.16b, v12.16b aesmc v17.16b, v17.16b ld1 {v2.16b}, [x3], #16 aese v8.16b, v12.16b aesmc v8.16b, v8.16b ld1 {v3.16b}, [x3], #16 aese v9.16b, v12.16b aesmc v9.16b, v9.16b aese v10.16b, v12.16b aesmc v10.16b, v10.16b aese v11.16b, v12.16b aesmc v11.16b, v11.16b ldr q12, [x0, #160] aese v14.16b, v13.16b eor v14.16b, v14.16b, v12.16b aese v15.16b, v13.16b eor v15.16b, v15.16b, v12.16b aese v16.16b, v13.16b eor v16.16b, v16.16b, v12.16b aese v17.16b, v13.16b eor v17.16b, v17.16b, v12.16b aese v8.16b, v13.16b eor v8.16b, v8.16b, v12.16b aese v9.16b, v13.16b eor v9.16b, v9.16b, v12.16b aese v10.16b, v13.16b eor v10.16b, v10.16b, v12.16b aese v11.16b, v13.16b eor v11.16b, v11.16b, v12.16b ld1 {v13.2d}, [x7] eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_both_8 L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_end_8: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b rbit v0.16b, v0.16b rbit v1.16b, v1.16b rbit v2.16b, v2.16b rbit v3.16b, v3.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v3.1d, v22.1d pmull2 v29.1q, v3.2d, v22.2d ext v31.16b, v3.16b, v3.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v2.1d pmull2 v26.1q, v23.2d, v2.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v2.16b, v2.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v1.1d pmull2 v26.1q, v24.2d, v1.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v1.16b, v1.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v0.1d pmull2 v26.1q, v25.2d, v0.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v0.16b, v0.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^5 pmull v31.1q, v4.1d, v21.1d pmull2 v26.1q, v4.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v4.1d pmull2 v26.1q, v26.2d, v4.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^6 pmull v31.1q, v5.1d, v20.1d pmull2 v26.1q, v5.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v5.1d pmull2 v26.1q, v26.2d, v5.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^7 pmull v31.1q, v6.1d, v19.1d pmull2 v26.1q, v6.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v6.1d pmull2 v26.1q, v26.2d, v6.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^8 pmull v31.1q, v7.1d, v18.1d pmull2 v26.1q, v7.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v7.1d pmull2 v26.1q, v26.2d, v7.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_4: ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 ld1 {v8.2d, v9.2d}, [x0], #32 ld1 {v10.2d}, [x0] cmp w8, #1 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_done beq L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_1 cmp w8, #4 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_2 add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rev w13, w13 mov v14.s[3], w13 rev w12, w12 mov v15.s[3], w12 rev w11, w11 mov v16.s[3], w11 rev w10, w9 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v16.16b, v0.16b aesmc v16.16b, v16.16b aese v17.16b, v0.16b aesmc v17.16b, v17.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v16.16b, v1.16b aesmc v16.16b, v16.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v16.16b, v3.16b aesmc v16.16b, v16.16b aese v17.16b, v3.16b aesmc v17.16b, v17.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v16.16b, v4.16b aesmc v16.16b, v16.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v16.16b, v7.16b aesmc v16.16b, v16.16b ld1 {v18.16b}, [x3], #16 aese v17.16b, v7.16b aesmc v17.16b, v17.16b ld1 {v19.16b}, [x3], #16 aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b ld1 {v20.16b}, [x3], #16 aese v17.16b, v8.16b aesmc v17.16b, v17.16b ld1 {v21.16b}, [x3], #16 aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w8, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_end_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_both_4: add w13, w9, #1 mov v14.16b, v13.16b add w12, w9, #2 mov v15.16b, v13.16b add w11, w9, #3 mov v16.16b, v13.16b add w9, w9, #4 mov v17.16b, v13.16b rbit v18.16b, v18.16b rev w13, w13 rbit v19.16b, v19.16b mov v14.s[3], w13 rev w12, w12 rbit v20.16b, v20.16b mov v15.s[3], w12 rev w11, w11 rbit v21.16b, v21.16b mov v16.s[3], w11 rev w10, w9 mov v17.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b eor v18.16b, v18.16b, v26.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d aese v16.16b, v0.16b aesmc v16.16b, v16.16b ext v31.16b, v21.16b, v21.16b, #8 aese v17.16b, v0.16b aesmc v17.16b, v17.16b pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d aese v14.16b, v1.16b aesmc v14.16b, v14.16b eor v30.16b, v30.16b, v31.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d aese v16.16b, v1.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v1.16b aesmc v17.16b, v17.16b ext v26.16b, v20.16b, v20.16b, #8 aese v14.16b, v2.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d aese v15.16b, v2.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v2.16b aesmc v16.16b, v16.16b aese v17.16b, v2.16b aesmc v17.16b, v17.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d aese v14.16b, v3.16b aesmc v14.16b, v14.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b ext v26.16b, v19.16b, v19.16b, #8 aese v16.16b, v3.16b aesmc v16.16b, v16.16b pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d aese v17.16b, v3.16b aesmc v17.16b, v17.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d aese v16.16b, v4.16b aesmc v16.16b, v16.16b eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b aese v17.16b, v4.16b aesmc v17.16b, v17.16b ext v26.16b, v18.16b, v18.16b, #8 aese v14.16b, v5.16b aesmc v14.16b, v14.16b pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d aese v15.16b, v5.16b aesmc v15.16b, v15.16b eor3 v30.16b, v30.16b, v26.16b, v31.16b aese v16.16b, v5.16b aesmc v16.16b, v16.16b aese v17.16b, v5.16b aesmc v17.16b, v17.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 aese v14.16b, v6.16b aesmc v14.16b, v14.16b pmull2 v29.1q, v29.2d, v27.2d aese v15.16b, v6.16b aesmc v15.16b, v15.16b eor3 v31.16b, v31.16b, v29.16b, v30.16b aese v16.16b, v6.16b aesmc v16.16b, v16.16b pmull2 v30.1q, v31.2d, v27.2d aese v17.16b, v6.16b aesmc v17.16b, v17.16b subs w8, w8, #4 mov v28.d[1], v31.d[0] aese v14.16b, v7.16b aesmc v14.16b, v14.16b eor v26.16b, v28.16b, v30.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b # Done GHASH aese v16.16b, v7.16b aesmc v16.16b, v16.16b aese v17.16b, v7.16b aesmc v17.16b, v17.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v16.16b, v8.16b aesmc v16.16b, v16.16b aese v17.16b, v8.16b aesmc v17.16b, v17.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b aese v16.16b, v9.16b eor v16.16b, v16.16b, v10.16b aese v17.16b, v9.16b eor v17.16b, v17.16b, v10.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b eor v16.16b, v16.16b, v20.16b eor v17.16b, v17.16b, v21.16b cmp w8, #4 st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 bge L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_both_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_end_4: rbit v18.16b, v18.16b rbit v19.16b, v19.16b rbit v20.16b, v20.16b rbit v21.16b, v21.16b eor v18.16b, v18.16b, v26.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v20.1d pmull2 v26.1q, v23.2d, v20.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v20.16b, v20.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^3 pmull v31.1q, v24.1d, v19.1d pmull2 v26.1q, v24.2d, v19.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v19.16b, v19.16b, #8 pmull v31.1q, v26.1d, v24.1d pmull2 v26.1q, v26.2d, v24.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # X += C * H^4 pmull v31.1q, v25.1d, v18.1d pmull2 v26.1q, v25.2d, v18.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v18.16b, v18.16b, #8 pmull v31.1q, v26.1d, v25.1d pmull2 v26.1q, v26.2d, v25.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cmp w8, #1 beq L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_1 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_done L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_2: add w13, w9, #1 mov v14.16b, v13.16b add w9, w9, #2 mov v15.16b, v13.16b rev w13, w13 mov v14.s[3], w13 rev w10, w9 mov v15.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v15.16b, v0.16b aesmc v15.16b, v15.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v15.16b, v1.16b aesmc v15.16b, v15.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v15.16b, v2.16b aesmc v15.16b, v15.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v15.16b, v3.16b aesmc v15.16b, v15.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v15.16b, v4.16b aesmc v15.16b, v15.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v15.16b, v5.16b aesmc v15.16b, v15.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v15.16b, v6.16b aesmc v15.16b, v15.16b subs w8, w8, #2 aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v15.16b, v7.16b aesmc v15.16b, v15.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b ld1 {v18.16b}, [x3], #16 aese v15.16b, v8.16b aesmc v15.16b, v15.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v19.16b}, [x3], #16 aese v15.16b, v9.16b eor v15.16b, v15.16b, v10.16b eor v14.16b, v14.16b, v18.16b eor v15.16b, v15.16b, v19.16b st1 {v14.16b, v15.16b}, [x2], #32 rbit v18.16b, v18.16b rbit v19.16b, v19.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v19.1d, v22.1d pmull2 v29.1q, v19.2d, v22.2d ext v31.16b, v19.16b, v19.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # X += C * H^2 pmull v31.1q, v23.1d, v21.1d pmull2 v26.1q, v23.2d, v21.2d eor v28.16b, v28.16b, v31.16b eor v29.16b, v29.16b, v26.16b ext v26.16b, v21.16b, v21.16b, #8 pmull v31.1q, v26.1d, v23.1d pmull2 v26.1q, v26.2d, v23.2d eor3 v30.16b, v30.16b, v26.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH cbz w8, L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_done L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_1: add w9, w9, #1 mov v14.16b, v13.16b rev w10, w9 mov v14.s[3], w10 aese v14.16b, v0.16b aesmc v14.16b, v14.16b aese v14.16b, v1.16b aesmc v14.16b, v14.16b aese v14.16b, v2.16b aesmc v14.16b, v14.16b aese v14.16b, v3.16b aesmc v14.16b, v14.16b aese v14.16b, v4.16b aesmc v14.16b, v14.16b aese v14.16b, v5.16b aesmc v14.16b, v14.16b aese v14.16b, v6.16b aesmc v14.16b, v14.16b aese v14.16b, v7.16b aesmc v14.16b, v14.16b aese v14.16b, v8.16b aesmc v14.16b, v14.16b aese v14.16b, v9.16b eor v14.16b, v14.16b, v10.16b ld1 {v18.16b}, [x3], #16 eor v14.16b, v14.16b, v18.16b st1 {v14.16b}, [x2], #16 rbit v18.16b, v18.16b eor v21.16b, v26.16b, v18.16b # X = C * H^1 pmull v28.1q, v21.1d, v22.1d pmull2 v29.1q, v21.2d, v22.2d ext v31.16b, v21.16b, v21.16b, #8 pmull v30.1q, v31.1d, v22.1d pmull2 v31.1q, v31.2d, v22.2d eor v30.16b, v30.16b, v31.16b # Reduce ext v31.16b, v28.16b, v29.16b, #8 pmull2 v29.1q, v29.2d, v27.2d eor3 v31.16b, v31.16b, v29.16b, v30.16b pmull2 v30.1q, v31.2d, v27.2d mov v28.d[1], v31.d[0] eor v26.16b, v28.16b, v30.16b # Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_done: #endif /* !NO_AES_128 */ L_aes_gcm_decrypt_update_arm64_crypto_eor3_done: rev w9, w9 mov v13.s[3], w9 st1 {v26.2d}, [x5] st1 {v13.2d}, [x7] ldr x17, [x29, #24] ldp d8, d9, [x29, #32] ldp d10, d11, [x29, #48] ldp d12, d13, [x29, #64] ldp d14, d15, [x29, #80] ldp x29, x30, [sp], #0x60 ret #ifndef __APPLE__ .size AES_GCM_decrypt_update_AARCH64_EOR3,.-AES_GCM_decrypt_update_AARCH64_EOR3 #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_decrypt_final_AARCH64_EOR3 .type AES_GCM_decrypt_final_AARCH64_EOR3,@function .align 2 AES_GCM_decrypt_final_AARCH64_EOR3: #else .section __TEXT,__text .globl _AES_GCM_decrypt_final_AARCH64_EOR3 .p2align 2 _AES_GCM_decrypt_final_AARCH64_EOR3: #endif /* __APPLE__ */ ld1 {v5.2d}, [x0] movi v6.16b, #0x87 ld1 {v4.2d}, [x5] ushr v6.2d, v6.2d, #56 ld1 {v7.2d}, [x6] lsl x4, x4, #3 rbit x4, x4 mov v0.d[0], x4 lsl x3, x3, #3 rbit x3, x3 mov v0.d[1], x3 eor v5.16b, v5.16b, v0.16b pmull v0.1q, v5.1d, v4.1d pmull2 v1.1q, v5.2d, v4.2d ext v3.16b, v5.16b, v5.16b, #8 pmull v2.1q, v3.1d, v4.1d pmull2 v3.1q, v3.2d, v4.2d eor v2.16b, v2.16b, v3.16b # Reduce ext v3.16b, v0.16b, v1.16b, #8 pmull2 v1.1q, v1.2d, v6.2d eor3 v3.16b, v3.16b, v1.16b, v2.16b pmull2 v2.1q, v3.2d, v6.2d mov v0.d[1], v3.d[0] eor v5.16b, v0.16b, v2.16b rbit v5.16b, v5.16b eor v5.16b, v5.16b, v7.16b cmp w2, #16 blt L_aes_gcm_decrypt_final_arm64_crypto_eor3_part_tag ld1 {v0.16b}, [x1] b L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_loaded L_aes_gcm_decrypt_final_arm64_crypto_eor3_part_tag: eor v0.16b, v0.16b, v0.16b mov x10, x2 st1 {v0.2d}, [x0] cmp x10, #8 blt L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_start_dw ldr x9, [x1], #8 sub x10, x10, #8 str x9, [x0], #8 L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_start_dw: cmp x10, #4 blt L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_start_sw ldr w9, [x1], #4 sub x10, x10, #4 str w9, [x0], #4 L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_start_sw: cmp x10, #2 blt L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_start_byte ldrh w9, [x1], #2 sub x10, x10, #2 strh w9, [x0], #2 L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_start_byte: cbz x10, L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_end_bytes ldrb w9, [x1], #1 subs x10, x10, #1 strb w9, [x0], #1 bne L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_start_byte L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_end_bytes: sub x0, x0, x2 ld1 {v0.2d}, [x0] mov x10, #16 st1 {v5.2d}, [x0] sub x10, x10, x2 add x0, x0, x2 L_aes_gcm_decrypt_final_arm64_crypto_eor3_calc_tag_byte: strb wzr, [x0], #1 subs x10, x10, #1 bne L_aes_gcm_decrypt_final_arm64_crypto_eor3_calc_tag_byte subs x0, x0, #16 ld1 {v5.2d}, [x0] L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_loaded: eor v0.16b, v0.16b, v5.16b mov x9, v0.d[0] mov x10, v0.d[1] mov w11, #-180 orr x9, x9, x10 cmp x9, #0 csetm x8, ne and x8, x8, x11 add w8, w8, #0xb4 str w8, [x7] ret #ifndef __APPLE__ .size AES_GCM_decrypt_final_AARCH64_EOR3,.-AES_GCM_decrypt_final_AARCH64_EOR3 #endif /* __APPLE__ */ #endif /* !WOLFSSL_ARMASM_CRYPTO_SHA3 */ #endif /* WOLFSSL_AESGCM_STREAM */ #endif /* HAVE_AESGCM */ #ifdef WOLFSSL_AES_XTS #ifndef __APPLE__ .text .globl AES_XTS_encrypt_AARCH64 .type AES_XTS_encrypt_AARCH64,@function .align 2 AES_XTS_encrypt_AARCH64: #else .section __TEXT,__text .globl _AES_XTS_encrypt_AARCH64 .p2align 2 _AES_XTS_encrypt_AARCH64: #endif /* __APPLE__ */ stp x29, x30, [sp, #-48]! add x29, sp, #0 stp x17, x19, [x29, #24] ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x5], #0x40 ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x5], #0x40 ld1 {v4.16b}, [x3] lsr w8, w2, #4 and w2, w2, #15 mov x19, #0x87 cmp x7, #12 blt L_aes_xts_encrypt_arm64_crypto_start_128 bgt L_aes_xts_encrypt_arm64_crypto_start_256 # AES_XTS_192 #ifndef NO_AES_192 ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x5], #0x40 ld1 {v28.2d}, [x5] aese v4.16b, v16.16b aesmc v4.16b, v4.16b aese v4.16b, v17.16b aesmc v4.16b, v4.16b aese v4.16b, v18.16b aesmc v4.16b, v4.16b aese v4.16b, v19.16b aesmc v4.16b, v4.16b aese v4.16b, v20.16b aesmc v4.16b, v4.16b aese v4.16b, v21.16b aesmc v4.16b, v4.16b aese v4.16b, v22.16b aesmc v4.16b, v4.16b aese v4.16b, v23.16b aesmc v4.16b, v4.16b aese v4.16b, v24.16b aesmc v4.16b, v4.16b aese v4.16b, v25.16b aesmc v4.16b, v4.16b aese v4.16b, v26.16b aesmc v4.16b, v4.16b aese v4.16b, v27.16b eor v4.16b, v4.16b, v28.16b mov x10, v4.d[0] mov x11, v4.d[1] ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 ld1 {v28.2d}, [x4] and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 and x9, x19, x13, asr 63 extr x15, x13, x12, #63 eor x14, x9, x12, lsl 1 and x9, x19, x15, asr 63 extr x17, x15, x14, #63 eor x16, x9, x14, lsl 1 cmp w8, #4 blt L_aes_xts_encrypt_arm64_crypto_192_start_2 L_aes_xts_encrypt_arm64_crypto_192_start_4: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 mov v5.d[0], x12 mov v5.d[1], x13 mov v6.d[0], x14 mov v6.d[1], x15 mov v7.d[0], x16 mov v7.d[1], x17 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b and x9, x19, x17, asr 63 aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v2.16b, v16.16b aesmc v2.16b, v2.16b extr x11, x17, x16, #63 aese v3.16b, v16.16b aesmc v3.16b, v3.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b eor x10, x9, x16, lsl 1 aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v2.16b, v17.16b aesmc v2.16b, v2.16b and x9, x19, x11, asr 63 aese v3.16b, v17.16b aesmc v3.16b, v3.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b extr x13, x11, x10, #63 aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v2.16b, v18.16b aesmc v2.16b, v2.16b eor x12, x9, x10, lsl 1 aese v3.16b, v18.16b aesmc v3.16b, v3.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b and x9, x19, x13, asr 63 aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v2.16b, v19.16b aesmc v2.16b, v2.16b extr x15, x13, x12, #63 aese v3.16b, v19.16b aesmc v3.16b, v3.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b eor x14, x9, x12, lsl 1 aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v2.16b, v20.16b aesmc v2.16b, v2.16b and x9, x19, x15, asr 63 aese v3.16b, v20.16b aesmc v3.16b, v3.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b extr x17, x15, x14, #63 aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v2.16b, v21.16b aesmc v2.16b, v2.16b eor x16, x9, x14, lsl 1 aese v3.16b, v21.16b aesmc v3.16b, v3.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v2.16b, v22.16b aesmc v2.16b, v2.16b aese v3.16b, v22.16b aesmc v3.16b, v3.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v2.16b, v23.16b aesmc v2.16b, v2.16b aese v3.16b, v23.16b aesmc v3.16b, v3.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v2.16b, v24.16b aesmc v2.16b, v2.16b aese v3.16b, v24.16b aesmc v3.16b, v3.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v1.16b, v25.16b aesmc v1.16b, v1.16b aese v2.16b, v25.16b aesmc v2.16b, v2.16b aese v3.16b, v25.16b aesmc v3.16b, v3.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v1.16b, v26.16b aesmc v1.16b, v1.16b aese v2.16b, v26.16b aesmc v2.16b, v2.16b aese v3.16b, v26.16b aesmc v3.16b, v3.16b aese v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b aese v1.16b, v27.16b eor v1.16b, v1.16b, v28.16b aese v2.16b, v27.16b eor v2.16b, v2.16b, v28.16b aese v3.16b, v27.16b eor v3.16b, v3.16b, v28.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b mov v4.d[0], x10 mov v4.d[1], x11 sub w8, w8, #4 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w8, #4 bge L_aes_xts_encrypt_arm64_crypto_192_start_4 L_aes_xts_encrypt_arm64_crypto_192_start_2: cmp w8, #2 blt L_aes_xts_encrypt_arm64_crypto_192_start_1 ld1 {v0.16b, v1.16b}, [x0], #32 mov v5.d[0], x12 mov v5.d[1], x13 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b and x9, x19, x13, asr 63 aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b extr x11, x13, x12, #63 aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b eor x10, x9, x12, lsl 1 aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b and x9, x19, x11, asr 63 aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b extr x13, x11, x10, #63 aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b eor x12, x9, x10, lsl 1 aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v1.16b, v25.16b aesmc v1.16b, v1.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v1.16b, v26.16b aesmc v1.16b, v1.16b aese v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b aese v1.16b, v27.16b eor v1.16b, v1.16b, v28.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b mov v4.d[0], x10 mov v4.d[1], x11 sub w8, w8, #2 st1 {v0.16b, v1.16b}, [x1], #32 L_aes_xts_encrypt_arm64_crypto_192_start_1: cbz w8, L_aes_xts_encrypt_arm64_crypto_192_done ld1 {v0.16b}, [x0], #16 eor v0.16b, v0.16b, v4.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b and x9, x19, x11, asr 63 aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b extr x11, x11, x10, #63 aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b eor x10, x9, x10, lsl 1 aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b eor v0.16b, v0.16b, v4.16b mov v4.d[0], x10 mov v4.d[1], x11 st1 {v0.16b}, [x1], #16 L_aes_xts_encrypt_arm64_crypto_192_done: cbz w2, L_aes_xts_encrypt_arm64_crypto_192_partial_done sub x1, x1, #16 ld1 {v0.16b}, [x1], #16 st1 {v0.2d}, [x6] mov w9, w2 L_aes_xts_encrypt_arm64_crypto_192_start_byte: ldrb w12, [x6] ldrb w13, [x0], #1 strb w12, [x1], #1 strb w13, [x6], #1 subs w9, w9, #1 bgt L_aes_xts_encrypt_arm64_crypto_192_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 ld1 {v0.2d}, [x6] eor v0.16b, v0.16b, v4.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b eor v0.16b, v0.16b, v4.16b st1 {v0.16b}, [x1] L_aes_xts_encrypt_arm64_crypto_192_partial_done: #endif /* !NO_AES_192 */ b L_aes_xts_encrypt_arm64_crypto_done # AES_XTS_256 L_aes_xts_encrypt_arm64_crypto_start_256: #ifndef NO_AES_256 ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x5], #0x40 ld1 {v28.2d, v29.2d}, [x5], #32 ld1 {v30.2d}, [x5] aese v4.16b, v16.16b aesmc v4.16b, v4.16b aese v4.16b, v17.16b aesmc v4.16b, v4.16b aese v4.16b, v18.16b aesmc v4.16b, v4.16b aese v4.16b, v19.16b aesmc v4.16b, v4.16b aese v4.16b, v20.16b aesmc v4.16b, v4.16b aese v4.16b, v21.16b aesmc v4.16b, v4.16b aese v4.16b, v22.16b aesmc v4.16b, v4.16b aese v4.16b, v23.16b aesmc v4.16b, v4.16b aese v4.16b, v24.16b aesmc v4.16b, v4.16b aese v4.16b, v25.16b aesmc v4.16b, v4.16b aese v4.16b, v26.16b aesmc v4.16b, v4.16b aese v4.16b, v27.16b aesmc v4.16b, v4.16b aese v4.16b, v28.16b aesmc v4.16b, v4.16b aese v4.16b, v29.16b eor v4.16b, v4.16b, v30.16b mov x10, v4.d[0] mov x11, v4.d[1] ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 ld1 {v28.2d, v29.2d}, [x4], #32 ld1 {v30.2d}, [x4] and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 and x9, x19, x13, asr 63 extr x15, x13, x12, #63 eor x14, x9, x12, lsl 1 and x9, x19, x15, asr 63 extr x17, x15, x14, #63 eor x16, x9, x14, lsl 1 cmp w8, #4 blt L_aes_xts_encrypt_arm64_crypto_256_start_2 L_aes_xts_encrypt_arm64_crypto_256_start_4: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 mov v5.d[0], x12 mov v5.d[1], x13 mov v6.d[0], x14 mov v6.d[1], x15 mov v7.d[0], x16 mov v7.d[1], x17 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b and x9, x19, x17, asr 63 aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v2.16b, v16.16b aesmc v2.16b, v2.16b extr x11, x17, x16, #63 aese v3.16b, v16.16b aesmc v3.16b, v3.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b eor x10, x9, x16, lsl 1 aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v2.16b, v17.16b aesmc v2.16b, v2.16b and x9, x19, x11, asr 63 aese v3.16b, v17.16b aesmc v3.16b, v3.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b extr x13, x11, x10, #63 aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v2.16b, v18.16b aesmc v2.16b, v2.16b eor x12, x9, x10, lsl 1 aese v3.16b, v18.16b aesmc v3.16b, v3.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b and x9, x19, x13, asr 63 aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v2.16b, v19.16b aesmc v2.16b, v2.16b extr x15, x13, x12, #63 aese v3.16b, v19.16b aesmc v3.16b, v3.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b eor x14, x9, x12, lsl 1 aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v2.16b, v20.16b aesmc v2.16b, v2.16b and x9, x19, x15, asr 63 aese v3.16b, v20.16b aesmc v3.16b, v3.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b extr x17, x15, x14, #63 aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v2.16b, v21.16b aesmc v2.16b, v2.16b eor x16, x9, x14, lsl 1 aese v3.16b, v21.16b aesmc v3.16b, v3.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v2.16b, v22.16b aesmc v2.16b, v2.16b aese v3.16b, v22.16b aesmc v3.16b, v3.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v2.16b, v23.16b aesmc v2.16b, v2.16b aese v3.16b, v23.16b aesmc v3.16b, v3.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v2.16b, v24.16b aesmc v2.16b, v2.16b aese v3.16b, v24.16b aesmc v3.16b, v3.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v1.16b, v25.16b aesmc v1.16b, v1.16b aese v2.16b, v25.16b aesmc v2.16b, v2.16b aese v3.16b, v25.16b aesmc v3.16b, v3.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v1.16b, v26.16b aesmc v1.16b, v1.16b aese v2.16b, v26.16b aesmc v2.16b, v2.16b aese v3.16b, v26.16b aesmc v3.16b, v3.16b aese v0.16b, v27.16b aesmc v0.16b, v0.16b aese v1.16b, v27.16b aesmc v1.16b, v1.16b aese v2.16b, v27.16b aesmc v2.16b, v2.16b aese v3.16b, v27.16b aesmc v3.16b, v3.16b aese v0.16b, v28.16b aesmc v0.16b, v0.16b aese v1.16b, v28.16b aesmc v1.16b, v1.16b aese v2.16b, v28.16b aesmc v2.16b, v2.16b aese v3.16b, v28.16b aesmc v3.16b, v3.16b aese v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b aese v1.16b, v29.16b eor v1.16b, v1.16b, v30.16b aese v2.16b, v29.16b eor v2.16b, v2.16b, v30.16b aese v3.16b, v29.16b eor v3.16b, v3.16b, v30.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b mov v4.d[0], x10 mov v4.d[1], x11 sub w8, w8, #4 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w8, #4 bge L_aes_xts_encrypt_arm64_crypto_256_start_4 L_aes_xts_encrypt_arm64_crypto_256_start_2: cmp w8, #2 blt L_aes_xts_encrypt_arm64_crypto_256_start_1 ld1 {v0.16b, v1.16b}, [x0], #32 mov v5.d[0], x12 mov v5.d[1], x13 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b and x9, x19, x13, asr 63 aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b extr x11, x13, x12, #63 aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b eor x10, x9, x12, lsl 1 aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b and x9, x19, x11, asr 63 aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b extr x13, x11, x10, #63 aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b eor x12, x9, x10, lsl 1 aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v1.16b, v25.16b aesmc v1.16b, v1.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v1.16b, v26.16b aesmc v1.16b, v1.16b aese v0.16b, v27.16b aesmc v0.16b, v0.16b aese v1.16b, v27.16b aesmc v1.16b, v1.16b aese v0.16b, v28.16b aesmc v0.16b, v0.16b aese v1.16b, v28.16b aesmc v1.16b, v1.16b aese v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b aese v1.16b, v29.16b eor v1.16b, v1.16b, v30.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b mov v4.d[0], x10 mov v4.d[1], x11 sub w8, w8, #2 st1 {v0.16b, v1.16b}, [x1], #32 L_aes_xts_encrypt_arm64_crypto_256_start_1: cbz w8, L_aes_xts_encrypt_arm64_crypto_256_done ld1 {v0.16b}, [x0], #16 eor v0.16b, v0.16b, v4.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b and x9, x19, x11, asr 63 aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b extr x11, x11, x10, #63 aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b eor x10, x9, x10, lsl 1 aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v0.16b, v27.16b aesmc v0.16b, v0.16b aese v0.16b, v28.16b aesmc v0.16b, v0.16b aese v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b eor v0.16b, v0.16b, v4.16b mov v4.d[0], x10 mov v4.d[1], x11 st1 {v0.16b}, [x1], #16 L_aes_xts_encrypt_arm64_crypto_256_done: cbz w2, L_aes_xts_encrypt_arm64_crypto_256_partial_done sub x1, x1, #16 ld1 {v0.16b}, [x1], #16 st1 {v0.2d}, [x6] mov w9, w2 L_aes_xts_encrypt_arm64_crypto_256_start_byte: ldrb w12, [x6] ldrb w13, [x0], #1 strb w12, [x1], #1 strb w13, [x6], #1 subs w9, w9, #1 bgt L_aes_xts_encrypt_arm64_crypto_256_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 ld1 {v0.2d}, [x6] eor v0.16b, v0.16b, v4.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v0.16b, v25.16b aesmc v0.16b, v0.16b aese v0.16b, v26.16b aesmc v0.16b, v0.16b aese v0.16b, v27.16b aesmc v0.16b, v0.16b aese v0.16b, v28.16b aesmc v0.16b, v0.16b aese v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b eor v0.16b, v0.16b, v4.16b st1 {v0.16b}, [x1] L_aes_xts_encrypt_arm64_crypto_256_partial_done: #endif /* !NO_AES_256 */ b L_aes_xts_encrypt_arm64_crypto_done # AES_XTS_128 L_aes_xts_encrypt_arm64_crypto_start_128: #ifndef NO_AES_128 ld1 {v24.2d, v25.2d}, [x5], #32 ld1 {v26.2d}, [x5] aese v4.16b, v16.16b aesmc v4.16b, v4.16b aese v4.16b, v17.16b aesmc v4.16b, v4.16b aese v4.16b, v18.16b aesmc v4.16b, v4.16b aese v4.16b, v19.16b aesmc v4.16b, v4.16b aese v4.16b, v20.16b aesmc v4.16b, v4.16b aese v4.16b, v21.16b aesmc v4.16b, v4.16b aese v4.16b, v22.16b aesmc v4.16b, v4.16b aese v4.16b, v23.16b aesmc v4.16b, v4.16b aese v4.16b, v24.16b aesmc v4.16b, v4.16b aese v4.16b, v25.16b eor v4.16b, v4.16b, v26.16b mov x10, v4.d[0] mov x11, v4.d[1] ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 ld1 {v24.2d, v25.2d}, [x4], #32 ld1 {v26.2d}, [x4] and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 and x9, x19, x13, asr 63 extr x15, x13, x12, #63 eor x14, x9, x12, lsl 1 and x9, x19, x15, asr 63 extr x17, x15, x14, #63 eor x16, x9, x14, lsl 1 cmp w8, #4 blt L_aes_xts_encrypt_arm64_crypto_128_start_2 L_aes_xts_encrypt_arm64_crypto_128_start_4: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 mov v5.d[0], x12 mov v5.d[1], x13 mov v6.d[0], x14 mov v6.d[1], x15 mov v7.d[0], x16 mov v7.d[1], x17 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b and x9, x19, x17, asr 63 aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v2.16b, v16.16b aesmc v2.16b, v2.16b extr x11, x17, x16, #63 aese v3.16b, v16.16b aesmc v3.16b, v3.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b eor x10, x9, x16, lsl 1 aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v2.16b, v17.16b aesmc v2.16b, v2.16b and x9, x19, x11, asr 63 aese v3.16b, v17.16b aesmc v3.16b, v3.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b extr x13, x11, x10, #63 aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v2.16b, v18.16b aesmc v2.16b, v2.16b eor x12, x9, x10, lsl 1 aese v3.16b, v18.16b aesmc v3.16b, v3.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b and x9, x19, x13, asr 63 aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v2.16b, v19.16b aesmc v2.16b, v2.16b extr x15, x13, x12, #63 aese v3.16b, v19.16b aesmc v3.16b, v3.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b eor x14, x9, x12, lsl 1 aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v2.16b, v20.16b aesmc v2.16b, v2.16b and x9, x19, x15, asr 63 aese v3.16b, v20.16b aesmc v3.16b, v3.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b extr x17, x15, x14, #63 aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v2.16b, v21.16b aesmc v2.16b, v2.16b eor x16, x9, x14, lsl 1 aese v3.16b, v21.16b aesmc v3.16b, v3.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v2.16b, v22.16b aesmc v2.16b, v2.16b aese v3.16b, v22.16b aesmc v3.16b, v3.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v2.16b, v23.16b aesmc v2.16b, v2.16b aese v3.16b, v23.16b aesmc v3.16b, v3.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v2.16b, v24.16b aesmc v2.16b, v2.16b aese v3.16b, v24.16b aesmc v3.16b, v3.16b aese v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b aese v1.16b, v25.16b eor v1.16b, v1.16b, v26.16b aese v2.16b, v25.16b eor v2.16b, v2.16b, v26.16b aese v3.16b, v25.16b eor v3.16b, v3.16b, v26.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b mov v4.d[0], x10 mov v4.d[1], x11 sub w8, w8, #4 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w8, #4 bge L_aes_xts_encrypt_arm64_crypto_128_start_4 L_aes_xts_encrypt_arm64_crypto_128_start_2: cmp w8, #2 blt L_aes_xts_encrypt_arm64_crypto_128_start_1 ld1 {v0.16b, v1.16b}, [x0], #32 mov v5.d[0], x12 mov v5.d[1], x13 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b and x9, x19, x13, asr 63 aese v1.16b, v16.16b aesmc v1.16b, v1.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b extr x11, x13, x12, #63 aese v1.16b, v17.16b aesmc v1.16b, v1.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b eor x10, x9, x12, lsl 1 aese v1.16b, v18.16b aesmc v1.16b, v1.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b and x9, x19, x11, asr 63 aese v1.16b, v19.16b aesmc v1.16b, v1.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b extr x13, x11, x10, #63 aese v1.16b, v20.16b aesmc v1.16b, v1.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b eor x12, x9, x10, lsl 1 aese v1.16b, v21.16b aesmc v1.16b, v1.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v1.16b, v22.16b aesmc v1.16b, v1.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v1.16b, v23.16b aesmc v1.16b, v1.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v1.16b, v24.16b aesmc v1.16b, v1.16b aese v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b aese v1.16b, v25.16b eor v1.16b, v1.16b, v26.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b mov v4.d[0], x10 mov v4.d[1], x11 sub w8, w8, #2 st1 {v0.16b, v1.16b}, [x1], #32 L_aes_xts_encrypt_arm64_crypto_128_start_1: cbz w8, L_aes_xts_encrypt_arm64_crypto_128_done ld1 {v0.16b}, [x0], #16 eor v0.16b, v0.16b, v4.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b and x9, x19, x11, asr 63 aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b extr x11, x11, x10, #63 aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b eor x10, x9, x10, lsl 1 aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b eor v0.16b, v0.16b, v4.16b mov v4.d[0], x10 mov v4.d[1], x11 st1 {v0.16b}, [x1], #16 L_aes_xts_encrypt_arm64_crypto_128_done: cbz w2, L_aes_xts_encrypt_arm64_crypto_128_partial_done sub x1, x1, #16 ld1 {v0.16b}, [x1], #16 st1 {v0.2d}, [x6] mov w9, w2 L_aes_xts_encrypt_arm64_crypto_128_start_byte: ldrb w12, [x6] ldrb w13, [x0], #1 strb w12, [x1], #1 strb w13, [x6], #1 subs w9, w9, #1 bgt L_aes_xts_encrypt_arm64_crypto_128_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 ld1 {v0.2d}, [x6] eor v0.16b, v0.16b, v4.16b aese v0.16b, v16.16b aesmc v0.16b, v0.16b aese v0.16b, v17.16b aesmc v0.16b, v0.16b aese v0.16b, v18.16b aesmc v0.16b, v0.16b aese v0.16b, v19.16b aesmc v0.16b, v0.16b aese v0.16b, v20.16b aesmc v0.16b, v0.16b aese v0.16b, v21.16b aesmc v0.16b, v0.16b aese v0.16b, v22.16b aesmc v0.16b, v0.16b aese v0.16b, v23.16b aesmc v0.16b, v0.16b aese v0.16b, v24.16b aesmc v0.16b, v0.16b aese v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b eor v0.16b, v0.16b, v4.16b st1 {v0.16b}, [x1] L_aes_xts_encrypt_arm64_crypto_128_partial_done: #endif /* !NO_AES_128 */ L_aes_xts_encrypt_arm64_crypto_done: ldp x17, x19, [x29, #24] ldp x29, x30, [sp], #48 ret #ifndef __APPLE__ .size AES_XTS_encrypt_AARCH64,.-AES_XTS_encrypt_AARCH64 #endif /* __APPLE__ */ #ifdef HAVE_AES_DECRYPT #ifndef __APPLE__ .text .globl AES_XTS_decrypt_AARCH64 .type AES_XTS_decrypt_AARCH64,@function .align 2 AES_XTS_decrypt_AARCH64: #else .section __TEXT,__text .globl _AES_XTS_decrypt_AARCH64 .p2align 2 _AES_XTS_decrypt_AARCH64: #endif /* __APPLE__ */ stp x29, x30, [sp, #-48]! add x29, sp, #0 stp x17, x19, [x29, #24] ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x5], #0x40 ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x5], #0x40 ld1 {v4.16b}, [x3] lsr w8, w2, #4 ands w2, w2, #15 mov x19, #0x87 cset w9, ne sub w8, w8, w9 cmp x7, #12 blt L_aes_xts_decrypt_arm64_crypto_start_128 bgt L_aes_xts_decrypt_arm64_crypto_start_256 # AES_XTS_192 #ifndef NO_AES_192 ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x5], #0x40 ld1 {v28.2d}, [x5] aese v4.16b, v16.16b aesmc v4.16b, v4.16b aese v4.16b, v17.16b aesmc v4.16b, v4.16b aese v4.16b, v18.16b aesmc v4.16b, v4.16b aese v4.16b, v19.16b aesmc v4.16b, v4.16b aese v4.16b, v20.16b aesmc v4.16b, v4.16b aese v4.16b, v21.16b aesmc v4.16b, v4.16b aese v4.16b, v22.16b aesmc v4.16b, v4.16b aese v4.16b, v23.16b aesmc v4.16b, v4.16b aese v4.16b, v24.16b aesmc v4.16b, v4.16b aese v4.16b, v25.16b aesmc v4.16b, v4.16b aese v4.16b, v26.16b aesmc v4.16b, v4.16b aese v4.16b, v27.16b eor v4.16b, v4.16b, v28.16b mov x10, v4.d[0] mov x11, v4.d[1] ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 ld1 {v28.2d}, [x4] and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 and x9, x19, x13, asr 63 extr x15, x13, x12, #63 eor x14, x9, x12, lsl 1 and x9, x19, x15, asr 63 extr x17, x15, x14, #63 eor x16, x9, x14, lsl 1 cmp w8, #4 blt L_aes_xts_decrypt_arm64_crypto_192_start_2 L_aes_xts_decrypt_arm64_crypto_192_start_4: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 mov v5.d[0], x12 mov v5.d[1], x13 mov v6.d[0], x14 mov v6.d[1], x15 mov v7.d[0], x16 mov v7.d[1], x17 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b and x9, x19, x17, asr 63 aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v2.16b, v16.16b aesimc v2.16b, v2.16b extr x11, x17, x16, #63 aesd v3.16b, v16.16b aesimc v3.16b, v3.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b eor x10, x9, x16, lsl 1 aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v2.16b, v17.16b aesimc v2.16b, v2.16b and x9, x19, x11, asr 63 aesd v3.16b, v17.16b aesimc v3.16b, v3.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b extr x13, x11, x10, #63 aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v2.16b, v18.16b aesimc v2.16b, v2.16b eor x12, x9, x10, lsl 1 aesd v3.16b, v18.16b aesimc v3.16b, v3.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b and x9, x19, x13, asr 63 aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v2.16b, v19.16b aesimc v2.16b, v2.16b extr x15, x13, x12, #63 aesd v3.16b, v19.16b aesimc v3.16b, v3.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b eor x14, x9, x12, lsl 1 aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v2.16b, v20.16b aesimc v2.16b, v2.16b and x9, x19, x15, asr 63 aesd v3.16b, v20.16b aesimc v3.16b, v3.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b extr x17, x15, x14, #63 aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v2.16b, v21.16b aesimc v2.16b, v2.16b eor x16, x9, x14, lsl 1 aesd v3.16b, v21.16b aesimc v3.16b, v3.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v2.16b, v22.16b aesimc v2.16b, v2.16b aesd v3.16b, v22.16b aesimc v3.16b, v3.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v2.16b, v23.16b aesimc v2.16b, v2.16b aesd v3.16b, v23.16b aesimc v3.16b, v3.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v2.16b, v24.16b aesimc v2.16b, v2.16b aesd v3.16b, v24.16b aesimc v3.16b, v3.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v1.16b, v25.16b aesimc v1.16b, v1.16b aesd v2.16b, v25.16b aesimc v2.16b, v2.16b aesd v3.16b, v25.16b aesimc v3.16b, v3.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v1.16b, v26.16b aesimc v1.16b, v1.16b aesd v2.16b, v26.16b aesimc v2.16b, v2.16b aesd v3.16b, v26.16b aesimc v3.16b, v3.16b aesd v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b aesd v1.16b, v27.16b eor v1.16b, v1.16b, v28.16b aesd v2.16b, v27.16b eor v2.16b, v2.16b, v28.16b aesd v3.16b, v27.16b eor v3.16b, v3.16b, v28.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b mov v4.d[0], x10 mov v4.d[1], x11 sub w8, w8, #4 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w8, #4 bge L_aes_xts_decrypt_arm64_crypto_192_start_4 L_aes_xts_decrypt_arm64_crypto_192_start_2: cmp w8, #2 blt L_aes_xts_decrypt_arm64_crypto_192_start_1 ld1 {v0.16b, v1.16b}, [x0], #32 mov v5.d[0], x12 mov v5.d[1], x13 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b and x9, x19, x13, asr 63 aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b extr x11, x13, x12, #63 aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b eor x10, x9, x12, lsl 1 aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b and x9, x19, x11, asr 63 aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b extr x13, x11, x10, #63 aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b eor x12, x9, x10, lsl 1 aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v1.16b, v25.16b aesimc v1.16b, v1.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v1.16b, v26.16b aesimc v1.16b, v1.16b aesd v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b aesd v1.16b, v27.16b eor v1.16b, v1.16b, v28.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b mov v4.d[0], x10 mov v4.d[1], x11 sub w8, w8, #2 st1 {v0.16b, v1.16b}, [x1], #32 L_aes_xts_decrypt_arm64_crypto_192_start_1: cbz w8, L_aes_xts_decrypt_arm64_crypto_192_done ld1 {v0.16b}, [x0], #16 eor v0.16b, v0.16b, v4.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b and x9, x19, x11, asr 63 aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b extr x11, x11, x10, #63 aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b eor x10, x9, x10, lsl 1 aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b eor v0.16b, v0.16b, v4.16b mov v4.d[0], x10 mov v4.d[1], x11 st1 {v0.16b}, [x1], #16 L_aes_xts_decrypt_arm64_crypto_192_done: cbz w2, L_aes_xts_decrypt_arm64_crypto_192_partial_done and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 mov v5.d[0], x12 mov v5.d[1], x13 ld1 {v0.16b}, [x0], #16 eor v0.16b, v0.16b, v5.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b eor v0.16b, v0.16b, v5.16b st1 {v0.2d}, [x6] add x1, x1, #16 mov w9, w2 L_aes_xts_decrypt_arm64_crypto_192_start_byte: ldrb w12, [x6] ldrb w13, [x0], #1 strb w12, [x1], #1 strb w13, [x6], #1 subs w9, w9, #1 bgt L_aes_xts_decrypt_arm64_crypto_192_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 ld1 {v0.2d}, [x6] eor v0.16b, v0.16b, v4.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v0.16b, v27.16b eor v0.16b, v0.16b, v28.16b eor v0.16b, v0.16b, v4.16b st1 {v0.16b}, [x1] L_aes_xts_decrypt_arm64_crypto_192_partial_done: #endif /* !NO_AES_192 */ b L_aes_xts_decrypt_arm64_crypto_done # AES_XTS_256 L_aes_xts_decrypt_arm64_crypto_start_256: #ifndef NO_AES_256 ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x5], #0x40 ld1 {v28.2d, v29.2d}, [x5], #32 ld1 {v30.2d}, [x5] aese v4.16b, v16.16b aesmc v4.16b, v4.16b aese v4.16b, v17.16b aesmc v4.16b, v4.16b aese v4.16b, v18.16b aesmc v4.16b, v4.16b aese v4.16b, v19.16b aesmc v4.16b, v4.16b aese v4.16b, v20.16b aesmc v4.16b, v4.16b aese v4.16b, v21.16b aesmc v4.16b, v4.16b aese v4.16b, v22.16b aesmc v4.16b, v4.16b aese v4.16b, v23.16b aesmc v4.16b, v4.16b aese v4.16b, v24.16b aesmc v4.16b, v4.16b aese v4.16b, v25.16b aesmc v4.16b, v4.16b aese v4.16b, v26.16b aesmc v4.16b, v4.16b aese v4.16b, v27.16b aesmc v4.16b, v4.16b aese v4.16b, v28.16b aesmc v4.16b, v4.16b aese v4.16b, v29.16b eor v4.16b, v4.16b, v30.16b mov x10, v4.d[0] mov x11, v4.d[1] ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 ld1 {v28.2d, v29.2d}, [x4], #32 ld1 {v30.2d}, [x4] and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 and x9, x19, x13, asr 63 extr x15, x13, x12, #63 eor x14, x9, x12, lsl 1 and x9, x19, x15, asr 63 extr x17, x15, x14, #63 eor x16, x9, x14, lsl 1 cmp w8, #4 blt L_aes_xts_decrypt_arm64_crypto_256_start_2 L_aes_xts_decrypt_arm64_crypto_256_start_4: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 mov v5.d[0], x12 mov v5.d[1], x13 mov v6.d[0], x14 mov v6.d[1], x15 mov v7.d[0], x16 mov v7.d[1], x17 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b and x9, x19, x17, asr 63 aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v2.16b, v16.16b aesimc v2.16b, v2.16b extr x11, x17, x16, #63 aesd v3.16b, v16.16b aesimc v3.16b, v3.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b eor x10, x9, x16, lsl 1 aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v2.16b, v17.16b aesimc v2.16b, v2.16b and x9, x19, x11, asr 63 aesd v3.16b, v17.16b aesimc v3.16b, v3.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b extr x13, x11, x10, #63 aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v2.16b, v18.16b aesimc v2.16b, v2.16b eor x12, x9, x10, lsl 1 aesd v3.16b, v18.16b aesimc v3.16b, v3.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b and x9, x19, x13, asr 63 aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v2.16b, v19.16b aesimc v2.16b, v2.16b extr x15, x13, x12, #63 aesd v3.16b, v19.16b aesimc v3.16b, v3.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b eor x14, x9, x12, lsl 1 aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v2.16b, v20.16b aesimc v2.16b, v2.16b and x9, x19, x15, asr 63 aesd v3.16b, v20.16b aesimc v3.16b, v3.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b extr x17, x15, x14, #63 aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v2.16b, v21.16b aesimc v2.16b, v2.16b eor x16, x9, x14, lsl 1 aesd v3.16b, v21.16b aesimc v3.16b, v3.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v2.16b, v22.16b aesimc v2.16b, v2.16b aesd v3.16b, v22.16b aesimc v3.16b, v3.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v2.16b, v23.16b aesimc v2.16b, v2.16b aesd v3.16b, v23.16b aesimc v3.16b, v3.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v2.16b, v24.16b aesimc v2.16b, v2.16b aesd v3.16b, v24.16b aesimc v3.16b, v3.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v1.16b, v25.16b aesimc v1.16b, v1.16b aesd v2.16b, v25.16b aesimc v2.16b, v2.16b aesd v3.16b, v25.16b aesimc v3.16b, v3.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v1.16b, v26.16b aesimc v1.16b, v1.16b aesd v2.16b, v26.16b aesimc v2.16b, v2.16b aesd v3.16b, v26.16b aesimc v3.16b, v3.16b aesd v0.16b, v27.16b aesimc v0.16b, v0.16b aesd v1.16b, v27.16b aesimc v1.16b, v1.16b aesd v2.16b, v27.16b aesimc v2.16b, v2.16b aesd v3.16b, v27.16b aesimc v3.16b, v3.16b aesd v0.16b, v28.16b aesimc v0.16b, v0.16b aesd v1.16b, v28.16b aesimc v1.16b, v1.16b aesd v2.16b, v28.16b aesimc v2.16b, v2.16b aesd v3.16b, v28.16b aesimc v3.16b, v3.16b aesd v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b aesd v1.16b, v29.16b eor v1.16b, v1.16b, v30.16b aesd v2.16b, v29.16b eor v2.16b, v2.16b, v30.16b aesd v3.16b, v29.16b eor v3.16b, v3.16b, v30.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b mov v4.d[0], x10 mov v4.d[1], x11 sub w8, w8, #4 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w8, #4 bge L_aes_xts_decrypt_arm64_crypto_256_start_4 L_aes_xts_decrypt_arm64_crypto_256_start_2: cmp w8, #2 blt L_aes_xts_decrypt_arm64_crypto_256_start_1 ld1 {v0.16b, v1.16b}, [x0], #32 mov v5.d[0], x12 mov v5.d[1], x13 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b and x9, x19, x13, asr 63 aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b extr x11, x13, x12, #63 aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b eor x10, x9, x12, lsl 1 aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b and x9, x19, x11, asr 63 aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b extr x13, x11, x10, #63 aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b eor x12, x9, x10, lsl 1 aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v1.16b, v25.16b aesimc v1.16b, v1.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v1.16b, v26.16b aesimc v1.16b, v1.16b aesd v0.16b, v27.16b aesimc v0.16b, v0.16b aesd v1.16b, v27.16b aesimc v1.16b, v1.16b aesd v0.16b, v28.16b aesimc v0.16b, v0.16b aesd v1.16b, v28.16b aesimc v1.16b, v1.16b aesd v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b aesd v1.16b, v29.16b eor v1.16b, v1.16b, v30.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b mov v4.d[0], x10 mov v4.d[1], x11 sub w8, w8, #2 st1 {v0.16b, v1.16b}, [x1], #32 L_aes_xts_decrypt_arm64_crypto_256_start_1: cbz w8, L_aes_xts_decrypt_arm64_crypto_256_done ld1 {v0.16b}, [x0], #16 eor v0.16b, v0.16b, v4.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b and x9, x19, x11, asr 63 aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b extr x11, x11, x10, #63 aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b eor x10, x9, x10, lsl 1 aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v0.16b, v27.16b aesimc v0.16b, v0.16b aesd v0.16b, v28.16b aesimc v0.16b, v0.16b aesd v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b eor v0.16b, v0.16b, v4.16b mov v4.d[0], x10 mov v4.d[1], x11 st1 {v0.16b}, [x1], #16 L_aes_xts_decrypt_arm64_crypto_256_done: cbz w2, L_aes_xts_decrypt_arm64_crypto_256_partial_done and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 mov v5.d[0], x12 mov v5.d[1], x13 ld1 {v0.16b}, [x0], #16 eor v0.16b, v0.16b, v5.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v0.16b, v27.16b aesimc v0.16b, v0.16b aesd v0.16b, v28.16b aesimc v0.16b, v0.16b aesd v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b eor v0.16b, v0.16b, v5.16b st1 {v0.2d}, [x6] add x1, x1, #16 mov w9, w2 L_aes_xts_decrypt_arm64_crypto_256_start_byte: ldrb w12, [x6] ldrb w13, [x0], #1 strb w12, [x1], #1 strb w13, [x6], #1 subs w9, w9, #1 bgt L_aes_xts_decrypt_arm64_crypto_256_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 ld1 {v0.2d}, [x6] eor v0.16b, v0.16b, v4.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v0.16b, v25.16b aesimc v0.16b, v0.16b aesd v0.16b, v26.16b aesimc v0.16b, v0.16b aesd v0.16b, v27.16b aesimc v0.16b, v0.16b aesd v0.16b, v28.16b aesimc v0.16b, v0.16b aesd v0.16b, v29.16b eor v0.16b, v0.16b, v30.16b eor v0.16b, v0.16b, v4.16b st1 {v0.16b}, [x1] L_aes_xts_decrypt_arm64_crypto_256_partial_done: #endif /* !NO_AES_256 */ b L_aes_xts_decrypt_arm64_crypto_done # AES_XTS_128 L_aes_xts_decrypt_arm64_crypto_start_128: #ifndef NO_AES_128 ld1 {v24.2d, v25.2d}, [x5], #32 ld1 {v26.2d}, [x5] aese v4.16b, v16.16b aesmc v4.16b, v4.16b aese v4.16b, v17.16b aesmc v4.16b, v4.16b aese v4.16b, v18.16b aesmc v4.16b, v4.16b aese v4.16b, v19.16b aesmc v4.16b, v4.16b aese v4.16b, v20.16b aesmc v4.16b, v4.16b aese v4.16b, v21.16b aesmc v4.16b, v4.16b aese v4.16b, v22.16b aesmc v4.16b, v4.16b aese v4.16b, v23.16b aesmc v4.16b, v4.16b aese v4.16b, v24.16b aesmc v4.16b, v4.16b aese v4.16b, v25.16b eor v4.16b, v4.16b, v26.16b mov x10, v4.d[0] mov x11, v4.d[1] ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 ld1 {v24.2d, v25.2d}, [x4], #32 ld1 {v26.2d}, [x4] and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 and x9, x19, x13, asr 63 extr x15, x13, x12, #63 eor x14, x9, x12, lsl 1 and x9, x19, x15, asr 63 extr x17, x15, x14, #63 eor x16, x9, x14, lsl 1 cmp w8, #4 blt L_aes_xts_decrypt_arm64_crypto_128_start_2 L_aes_xts_decrypt_arm64_crypto_128_start_4: ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 mov v5.d[0], x12 mov v5.d[1], x13 mov v6.d[0], x14 mov v6.d[1], x15 mov v7.d[0], x16 mov v7.d[1], x17 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b and x9, x19, x17, asr 63 aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v2.16b, v16.16b aesimc v2.16b, v2.16b extr x11, x17, x16, #63 aesd v3.16b, v16.16b aesimc v3.16b, v3.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b eor x10, x9, x16, lsl 1 aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v2.16b, v17.16b aesimc v2.16b, v2.16b and x9, x19, x11, asr 63 aesd v3.16b, v17.16b aesimc v3.16b, v3.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b extr x13, x11, x10, #63 aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v2.16b, v18.16b aesimc v2.16b, v2.16b eor x12, x9, x10, lsl 1 aesd v3.16b, v18.16b aesimc v3.16b, v3.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b and x9, x19, x13, asr 63 aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v2.16b, v19.16b aesimc v2.16b, v2.16b extr x15, x13, x12, #63 aesd v3.16b, v19.16b aesimc v3.16b, v3.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b eor x14, x9, x12, lsl 1 aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v2.16b, v20.16b aesimc v2.16b, v2.16b and x9, x19, x15, asr 63 aesd v3.16b, v20.16b aesimc v3.16b, v3.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b extr x17, x15, x14, #63 aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v2.16b, v21.16b aesimc v2.16b, v2.16b eor x16, x9, x14, lsl 1 aesd v3.16b, v21.16b aesimc v3.16b, v3.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v2.16b, v22.16b aesimc v2.16b, v2.16b aesd v3.16b, v22.16b aesimc v3.16b, v3.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v2.16b, v23.16b aesimc v2.16b, v2.16b aesd v3.16b, v23.16b aesimc v3.16b, v3.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v2.16b, v24.16b aesimc v2.16b, v2.16b aesd v3.16b, v24.16b aesimc v3.16b, v3.16b aesd v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b aesd v1.16b, v25.16b eor v1.16b, v1.16b, v26.16b aesd v2.16b, v25.16b eor v2.16b, v2.16b, v26.16b aesd v3.16b, v25.16b eor v3.16b, v3.16b, v26.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b mov v4.d[0], x10 mov v4.d[1], x11 sub w8, w8, #4 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 cmp w8, #4 bge L_aes_xts_decrypt_arm64_crypto_128_start_4 L_aes_xts_decrypt_arm64_crypto_128_start_2: cmp w8, #2 blt L_aes_xts_decrypt_arm64_crypto_128_start_1 ld1 {v0.16b, v1.16b}, [x0], #32 mov v5.d[0], x12 mov v5.d[1], x13 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b and x9, x19, x13, asr 63 aesd v1.16b, v16.16b aesimc v1.16b, v1.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b extr x11, x13, x12, #63 aesd v1.16b, v17.16b aesimc v1.16b, v1.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b eor x10, x9, x12, lsl 1 aesd v1.16b, v18.16b aesimc v1.16b, v1.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b and x9, x19, x11, asr 63 aesd v1.16b, v19.16b aesimc v1.16b, v1.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b extr x13, x11, x10, #63 aesd v1.16b, v20.16b aesimc v1.16b, v1.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b eor x12, x9, x10, lsl 1 aesd v1.16b, v21.16b aesimc v1.16b, v1.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v1.16b, v22.16b aesimc v1.16b, v1.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v1.16b, v23.16b aesimc v1.16b, v1.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v1.16b, v24.16b aesimc v1.16b, v1.16b aesd v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b aesd v1.16b, v25.16b eor v1.16b, v1.16b, v26.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b mov v4.d[0], x10 mov v4.d[1], x11 sub w8, w8, #2 st1 {v0.16b, v1.16b}, [x1], #32 L_aes_xts_decrypt_arm64_crypto_128_start_1: cbz w8, L_aes_xts_decrypt_arm64_crypto_128_done ld1 {v0.16b}, [x0], #16 eor v0.16b, v0.16b, v4.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b and x9, x19, x11, asr 63 aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b extr x11, x11, x10, #63 aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b eor x10, x9, x10, lsl 1 aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b eor v0.16b, v0.16b, v4.16b mov v4.d[0], x10 mov v4.d[1], x11 st1 {v0.16b}, [x1], #16 L_aes_xts_decrypt_arm64_crypto_128_done: cbz w2, L_aes_xts_decrypt_arm64_crypto_128_partial_done and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 mov v5.d[0], x12 mov v5.d[1], x13 ld1 {v0.16b}, [x0], #16 eor v0.16b, v0.16b, v5.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b eor v0.16b, v0.16b, v5.16b st1 {v0.2d}, [x6] add x1, x1, #16 mov w9, w2 L_aes_xts_decrypt_arm64_crypto_128_start_byte: ldrb w12, [x6] ldrb w13, [x0], #1 strb w12, [x1], #1 strb w13, [x6], #1 subs w9, w9, #1 bgt L_aes_xts_decrypt_arm64_crypto_128_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 ld1 {v0.2d}, [x6] eor v0.16b, v0.16b, v4.16b aesd v0.16b, v16.16b aesimc v0.16b, v0.16b aesd v0.16b, v17.16b aesimc v0.16b, v0.16b aesd v0.16b, v18.16b aesimc v0.16b, v0.16b aesd v0.16b, v19.16b aesimc v0.16b, v0.16b aesd v0.16b, v20.16b aesimc v0.16b, v0.16b aesd v0.16b, v21.16b aesimc v0.16b, v0.16b aesd v0.16b, v22.16b aesimc v0.16b, v0.16b aesd v0.16b, v23.16b aesimc v0.16b, v0.16b aesd v0.16b, v24.16b aesimc v0.16b, v0.16b aesd v0.16b, v25.16b eor v0.16b, v0.16b, v26.16b eor v0.16b, v0.16b, v4.16b st1 {v0.16b}, [x1] L_aes_xts_decrypt_arm64_crypto_128_partial_done: #endif /* !NO_AES_128 */ L_aes_xts_decrypt_arm64_crypto_done: ldp x17, x19, [x29, #24] ldp x29, x30, [sp], #48 ret #ifndef __APPLE__ .size AES_XTS_decrypt_AARCH64,.-AES_XTS_decrypt_AARCH64 #endif /* __APPLE__ */ #endif /* HAVE_AES_DECRYPT */ #endif /* WOLFSSL_AES_XTS */ #endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */ #ifndef WOLFSSL_ARMASM_NO_NEON #if defined(HAVE_AES_DECRYPT) || defined(HAVE_AES_CBC) || \ defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) #ifndef __APPLE__ .text .section .rodata .type L_AES_ARM64_NEON_te, %object .size L_AES_ARM64_NEON_te, 256 #else .section __DATA,__data #endif /* __APPLE__ */ # 8-byte aligned, 64-bit aligned #ifndef __APPLE__ .align 3 #else .p2align 3 #endif /* __APPLE__ */ L_AES_ARM64_NEON_te: .byte 0x63,0x7c,0x77,0x7b,0xf2,0x6b,0x6f,0xc5 .byte 0x30,0x01,0x67,0x2b,0xfe,0xd7,0xab,0x76 .byte 0xca,0x82,0xc9,0x7d,0xfa,0x59,0x47,0xf0 .byte 0xad,0xd4,0xa2,0xaf,0x9c,0xa4,0x72,0xc0 .byte 0xb7,0xfd,0x93,0x26,0x36,0x3f,0xf7,0xcc .byte 0x34,0xa5,0xe5,0xf1,0x71,0xd8,0x31,0x15 .byte 0x04,0xc7,0x23,0xc3,0x18,0x96,0x05,0x9a .byte 0x07,0x12,0x80,0xe2,0xeb,0x27,0xb2,0x75 .byte 0x09,0x83,0x2c,0x1a,0x1b,0x6e,0x5a,0xa0 .byte 0x52,0x3b,0xd6,0xb3,0x29,0xe3,0x2f,0x84 .byte 0x53,0xd1,0x00,0xed,0x20,0xfc,0xb1,0x5b .byte 0x6a,0xcb,0xbe,0x39,0x4a,0x4c,0x58,0xcf .byte 0xd0,0xef,0xaa,0xfb,0x43,0x4d,0x33,0x85 .byte 0x45,0xf9,0x02,0x7f,0x50,0x3c,0x9f,0xa8 .byte 0x51,0xa3,0x40,0x8f,0x92,0x9d,0x38,0xf5 .byte 0xbc,0xb6,0xda,0x21,0x10,0xff,0xf3,0xd2 .byte 0xcd,0x0c,0x13,0xec,0x5f,0x97,0x44,0x17 .byte 0xc4,0xa7,0x7e,0x3d,0x64,0x5d,0x19,0x73 .byte 0x60,0x81,0x4f,0xdc,0x22,0x2a,0x90,0x88 .byte 0x46,0xee,0xb8,0x14,0xde,0x5e,0x0b,0xdb .byte 0xe0,0x32,0x3a,0x0a,0x49,0x06,0x24,0x5c .byte 0xc2,0xd3,0xac,0x62,0x91,0x95,0xe4,0x79 .byte 0xe7,0xc8,0x37,0x6d,0x8d,0xd5,0x4e,0xa9 .byte 0x6c,0x56,0xf4,0xea,0x65,0x7a,0xae,0x08 .byte 0xba,0x78,0x25,0x2e,0x1c,0xa6,0xb4,0xc6 .byte 0xe8,0xdd,0x74,0x1f,0x4b,0xbd,0x8b,0x8a .byte 0x70,0x3e,0xb5,0x66,0x48,0x03,0xf6,0x0e .byte 0x61,0x35,0x57,0xb9,0x86,0xc1,0x1d,0x9e .byte 0xe1,0xf8,0x98,0x11,0x69,0xd9,0x8e,0x94 .byte 0x9b,0x1e,0x87,0xe9,0xce,0x55,0x28,0xdf .byte 0x8c,0xa1,0x89,0x0d,0xbf,0xe6,0x42,0x68 .byte 0x41,0x99,0x2d,0x0f,0xb0,0x54,0xbb,0x16 #ifndef __APPLE__ .text .section .rodata .type L_AES_ARM64_NEON_shift_rows_shuffle, %object .size L_AES_ARM64_NEON_shift_rows_shuffle, 16 #else .section __DATA,__data #endif /* __APPLE__ */ # 8-byte aligned, 64-bit aligned #ifndef __APPLE__ .align 3 #else .p2align 3 #endif /* __APPLE__ */ L_AES_ARM64_NEON_shift_rows_shuffle: .byte 0x0c,0x09,0x06,0x03,0x00,0x0d,0x0a,0x07 .byte 0x04,0x01,0x0e,0x0b,0x08,0x05,0x02,0x0f #endif /* HAVE_AES_DECRYPT || HAVE_AES_CBC || HAVE_AESCCM || HAVE_AESGCM || * WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER */ #ifdef HAVE_AES_DECRYPT #ifndef __APPLE__ .text .globl AES_invert_key_NEON .type AES_invert_key_NEON,@function .align 2 AES_invert_key_NEON: #else .section __TEXT,__text .globl _AES_invert_key_NEON .p2align 2 _AES_invert_key_NEON: #endif /* __APPLE__ */ add x3, x0, x1, lsl 4 mov x2, x0 mov w4, w1 L_AES_invert_key_NEON_loop: ld1 {v0.2d}, [x2] ld1 {v1.2d}, [x3] st1 {v0.2d}, [x3] st1 {v1.2d}, [x2], #16 subs w4, w4, #2 sub x3, x3, #16 bne L_AES_invert_key_NEON_loop movi v2.16b, #27 add x2, x0, #16 sub w4, w1, #1 L_AES_invert_key_NEON_mix_loop: ld1 {v0.2d}, [x2] sshr v5.16b, v0.16b, #7 ushr v6.16b, v0.16b, #6 ushr v3.16b, v0.16b, #5 and v5.16b, v5.16b, v2.16b pmul v6.16b, v6.16b, v2.16b pmul v3.16b, v3.16b, v2.16b shl v4.16b, v0.16b, #1 eor v5.16b, v5.16b, v4.16b shl v4.16b, v0.16b, #3 eor v3.16b, v3.16b, v4.16b shl v4.16b, v0.16b, #2 eor v6.16b, v6.16b, v4.16b eor v4.16b, v5.16b, v3.16b eor v3.16b, v3.16b, v0.16b eor v5.16b, v6.16b, v3.16b eor v6.16b, v6.16b, v4.16b eor v4.16b, v4.16b, v0.16b shl v0.4s, v4.4s, #8 rev32 v5.8h, v5.8h sri v0.4s, v4.4s, #24 eor v0.16b, v0.16b, v6.16b shl v4.4s, v3.4s, #24 eor v0.16b, v0.16b, v5.16b sri v4.4s, v3.4s, #8 eor v0.16b, v0.16b, v4.16b st1 {v0.2d}, [x2], #16 subs w4, w4, #1 bne L_AES_invert_key_NEON_mix_loop ret #ifndef __APPLE__ .size AES_invert_key_NEON,.-AES_invert_key_NEON #endif /* __APPLE__ */ #endif /* HAVE_AES_DECRYPT */ #ifndef __APPLE__ .text .section .rodata .type L_AES_ARM64_NEON_rcon, %object .size L_AES_ARM64_NEON_rcon, 40 #else .section __DATA,__data #endif /* __APPLE__ */ # 8-byte aligned, 64-bit aligned #ifndef __APPLE__ .align 3 #else .p2align 3 #endif /* __APPLE__ */ L_AES_ARM64_NEON_rcon: .long 0x01000000,0x02000000,0x04000000,0x08000000 .long 0x10000000,0x20000000,0x40000000,0x80000000 .long 0x1b000000,0x36000000 #ifndef __APPLE__ .text .globl AES_set_encrypt_key_NEON .type AES_set_encrypt_key_NEON,@function .align 2 AES_set_encrypt_key_NEON: #else .section __TEXT,__text .globl _AES_set_encrypt_key_NEON .p2align 2 _AES_set_encrypt_key_NEON: #endif /* __APPLE__ */ stp x29, x30, [sp, #-80]! add x29, sp, #0 stp d8, d9, [x29, #16] stp d10, d11, [x29, #32] stp d12, d13, [x29, #48] stp d14, d15, [x29, #64] #ifndef __APPLE__ adrp x4, L_AES_ARM64_NEON_rcon add x4, x4, :lo12:L_AES_ARM64_NEON_rcon #else adrp x4, L_AES_ARM64_NEON_rcon@PAGE add x4, x4, L_AES_ARM64_NEON_rcon@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x5, L_AES_ARM64_NEON_te add x5, x5, :lo12:L_AES_ARM64_NEON_te #else adrp x5, L_AES_ARM64_NEON_te@PAGE add x5, x5, L_AES_ARM64_NEON_te@PAGEOFF #endif /* __APPLE__ */ ld1 {v6.16b, v7.16b, v8.16b, v9.16b}, [x5], #0x40 ld1 {v10.16b, v11.16b, v12.16b, v13.16b}, [x5], #0x40 ld1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x5], #0x40 ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x5] movi v2.16b, #0x40 movi v3.16b, #0x80 movi v4.16b, #0xc0 movi v5.16b, #27 eor v26.16b, v26.16b, v26.16b cmp x1, #0x80 beq L_AES_set_encrypt_key_NEON_start_128 cmp x1, #0xc0 beq L_AES_set_encrypt_key_NEON_start_192 ld1 {v0.16b}, [x0], #16 ld1 {v1.16b}, [x0] rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b st1 {v0.2d}, [x2], #16 st1 {v1.2d}, [x2], #16 mov x3, #6 L_AES_set_encrypt_key_NEON_loop_256: eor v22.16b, v1.16b, v2.16b eor v23.16b, v1.16b, v3.16b eor v24.16b, v1.16b, v4.16b tbl v25.16b, {v6.16b, v7.16b, v8.16b, v9.16b}, v1.16b tbl v22.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v22.16b tbl v23.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v23.16b tbl v24.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v24.16b orr v25.16b, v25.16b, v22.16b orr v23.16b, v23.16b, v24.16b orr v25.16b, v25.16b, v23.16b ext v25.16b, v25.16b, v26.16b, #12 shl v22.4s, v25.4s, #8 sri v22.4s, v25.4s, #24 eor v0.16b, v0.16b, v22.16b ld1r {v25.4s}, [x4], #4 dup v22.4s, v0.s[0] dup v23.2s, v0.s[1] dup v24.2s, v0.s[2] ext v22.16b, v26.16b, v22.16b, #12 ext v23.16b, v26.16b, v23.16b, #8 eor v0.16b, v0.16b, v22.16b ext v24.16b, v26.16b, v24.16b, #4 eor v0.16b, v0.16b, v23.16b eor v0.16b, v0.16b, v24.16b eor v0.16b, v0.16b, v25.16b st1 {v0.2d}, [x2], #16 eor v22.16b, v0.16b, v2.16b eor v23.16b, v0.16b, v3.16b eor v24.16b, v0.16b, v4.16b tbl v25.16b, {v6.16b, v7.16b, v8.16b, v9.16b}, v0.16b tbl v22.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v22.16b tbl v23.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v23.16b tbl v24.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v24.16b orr v25.16b, v25.16b, v22.16b orr v23.16b, v23.16b, v24.16b orr v25.16b, v25.16b, v23.16b ext v25.16b, v25.16b, v26.16b, #12 eor v1.16b, v1.16b, v25.16b dup v22.4s, v1.s[0] dup v23.2s, v1.s[1] dup v24.2s, v1.s[2] ext v22.16b, v26.16b, v22.16b, #12 ext v23.16b, v26.16b, v23.16b, #8 eor v1.16b, v1.16b, v22.16b ext v24.16b, v26.16b, v24.16b, #4 eor v1.16b, v1.16b, v23.16b eor v1.16b, v1.16b, v24.16b st1 {v1.2d}, [x2], #16 subs x3, x3, #1 bne L_AES_set_encrypt_key_NEON_loop_256 eor v22.16b, v1.16b, v2.16b eor v23.16b, v1.16b, v3.16b eor v24.16b, v1.16b, v4.16b tbl v25.16b, {v6.16b, v7.16b, v8.16b, v9.16b}, v1.16b tbl v22.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v22.16b tbl v23.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v23.16b tbl v24.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v24.16b orr v25.16b, v25.16b, v22.16b orr v23.16b, v23.16b, v24.16b orr v25.16b, v25.16b, v23.16b ext v25.16b, v25.16b, v26.16b, #12 shl v22.4s, v25.4s, #8 sri v22.4s, v25.4s, #24 eor v0.16b, v0.16b, v22.16b ld1r {v25.4s}, [x4], #4 dup v22.4s, v0.s[0] dup v23.2s, v0.s[1] dup v24.2s, v0.s[2] ext v22.16b, v26.16b, v22.16b, #12 ext v23.16b, v26.16b, v23.16b, #8 eor v0.16b, v0.16b, v22.16b ext v24.16b, v26.16b, v24.16b, #4 eor v0.16b, v0.16b, v23.16b eor v0.16b, v0.16b, v24.16b eor v0.16b, v0.16b, v25.16b st1 {v0.2d}, [x2], #16 b L_AES_set_encrypt_key_NEON_end L_AES_set_encrypt_key_NEON_start_192: ld1 {v0.16b}, [x0], #16 ld1 {v1.8b}, [x0] rev32 v0.16b, v0.16b rev32 v1.8b, v1.8b st1 {v0.16b}, [x2], #16 st1 {v1.8b}, [x2], #8 ext v1.16b, v1.16b, v1.16b, #8 mov x3, #7 L_AES_set_encrypt_key_NEON_loop_192: eor v22.16b, v1.16b, v2.16b eor v23.16b, v1.16b, v3.16b eor v24.16b, v1.16b, v4.16b tbl v25.16b, {v6.16b, v7.16b, v8.16b, v9.16b}, v1.16b tbl v22.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v22.16b tbl v23.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v23.16b tbl v24.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v24.16b orr v25.16b, v25.16b, v22.16b orr v23.16b, v23.16b, v24.16b orr v25.16b, v25.16b, v23.16b ext v25.16b, v25.16b, v26.16b, #12 shl v22.4s, v25.4s, #8 sri v22.4s, v25.4s, #24 eor v0.16b, v0.16b, v22.16b ld1r {v25.4s}, [x4], #4 dup v22.4s, v0.s[0] dup v23.2s, v0.s[1] dup v24.2s, v0.s[2] ext v22.16b, v26.16b, v22.16b, #12 ext v23.16b, v26.16b, v23.16b, #8 eor v0.16b, v0.16b, v22.16b ext v24.16b, v26.16b, v24.16b, #4 eor v0.16b, v0.16b, v23.16b eor v0.16b, v0.16b, v24.16b eor v0.16b, v0.16b, v25.16b st1 {v0.2d}, [x2], #16 mov v23.16b, v26.16b mov v23.s[2], v0.s[3] eor v1.16b, v1.16b, v23.16b mov v23.16b, v26.16b mov v23.s[3], v1.s[2] eor v1.16b, v1.16b, v23.16b st1 {v1.d}[1], [x2], #8 subs x3, x3, #1 bne L_AES_set_encrypt_key_NEON_loop_192 eor v22.16b, v1.16b, v2.16b eor v23.16b, v1.16b, v3.16b eor v24.16b, v1.16b, v4.16b tbl v25.16b, {v6.16b, v7.16b, v8.16b, v9.16b}, v1.16b tbl v22.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v22.16b tbl v23.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v23.16b tbl v24.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v24.16b orr v25.16b, v25.16b, v22.16b orr v23.16b, v23.16b, v24.16b orr v25.16b, v25.16b, v23.16b ext v25.16b, v25.16b, v26.16b, #12 shl v22.4s, v25.4s, #8 sri v22.4s, v25.4s, #24 eor v0.16b, v0.16b, v22.16b ld1r {v25.4s}, [x4], #4 dup v22.4s, v0.s[0] dup v23.2s, v0.s[1] dup v24.2s, v0.s[2] ext v22.16b, v26.16b, v22.16b, #12 ext v23.16b, v26.16b, v23.16b, #8 eor v0.16b, v0.16b, v22.16b ext v24.16b, v26.16b, v24.16b, #4 eor v0.16b, v0.16b, v23.16b eor v0.16b, v0.16b, v24.16b eor v0.16b, v0.16b, v25.16b st1 {v0.2d}, [x2], #16 b L_AES_set_encrypt_key_NEON_end L_AES_set_encrypt_key_NEON_start_128: ld1 {v0.16b}, [x0] rev32 v0.16b, v0.16b st1 {v0.2d}, [x2], #16 mov x3, #10 L_AES_set_encrypt_key_NEON_loop_128: eor v22.16b, v0.16b, v2.16b eor v23.16b, v0.16b, v3.16b eor v24.16b, v0.16b, v4.16b tbl v25.16b, {v6.16b, v7.16b, v8.16b, v9.16b}, v0.16b tbl v22.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v22.16b tbl v23.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v23.16b tbl v24.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v24.16b orr v25.16b, v25.16b, v22.16b orr v23.16b, v23.16b, v24.16b orr v25.16b, v25.16b, v23.16b ext v25.16b, v25.16b, v26.16b, #12 shl v22.4s, v25.4s, #8 sri v22.4s, v25.4s, #24 eor v0.16b, v0.16b, v22.16b ld1r {v25.4s}, [x4], #4 dup v22.4s, v0.s[0] dup v23.2s, v0.s[1] dup v24.2s, v0.s[2] ext v22.16b, v26.16b, v22.16b, #12 ext v23.16b, v26.16b, v23.16b, #8 eor v0.16b, v0.16b, v22.16b ext v24.16b, v26.16b, v24.16b, #4 eor v0.16b, v0.16b, v23.16b eor v0.16b, v0.16b, v24.16b eor v0.16b, v0.16b, v25.16b st1 {v0.2d}, [x2], #16 subs x3, x3, #1 bne L_AES_set_encrypt_key_NEON_loop_128 L_AES_set_encrypt_key_NEON_end: ldp d8, d9, [x29, #16] ldp d10, d11, [x29, #32] ldp d12, d13, [x29, #48] ldp d14, d15, [x29, #64] ldp x29, x30, [sp], #0x50 ret #ifndef __APPLE__ .size AES_set_encrypt_key_NEON,.-AES_set_encrypt_key_NEON #endif /* __APPLE__ */ #if defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || \ defined(HAVE_AES_ECB) #ifndef __APPLE__ .text .globl AES_ECB_encrypt_NEON .type AES_ECB_encrypt_NEON,@function .align 2 AES_ECB_encrypt_NEON: #else .section __TEXT,__text .globl _AES_ECB_encrypt_NEON .p2align 2 _AES_ECB_encrypt_NEON: #endif /* __APPLE__ */ stp x29, x30, [sp, #-80]! add x29, sp, #0 stp d8, d9, [x29, #16] stp d10, d11, [x29, #32] stp d12, d13, [x29, #48] stp d14, d15, [x29, #64] #ifndef __APPLE__ adrp x5, L_AES_ARM64_NEON_te add x5, x5, :lo12:L_AES_ARM64_NEON_te #else adrp x5, L_AES_ARM64_NEON_te@PAGE add x5, x5, L_AES_ARM64_NEON_te@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x6, L_AES_ARM64_NEON_shift_rows_shuffle add x6, x6, :lo12:L_AES_ARM64_NEON_shift_rows_shuffle #else adrp x6, L_AES_ARM64_NEON_shift_rows_shuffle@PAGE add x6, x6, L_AES_ARM64_NEON_shift_rows_shuffle@PAGEOFF #endif /* __APPLE__ */ ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x5], #0x40 ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x5], #0x40 ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x5], #0x40 ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x5] cmp x2, #0x40 blt L_AES_ECB_encrypt_NEON_start_2 L_AES_ECB_encrypt_NEON_loop_4: mov x8, x3 ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 ld1 {v4.2d}, [x8], #16 rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b rev32 v2.16b, v2.16b rev32 v3.16b, v3.16b # Round: 0 - XOR in key schedule eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b sub w7, w4, #2 L_AES_ECB_encrypt_NEON_loop_nr_4: tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b eor v10.16b, v2.16b, v12.16b eor v11.16b, v3.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v13.16b eor v9.16b, v1.16b, v13.16b eor v10.16b, v2.16b, v13.16b eor v11.16b, v3.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b eor v10.16b, v2.16b, v14.16b eor v11.16b, v3.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b ld1 {v0.16b}, [x6] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b tbl v6.16b, {v6.16b}, v0.16b tbl v7.16b, {v7.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 sshr v10.16b, v6.16b, #7 sshr v11.16b, v7.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 shl v14.16b, v6.16b, #1 shl v15.16b, v7.16b, #1 movi v0.16b, #27 and v8.16b, v8.16b, v0.16b and v9.16b, v9.16b, v0.16b and v10.16b, v10.16b, v0.16b and v11.16b, v11.16b, v0.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b eor v2.16b, v10.16b, v6.16b eor v3.16b, v11.16b, v7.16b shl v12.4s, v0.4s, #8 shl v13.4s, v1.4s, #8 shl v14.4s, v2.4s, #8 shl v15.4s, v3.4s, #8 sri v12.4s, v0.4s, #24 sri v13.4s, v1.4s, #24 sri v14.4s, v2.4s, #24 sri v15.4s, v3.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 shl v2.4s, v6.4s, #24 shl v3.4s, v7.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 sri v2.4s, v6.4s, #8 sri v3.4s, v7.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h rev32 v6.8h, v6.8h rev32 v7.8h, v7.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b eor v6.16b, v6.16b, v2.16b eor v7.16b, v7.16b, v3.16b # XOR in Key Schedule ld1 {v0.2d}, [x8], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v6.16b, v6.16b, v0.16b eor v7.16b, v7.16b, v0.16b eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b eor v6.16b, v6.16b, v14.16b eor v7.16b, v7.16b, v15.16b # Round Done tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b eor v10.16b, v6.16b, v12.16b eor v11.16b, v7.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v13.16b eor v9.16b, v5.16b, v13.16b eor v10.16b, v6.16b, v13.16b eor v11.16b, v7.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b eor v10.16b, v6.16b, v14.16b eor v11.16b, v7.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b ld1 {v4.16b}, [x6] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b tbl v2.16b, {v2.16b}, v4.16b tbl v3.16b, {v3.16b}, v4.16b sshr v8.16b, v0.16b, #7 sshr v9.16b, v1.16b, #7 sshr v10.16b, v2.16b, #7 sshr v11.16b, v3.16b, #7 shl v12.16b, v0.16b, #1 shl v13.16b, v1.16b, #1 shl v14.16b, v2.16b, #1 shl v15.16b, v3.16b, #1 movi v4.16b, #27 and v8.16b, v8.16b, v4.16b and v9.16b, v9.16b, v4.16b and v10.16b, v10.16b, v4.16b and v11.16b, v11.16b, v4.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b eor v4.16b, v8.16b, v0.16b eor v5.16b, v9.16b, v1.16b eor v6.16b, v10.16b, v2.16b eor v7.16b, v11.16b, v3.16b shl v12.4s, v4.4s, #8 shl v13.4s, v5.4s, #8 shl v14.4s, v6.4s, #8 shl v15.4s, v7.4s, #8 sri v12.4s, v4.4s, #24 sri v13.4s, v5.4s, #24 sri v14.4s, v6.4s, #24 sri v15.4s, v7.4s, #24 shl v4.4s, v0.4s, #24 shl v5.4s, v1.4s, #24 shl v6.4s, v2.4s, #24 shl v7.4s, v3.4s, #24 sri v4.4s, v0.4s, #8 sri v5.4s, v1.4s, #8 sri v6.4s, v2.4s, #8 sri v7.4s, v3.4s, #8 rev32 v0.8h, v0.8h rev32 v1.8h, v1.8h rev32 v2.8h, v2.8h rev32 v3.8h, v3.8h eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b # XOR in Key Schedule ld1 {v4.2d}, [x8], #16 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b eor v0.16b, v0.16b, v12.16b eor v1.16b, v1.16b, v13.16b eor v2.16b, v2.16b, v14.16b eor v3.16b, v3.16b, v15.16b # Round Done subs w7, w7, #2 bne L_AES_ECB_encrypt_NEON_loop_nr_4 tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b eor v10.16b, v2.16b, v12.16b eor v11.16b, v3.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v13.16b eor v9.16b, v1.16b, v13.16b eor v10.16b, v2.16b, v13.16b eor v11.16b, v3.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b eor v10.16b, v2.16b, v14.16b eor v11.16b, v3.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b ld1 {v0.16b}, [x6] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b tbl v6.16b, {v6.16b}, v0.16b tbl v7.16b, {v7.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 sshr v10.16b, v6.16b, #7 sshr v11.16b, v7.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 shl v14.16b, v6.16b, #1 shl v15.16b, v7.16b, #1 movi v0.16b, #27 and v8.16b, v8.16b, v0.16b and v9.16b, v9.16b, v0.16b and v10.16b, v10.16b, v0.16b and v11.16b, v11.16b, v0.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b eor v2.16b, v10.16b, v6.16b eor v3.16b, v11.16b, v7.16b shl v12.4s, v0.4s, #8 shl v13.4s, v1.4s, #8 shl v14.4s, v2.4s, #8 shl v15.4s, v3.4s, #8 sri v12.4s, v0.4s, #24 sri v13.4s, v1.4s, #24 sri v14.4s, v2.4s, #24 sri v15.4s, v3.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 shl v2.4s, v6.4s, #24 shl v3.4s, v7.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 sri v2.4s, v6.4s, #8 sri v3.4s, v7.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h rev32 v6.8h, v6.8h rev32 v7.8h, v7.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b eor v6.16b, v6.16b, v2.16b eor v7.16b, v7.16b, v3.16b # XOR in Key Schedule ld1 {v0.2d}, [x8], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v6.16b, v6.16b, v0.16b eor v7.16b, v7.16b, v0.16b eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b eor v6.16b, v6.16b, v14.16b eor v7.16b, v7.16b, v15.16b # Round Done tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b eor v10.16b, v6.16b, v12.16b eor v11.16b, v7.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v13.16b eor v9.16b, v5.16b, v13.16b eor v10.16b, v6.16b, v13.16b eor v11.16b, v7.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b eor v10.16b, v6.16b, v14.16b eor v11.16b, v7.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b ld1 {v4.16b}, [x6] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b tbl v2.16b, {v2.16b}, v4.16b tbl v3.16b, {v3.16b}, v4.16b # XOR in Key Schedule ld1 {v4.2d}, [x8], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b # Round Done rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b rev32 v2.16b, v2.16b rev32 v3.16b, v3.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 sub x2, x2, #0x40 cmp x2, #0x40 bge L_AES_ECB_encrypt_NEON_loop_4 L_AES_ECB_encrypt_NEON_start_2: movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 movi v15.16b, #27 cmp x2, #16 beq L_AES_ECB_encrypt_NEON_start_1 blt L_AES_ECB_encrypt_NEON_data_done L_AES_ECB_encrypt_NEON_loop_2: mov x8, x3 ld1 {v0.16b, v1.16b}, [x0], #32 ld1 {v4.2d}, [x8], #16 rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b # Round: 0 - XOR in key schedule eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b sub w7, w4, #2 L_AES_ECB_encrypt_NEON_loop_nr_2: eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v0.16b, v13.16b eor v11.16b, v1.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b orr v4.16b, v4.16b, v10.16b orr v5.16b, v5.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b ld1 {v0.16b}, [x6] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 shl v10.16b, v4.16b, #1 shl v11.16b, v5.16b, #1 and v8.16b, v8.16b, v15.16b and v9.16b, v9.16b, v15.16b eor v8.16b, v8.16b, v10.16b eor v9.16b, v9.16b, v11.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b shl v10.4s, v0.4s, #8 shl v11.4s, v1.4s, #8 sri v10.4s, v0.4s, #24 sri v11.4s, v1.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b # XOR in Key Schedule ld1 {v0.2d}, [x8], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v4.16b, v4.16b, v10.16b eor v5.16b, v5.16b, v11.16b # Round Done eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v4.16b, v13.16b eor v11.16b, v5.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b orr v0.16b, v0.16b, v10.16b orr v1.16b, v1.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b ld1 {v4.16b}, [x6] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b sshr v8.16b, v0.16b, #7 sshr v9.16b, v1.16b, #7 shl v10.16b, v0.16b, #1 shl v11.16b, v1.16b, #1 and v8.16b, v8.16b, v15.16b and v9.16b, v9.16b, v15.16b eor v8.16b, v8.16b, v10.16b eor v9.16b, v9.16b, v11.16b eor v4.16b, v8.16b, v0.16b eor v5.16b, v9.16b, v1.16b shl v10.4s, v4.4s, #8 shl v11.4s, v5.4s, #8 sri v10.4s, v4.4s, #24 sri v11.4s, v5.4s, #24 shl v4.4s, v0.4s, #24 shl v5.4s, v1.4s, #24 sri v4.4s, v0.4s, #8 sri v5.4s, v1.4s, #8 rev32 v0.8h, v0.8h rev32 v1.8h, v1.8h eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b # XOR in Key Schedule ld1 {v4.2d}, [x8], #16 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v0.16b, v0.16b, v10.16b eor v1.16b, v1.16b, v11.16b # Round Done subs w7, w7, #2 bne L_AES_ECB_encrypt_NEON_loop_nr_2 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v0.16b, v13.16b eor v11.16b, v1.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b orr v4.16b, v4.16b, v10.16b orr v5.16b, v5.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b ld1 {v0.16b}, [x6] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 shl v10.16b, v4.16b, #1 shl v11.16b, v5.16b, #1 and v8.16b, v8.16b, v15.16b and v9.16b, v9.16b, v15.16b eor v8.16b, v8.16b, v10.16b eor v9.16b, v9.16b, v11.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b shl v10.4s, v0.4s, #8 shl v11.4s, v1.4s, #8 sri v10.4s, v0.4s, #24 sri v11.4s, v1.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b # XOR in Key Schedule ld1 {v0.2d}, [x8], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v4.16b, v4.16b, v10.16b eor v5.16b, v5.16b, v11.16b # Round Done eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v4.16b, v13.16b eor v11.16b, v5.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b orr v0.16b, v0.16b, v10.16b orr v1.16b, v1.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b ld1 {v4.16b}, [x6] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b # XOR in Key Schedule ld1 {v4.2d}, [x8], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b # Round Done rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b st1 {v0.16b, v1.16b}, [x1], #32 sub x2, x2, #32 cmp x2, #0 beq L_AES_ECB_encrypt_NEON_data_done L_AES_ECB_encrypt_NEON_start_1: ld1 {v3.2d}, [x6] mov x8, x3 ld1 {v0.16b}, [x0], #16 ld1 {v4.2d}, [x8], #16 rev32 v0.16b, v0.16b # Round: 0 - XOR in key schedule eor v0.16b, v0.16b, v4.16b sub w7, w4, #2 L_AES_ECB_encrypt_NEON_loop_nr_1: eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b ld1 {v0.2d}, [x8], #16 sshr v10.16b, v4.16b, #7 shl v9.16b, v4.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v4.8h eor v11.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v4.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v0.16b sri v9.4s, v4.4s, #8 sri v8.4s, v11.4s, #24 eor v4.16b, v10.16b, v9.16b eor v4.16b, v4.16b, v8.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b ld1 {v4.2d}, [x8], #16 sshr v10.16b, v0.16b, #7 shl v9.16b, v0.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v0.8h eor v11.16b, v10.16b, v0.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v0.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v4.16b sri v9.4s, v0.4s, #8 sri v8.4s, v11.4s, #24 eor v0.16b, v10.16b, v9.16b eor v0.16b, v0.16b, v8.16b subs w7, w7, #2 bne L_AES_ECB_encrypt_NEON_loop_nr_1 eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b ld1 {v0.2d}, [x8], #16 sshr v10.16b, v4.16b, #7 shl v9.16b, v4.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v4.8h eor v11.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v4.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v0.16b sri v9.4s, v4.4s, #8 sri v8.4s, v11.4s, #24 eor v4.16b, v10.16b, v9.16b eor v4.16b, v4.16b, v8.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b ld1 {v4.2d}, [x8], #16 # XOR in Key Schedule eor v0.16b, v0.16b, v4.16b rev32 v0.16b, v0.16b st1 {v0.16b}, [x1], #16 L_AES_ECB_encrypt_NEON_data_done: ldp d8, d9, [x29, #16] ldp d10, d11, [x29, #32] ldp d12, d13, [x29, #48] ldp d14, d15, [x29, #64] ldp x29, x30, [sp], #0x50 ret #ifndef __APPLE__ .size AES_ECB_encrypt_NEON,.-AES_ECB_encrypt_NEON #endif /* __APPLE__ */ #endif /* HAVE_AESCCM || HAVE_AESGCM || WOLFSSL_AES_DIRECT || * WOLFSSL_AES_COUNTER || HAVE_AES_ECB */ #ifdef HAVE_AES_CBC #ifndef __APPLE__ .text .globl AES_CBC_encrypt_NEON .type AES_CBC_encrypt_NEON,@function .align 2 AES_CBC_encrypt_NEON: #else .section __TEXT,__text .globl _AES_CBC_encrypt_NEON .p2align 2 _AES_CBC_encrypt_NEON: #endif /* __APPLE__ */ stp x29, x30, [sp, #-80]! add x29, sp, #0 stp d8, d9, [x29, #16] stp d10, d11, [x29, #32] stp d12, d13, [x29, #48] stp d14, d15, [x29, #64] #ifndef __APPLE__ adrp x6, L_AES_ARM64_NEON_te add x6, x6, :lo12:L_AES_ARM64_NEON_te #else adrp x6, L_AES_ARM64_NEON_te@PAGE add x6, x6, L_AES_ARM64_NEON_te@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x7, L_AES_ARM64_NEON_shift_rows_shuffle add x7, x7, :lo12:L_AES_ARM64_NEON_shift_rows_shuffle #else adrp x7, L_AES_ARM64_NEON_shift_rows_shuffle@PAGE add x7, x7, L_AES_ARM64_NEON_shift_rows_shuffle@PAGEOFF #endif /* __APPLE__ */ ld1 {v10.16b, v11.16b, v12.16b, v13.16b}, [x6], #0x40 ld1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x6], #0x40 ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x6], #0x40 ld1 {v22.16b, v23.16b, v24.16b, v25.16b}, [x6] movi v6.16b, #0x40 movi v7.16b, #0x80 movi v8.16b, #0xc0 movi v9.16b, #27 ld1 {v0.2d}, [x5] ld1 {v26.2d}, [x7] L_AES_CBC_encrypt_NEON_loop_block: add x9, x3, #16 ld1 {v1.16b}, [x0], #16 ld1 {v2.16b}, [x3] eor v0.16b, v0.16b, v1.16b rev32 v0.16b, v0.16b # Round: 0 - XOR in key schedule eor v0.16b, v0.16b, v2.16b sub w8, w4, #2 L_AES_CBC_encrypt_NEON_loop_nr: eor v2.16b, v0.16b, v6.16b eor v3.16b, v0.16b, v7.16b eor v4.16b, v0.16b, v8.16b tbl v1.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v0.16b tbl v2.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v2.16b tbl v3.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v3.16b tbl v4.16b, {v22.16b, v23.16b, v24.16b, v25.16b}, v4.16b orr v1.16b, v1.16b, v2.16b orr v3.16b, v3.16b, v4.16b orr v1.16b, v1.16b, v3.16b tbl v1.16b, {v1.16b}, v26.16b ld1 {v0.2d}, [x9], #16 sshr v4.16b, v1.16b, #7 shl v3.16b, v1.16b, #1 and v4.16b, v4.16b, v9.16b eor v4.16b, v4.16b, v3.16b rev32 v2.8h, v1.8h eor v5.16b, v4.16b, v1.16b eor v4.16b, v4.16b, v2.16b shl v3.4s, v1.4s, #24 shl v2.4s, v5.4s, #8 # XOR in Key Schedule eor v4.16b, v4.16b, v0.16b sri v3.4s, v1.4s, #8 sri v2.4s, v5.4s, #24 eor v1.16b, v4.16b, v3.16b eor v1.16b, v1.16b, v2.16b eor v2.16b, v1.16b, v6.16b eor v3.16b, v1.16b, v7.16b eor v4.16b, v1.16b, v8.16b tbl v0.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v1.16b tbl v2.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v2.16b tbl v3.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v3.16b tbl v4.16b, {v22.16b, v23.16b, v24.16b, v25.16b}, v4.16b orr v0.16b, v0.16b, v2.16b orr v3.16b, v3.16b, v4.16b orr v0.16b, v0.16b, v3.16b tbl v0.16b, {v0.16b}, v26.16b ld1 {v1.2d}, [x9], #16 sshr v4.16b, v0.16b, #7 shl v3.16b, v0.16b, #1 and v4.16b, v4.16b, v9.16b eor v4.16b, v4.16b, v3.16b rev32 v2.8h, v0.8h eor v5.16b, v4.16b, v0.16b eor v4.16b, v4.16b, v2.16b shl v3.4s, v0.4s, #24 shl v2.4s, v5.4s, #8 # XOR in Key Schedule eor v4.16b, v4.16b, v1.16b sri v3.4s, v0.4s, #8 sri v2.4s, v5.4s, #24 eor v0.16b, v4.16b, v3.16b eor v0.16b, v0.16b, v2.16b subs w8, w8, #2 bne L_AES_CBC_encrypt_NEON_loop_nr eor v2.16b, v0.16b, v6.16b eor v3.16b, v0.16b, v7.16b eor v4.16b, v0.16b, v8.16b tbl v1.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v0.16b tbl v2.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v2.16b tbl v3.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v3.16b tbl v4.16b, {v22.16b, v23.16b, v24.16b, v25.16b}, v4.16b orr v1.16b, v1.16b, v2.16b orr v3.16b, v3.16b, v4.16b orr v1.16b, v1.16b, v3.16b tbl v1.16b, {v1.16b}, v26.16b ld1 {v0.2d}, [x9], #16 sshr v4.16b, v1.16b, #7 shl v3.16b, v1.16b, #1 and v4.16b, v4.16b, v9.16b eor v4.16b, v4.16b, v3.16b rev32 v2.8h, v1.8h eor v5.16b, v4.16b, v1.16b eor v4.16b, v4.16b, v2.16b shl v3.4s, v1.4s, #24 shl v2.4s, v5.4s, #8 # XOR in Key Schedule eor v4.16b, v4.16b, v0.16b sri v3.4s, v1.4s, #8 sri v2.4s, v5.4s, #24 eor v1.16b, v4.16b, v3.16b eor v1.16b, v1.16b, v2.16b eor v2.16b, v1.16b, v6.16b eor v3.16b, v1.16b, v7.16b eor v4.16b, v1.16b, v8.16b tbl v0.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v1.16b tbl v2.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v2.16b tbl v3.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v3.16b tbl v4.16b, {v22.16b, v23.16b, v24.16b, v25.16b}, v4.16b orr v0.16b, v0.16b, v2.16b orr v3.16b, v3.16b, v4.16b orr v0.16b, v0.16b, v3.16b tbl v0.16b, {v0.16b}, v26.16b ld1 {v1.2d}, [x9], #16 # XOR in Key Schedule eor v0.16b, v0.16b, v1.16b rev32 v0.16b, v0.16b st1 {v0.16b}, [x1], #16 subs x2, x2, #16 bne L_AES_CBC_encrypt_NEON_loop_block st1 {v0.2d}, [x5] ldp d8, d9, [x29, #16] ldp d10, d11, [x29, #32] ldp d12, d13, [x29, #48] ldp d14, d15, [x29, #64] ldp x29, x30, [sp], #0x50 ret #ifndef __APPLE__ .size AES_CBC_encrypt_NEON,.-AES_CBC_encrypt_NEON #endif /* __APPLE__ */ #endif /* HAVE_AES_CBC */ #ifdef WOLFSSL_AES_COUNTER #ifndef __APPLE__ .text .globl AES_CTR_encrypt_NEON .type AES_CTR_encrypt_NEON,@function .align 2 AES_CTR_encrypt_NEON: #else .section __TEXT,__text .globl _AES_CTR_encrypt_NEON .p2align 2 _AES_CTR_encrypt_NEON: #endif /* __APPLE__ */ stp x29, x30, [sp, #-80]! add x29, sp, #0 stp d8, d9, [x29, #16] stp d10, d11, [x29, #32] stp d12, d13, [x29, #48] stp d14, d15, [x29, #64] #ifndef __APPLE__ adrp x6, L_AES_ARM64_NEON_te add x6, x6, :lo12:L_AES_ARM64_NEON_te #else adrp x6, L_AES_ARM64_NEON_te@PAGE add x6, x6, L_AES_ARM64_NEON_te@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x7, L_AES_ARM64_NEON_shift_rows_shuffle add x7, x7, :lo12:L_AES_ARM64_NEON_shift_rows_shuffle #else adrp x7, L_AES_ARM64_NEON_shift_rows_shuffle@PAGE add x7, x7, L_AES_ARM64_NEON_shift_rows_shuffle@PAGEOFF #endif /* __APPLE__ */ ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x6], #0x40 ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x6], #0x40 ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x6], #0x40 ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x6] ld1 {v2.2d}, [x5] rev64 v8.16b, v2.16b rev32 v2.16b, v2.16b mov x10, v8.d[1] mov x11, v8.d[0] cmp x2, #0x40 blt L_AES_CTR_encrypt_NEON_start_2 L_AES_CTR_encrypt_NEON_loop_4: mov x9, x3 ld1 {v4.2d}, [x9], #16 mov v8.d[1], x10 mov v8.d[0], x11 rev64 v8.16b, v8.16b rev32 v8.16b, v8.16b # Round: 0 - XOR in key schedule eor v0.16b, v8.16b, v4.16b adds x10, x10, #1 adc x11, x11, xzr mov v8.d[1], x10 mov v8.d[0], x11 rev64 v8.16b, v8.16b rev32 v8.16b, v8.16b eor v1.16b, v8.16b, v4.16b adds x10, x10, #1 adc x11, x11, xzr mov v8.d[1], x10 mov v8.d[0], x11 rev64 v8.16b, v8.16b rev32 v8.16b, v8.16b eor v2.16b, v8.16b, v4.16b adds x10, x10, #1 adc x11, x11, xzr mov v8.d[1], x10 mov v8.d[0], x11 rev64 v8.16b, v8.16b rev32 v8.16b, v8.16b eor v3.16b, v8.16b, v4.16b adds x10, x10, #1 adc x11, x11, xzr mov v8.d[1], x10 mov v8.d[0], x11 rev64 v8.16b, v8.16b rev32 v8.16b, v8.16b sub w8, w4, #2 L_AES_CTR_encrypt_NEON_loop_nr_4: tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b eor v10.16b, v2.16b, v12.16b eor v11.16b, v3.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v13.16b eor v9.16b, v1.16b, v13.16b eor v10.16b, v2.16b, v13.16b eor v11.16b, v3.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b eor v10.16b, v2.16b, v14.16b eor v11.16b, v3.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b ld1 {v0.16b}, [x7] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b tbl v6.16b, {v6.16b}, v0.16b tbl v7.16b, {v7.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 sshr v10.16b, v6.16b, #7 sshr v11.16b, v7.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 shl v14.16b, v6.16b, #1 shl v15.16b, v7.16b, #1 movi v0.16b, #27 and v8.16b, v8.16b, v0.16b and v9.16b, v9.16b, v0.16b and v10.16b, v10.16b, v0.16b and v11.16b, v11.16b, v0.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b eor v2.16b, v10.16b, v6.16b eor v3.16b, v11.16b, v7.16b shl v12.4s, v0.4s, #8 shl v13.4s, v1.4s, #8 shl v14.4s, v2.4s, #8 shl v15.4s, v3.4s, #8 sri v12.4s, v0.4s, #24 sri v13.4s, v1.4s, #24 sri v14.4s, v2.4s, #24 sri v15.4s, v3.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 shl v2.4s, v6.4s, #24 shl v3.4s, v7.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 sri v2.4s, v6.4s, #8 sri v3.4s, v7.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h rev32 v6.8h, v6.8h rev32 v7.8h, v7.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b eor v6.16b, v6.16b, v2.16b eor v7.16b, v7.16b, v3.16b # XOR in Key Schedule ld1 {v0.2d}, [x9], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v6.16b, v6.16b, v0.16b eor v7.16b, v7.16b, v0.16b eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b eor v6.16b, v6.16b, v14.16b eor v7.16b, v7.16b, v15.16b # Round Done tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b eor v10.16b, v6.16b, v12.16b eor v11.16b, v7.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v13.16b eor v9.16b, v5.16b, v13.16b eor v10.16b, v6.16b, v13.16b eor v11.16b, v7.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b eor v10.16b, v6.16b, v14.16b eor v11.16b, v7.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b ld1 {v4.16b}, [x7] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b tbl v2.16b, {v2.16b}, v4.16b tbl v3.16b, {v3.16b}, v4.16b sshr v8.16b, v0.16b, #7 sshr v9.16b, v1.16b, #7 sshr v10.16b, v2.16b, #7 sshr v11.16b, v3.16b, #7 shl v12.16b, v0.16b, #1 shl v13.16b, v1.16b, #1 shl v14.16b, v2.16b, #1 shl v15.16b, v3.16b, #1 movi v4.16b, #27 and v8.16b, v8.16b, v4.16b and v9.16b, v9.16b, v4.16b and v10.16b, v10.16b, v4.16b and v11.16b, v11.16b, v4.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b eor v4.16b, v8.16b, v0.16b eor v5.16b, v9.16b, v1.16b eor v6.16b, v10.16b, v2.16b eor v7.16b, v11.16b, v3.16b shl v12.4s, v4.4s, #8 shl v13.4s, v5.4s, #8 shl v14.4s, v6.4s, #8 shl v15.4s, v7.4s, #8 sri v12.4s, v4.4s, #24 sri v13.4s, v5.4s, #24 sri v14.4s, v6.4s, #24 sri v15.4s, v7.4s, #24 shl v4.4s, v0.4s, #24 shl v5.4s, v1.4s, #24 shl v6.4s, v2.4s, #24 shl v7.4s, v3.4s, #24 sri v4.4s, v0.4s, #8 sri v5.4s, v1.4s, #8 sri v6.4s, v2.4s, #8 sri v7.4s, v3.4s, #8 rev32 v0.8h, v0.8h rev32 v1.8h, v1.8h rev32 v2.8h, v2.8h rev32 v3.8h, v3.8h eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b # XOR in Key Schedule ld1 {v4.2d}, [x9], #16 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b eor v0.16b, v0.16b, v12.16b eor v1.16b, v1.16b, v13.16b eor v2.16b, v2.16b, v14.16b eor v3.16b, v3.16b, v15.16b # Round Done subs w8, w8, #2 bne L_AES_CTR_encrypt_NEON_loop_nr_4 tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b eor v10.16b, v2.16b, v12.16b eor v11.16b, v3.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v13.16b eor v9.16b, v1.16b, v13.16b eor v10.16b, v2.16b, v13.16b eor v11.16b, v3.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b eor v10.16b, v2.16b, v14.16b eor v11.16b, v3.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b ld1 {v0.16b}, [x7] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b tbl v6.16b, {v6.16b}, v0.16b tbl v7.16b, {v7.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 sshr v10.16b, v6.16b, #7 sshr v11.16b, v7.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 shl v14.16b, v6.16b, #1 shl v15.16b, v7.16b, #1 movi v0.16b, #27 and v8.16b, v8.16b, v0.16b and v9.16b, v9.16b, v0.16b and v10.16b, v10.16b, v0.16b and v11.16b, v11.16b, v0.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b eor v2.16b, v10.16b, v6.16b eor v3.16b, v11.16b, v7.16b shl v12.4s, v0.4s, #8 shl v13.4s, v1.4s, #8 shl v14.4s, v2.4s, #8 shl v15.4s, v3.4s, #8 sri v12.4s, v0.4s, #24 sri v13.4s, v1.4s, #24 sri v14.4s, v2.4s, #24 sri v15.4s, v3.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 shl v2.4s, v6.4s, #24 shl v3.4s, v7.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 sri v2.4s, v6.4s, #8 sri v3.4s, v7.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h rev32 v6.8h, v6.8h rev32 v7.8h, v7.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b eor v6.16b, v6.16b, v2.16b eor v7.16b, v7.16b, v3.16b # XOR in Key Schedule ld1 {v0.2d}, [x9], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v6.16b, v6.16b, v0.16b eor v7.16b, v7.16b, v0.16b eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b eor v6.16b, v6.16b, v14.16b eor v7.16b, v7.16b, v15.16b # Round Done tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b eor v10.16b, v6.16b, v12.16b eor v11.16b, v7.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v13.16b eor v9.16b, v5.16b, v13.16b eor v10.16b, v6.16b, v13.16b eor v11.16b, v7.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b eor v10.16b, v6.16b, v14.16b eor v11.16b, v7.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b ld1 {v4.16b}, [x7] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b tbl v2.16b, {v2.16b}, v4.16b tbl v3.16b, {v3.16b}, v4.16b # XOR in Key Schedule ld1 {v4.2d}, [x9], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b # Round Done rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b rev32 v2.16b, v2.16b rev32 v3.16b, v3.16b ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 sub x2, x2, #0x40 cmp x2, #0x40 bge L_AES_CTR_encrypt_NEON_loop_4 mov v2.d[1], x10 mov v2.d[0], x11 rev64 v2.16b, v2.16b rev32 v2.16b, v2.16b L_AES_CTR_encrypt_NEON_start_2: movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 movi v15.16b, #27 cmp x2, #16 beq L_AES_CTR_encrypt_NEON_start_1 blt L_AES_CTR_encrypt_NEON_data_done L_AES_CTR_encrypt_NEON_loop_2: mov x9, x3 ld1 {v4.2d}, [x9], #16 # Round: 0 - XOR in key schedule eor v0.16b, v2.16b, v4.16b adds x10, x10, #1 adc x11, x11, xzr mov v2.d[1], x10 mov v2.d[0], x11 rev64 v2.16b, v2.16b rev32 v2.16b, v2.16b eor v1.16b, v2.16b, v4.16b adds x10, x10, #1 adc x11, x11, xzr mov v2.d[1], x10 mov v2.d[0], x11 rev64 v2.16b, v2.16b rev32 v2.16b, v2.16b sub w8, w4, #2 L_AES_CTR_encrypt_NEON_loop_nr_2: eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v0.16b, v13.16b eor v11.16b, v1.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b orr v4.16b, v4.16b, v10.16b orr v5.16b, v5.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b ld1 {v0.16b}, [x7] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 shl v10.16b, v4.16b, #1 shl v11.16b, v5.16b, #1 and v8.16b, v8.16b, v15.16b and v9.16b, v9.16b, v15.16b eor v8.16b, v8.16b, v10.16b eor v9.16b, v9.16b, v11.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b shl v10.4s, v0.4s, #8 shl v11.4s, v1.4s, #8 sri v10.4s, v0.4s, #24 sri v11.4s, v1.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b # XOR in Key Schedule ld1 {v0.2d}, [x9], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v4.16b, v4.16b, v10.16b eor v5.16b, v5.16b, v11.16b # Round Done eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v4.16b, v13.16b eor v11.16b, v5.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b orr v0.16b, v0.16b, v10.16b orr v1.16b, v1.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b ld1 {v4.16b}, [x7] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b sshr v8.16b, v0.16b, #7 sshr v9.16b, v1.16b, #7 shl v10.16b, v0.16b, #1 shl v11.16b, v1.16b, #1 and v8.16b, v8.16b, v15.16b and v9.16b, v9.16b, v15.16b eor v8.16b, v8.16b, v10.16b eor v9.16b, v9.16b, v11.16b eor v4.16b, v8.16b, v0.16b eor v5.16b, v9.16b, v1.16b shl v10.4s, v4.4s, #8 shl v11.4s, v5.4s, #8 sri v10.4s, v4.4s, #24 sri v11.4s, v5.4s, #24 shl v4.4s, v0.4s, #24 shl v5.4s, v1.4s, #24 sri v4.4s, v0.4s, #8 sri v5.4s, v1.4s, #8 rev32 v0.8h, v0.8h rev32 v1.8h, v1.8h eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b # XOR in Key Schedule ld1 {v4.2d}, [x9], #16 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v0.16b, v0.16b, v10.16b eor v1.16b, v1.16b, v11.16b # Round Done subs w8, w8, #2 bne L_AES_CTR_encrypt_NEON_loop_nr_2 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v0.16b, v13.16b eor v11.16b, v1.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b orr v4.16b, v4.16b, v10.16b orr v5.16b, v5.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b ld1 {v0.16b}, [x7] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 shl v10.16b, v4.16b, #1 shl v11.16b, v5.16b, #1 and v8.16b, v8.16b, v15.16b and v9.16b, v9.16b, v15.16b eor v8.16b, v8.16b, v10.16b eor v9.16b, v9.16b, v11.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b shl v10.4s, v0.4s, #8 shl v11.4s, v1.4s, #8 sri v10.4s, v0.4s, #24 sri v11.4s, v1.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b # XOR in Key Schedule ld1 {v0.2d}, [x9], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v4.16b, v4.16b, v10.16b eor v5.16b, v5.16b, v11.16b # Round Done eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v4.16b, v13.16b eor v11.16b, v5.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b orr v0.16b, v0.16b, v10.16b orr v1.16b, v1.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b ld1 {v4.16b}, [x7] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b # XOR in Key Schedule ld1 {v4.2d}, [x9], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b # Round Done rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b ld1 {v4.16b, v5.16b}, [x0], #32 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b st1 {v0.16b, v1.16b}, [x1], #32 sub x2, x2, #32 cmp x2, #0 beq L_AES_CTR_encrypt_NEON_data_done L_AES_CTR_encrypt_NEON_start_1: ld1 {v3.2d}, [x7] mov x9, x3 ld1 {v4.2d}, [x9], #16 # Round: 0 - XOR in key schedule eor v0.16b, v2.16b, v4.16b sub w8, w4, #2 L_AES_CTR_encrypt_NEON_loop_nr_1: eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b ld1 {v0.2d}, [x9], #16 sshr v10.16b, v4.16b, #7 shl v9.16b, v4.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v4.8h eor v11.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v4.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v0.16b sri v9.4s, v4.4s, #8 sri v8.4s, v11.4s, #24 eor v4.16b, v10.16b, v9.16b eor v4.16b, v4.16b, v8.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b ld1 {v4.2d}, [x9], #16 sshr v10.16b, v0.16b, #7 shl v9.16b, v0.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v0.8h eor v11.16b, v10.16b, v0.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v0.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v4.16b sri v9.4s, v0.4s, #8 sri v8.4s, v11.4s, #24 eor v0.16b, v10.16b, v9.16b eor v0.16b, v0.16b, v8.16b subs w8, w8, #2 bne L_AES_CTR_encrypt_NEON_loop_nr_1 eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b ld1 {v0.2d}, [x9], #16 sshr v10.16b, v4.16b, #7 shl v9.16b, v4.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v4.8h eor v11.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v4.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v0.16b sri v9.4s, v4.4s, #8 sri v8.4s, v11.4s, #24 eor v4.16b, v10.16b, v9.16b eor v4.16b, v4.16b, v8.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b ld1 {v4.2d}, [x9], #16 # XOR in Key Schedule eor v0.16b, v0.16b, v4.16b rev32 v0.16b, v0.16b ld1 {v4.16b}, [x0], #16 eor v0.16b, v0.16b, v4.16b st1 {v0.16b}, [x1], #16 adds x10, x10, #1 adc x11, x11, xzr mov v2.d[1], x10 mov v2.d[0], x11 rev64 v2.16b, v2.16b rev32 v2.16b, v2.16b L_AES_CTR_encrypt_NEON_data_done: rev32 v2.16b, v2.16b st1 {v2.2d}, [x5] ldp d8, d9, [x29, #16] ldp d10, d11, [x29, #32] ldp d12, d13, [x29, #48] ldp d14, d15, [x29, #64] ldp x29, x30, [sp], #0x50 ret #ifndef __APPLE__ .size AES_CTR_encrypt_NEON,.-AES_CTR_encrypt_NEON #endif /* __APPLE__ */ #endif /* WOLFSSL_AES_COUNTER */ #ifdef HAVE_AES_DECRYPT #if defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || \ defined(HAVE_AES_CBC) || defined(HAVE_AES_ECB) #ifndef __APPLE__ .text .section .rodata .type L_AES_ARM64_NEON_td, %object .size L_AES_ARM64_NEON_td, 256 #else .section __DATA,__data #endif /* __APPLE__ */ # 8-byte aligned, 64-bit aligned #ifndef __APPLE__ .align 3 #else .p2align 3 #endif /* __APPLE__ */ L_AES_ARM64_NEON_td: .byte 0x52,0x09,0x6a,0xd5,0x30,0x36,0xa5,0x38 .byte 0xbf,0x40,0xa3,0x9e,0x81,0xf3,0xd7,0xfb .byte 0x7c,0xe3,0x39,0x82,0x9b,0x2f,0xff,0x87 .byte 0x34,0x8e,0x43,0x44,0xc4,0xde,0xe9,0xcb .byte 0x54,0x7b,0x94,0x32,0xa6,0xc2,0x23,0x3d .byte 0xee,0x4c,0x95,0x0b,0x42,0xfa,0xc3,0x4e .byte 0x08,0x2e,0xa1,0x66,0x28,0xd9,0x24,0xb2 .byte 0x76,0x5b,0xa2,0x49,0x6d,0x8b,0xd1,0x25 .byte 0x72,0xf8,0xf6,0x64,0x86,0x68,0x98,0x16 .byte 0xd4,0xa4,0x5c,0xcc,0x5d,0x65,0xb6,0x92 .byte 0x6c,0x70,0x48,0x50,0xfd,0xed,0xb9,0xda .byte 0x5e,0x15,0x46,0x57,0xa7,0x8d,0x9d,0x84 .byte 0x90,0xd8,0xab,0x00,0x8c,0xbc,0xd3,0x0a .byte 0xf7,0xe4,0x58,0x05,0xb8,0xb3,0x45,0x06 .byte 0xd0,0x2c,0x1e,0x8f,0xca,0x3f,0x0f,0x02 .byte 0xc1,0xaf,0xbd,0x03,0x01,0x13,0x8a,0x6b .byte 0x3a,0x91,0x11,0x41,0x4f,0x67,0xdc,0xea .byte 0x97,0xf2,0xcf,0xce,0xf0,0xb4,0xe6,0x73 .byte 0x96,0xac,0x74,0x22,0xe7,0xad,0x35,0x85 .byte 0xe2,0xf9,0x37,0xe8,0x1c,0x75,0xdf,0x6e .byte 0x47,0xf1,0x1a,0x71,0x1d,0x29,0xc5,0x89 .byte 0x6f,0xb7,0x62,0x0e,0xaa,0x18,0xbe,0x1b .byte 0xfc,0x56,0x3e,0x4b,0xc6,0xd2,0x79,0x20 .byte 0x9a,0xdb,0xc0,0xfe,0x78,0xcd,0x5a,0xf4 .byte 0x1f,0xdd,0xa8,0x33,0x88,0x07,0xc7,0x31 .byte 0xb1,0x12,0x10,0x59,0x27,0x80,0xec,0x5f .byte 0x60,0x51,0x7f,0xa9,0x19,0xb5,0x4a,0x0d .byte 0x2d,0xe5,0x7a,0x9f,0x93,0xc9,0x9c,0xef .byte 0xa0,0xe0,0x3b,0x4d,0xae,0x2a,0xf5,0xb0 .byte 0xc8,0xeb,0xbb,0x3c,0x83,0x53,0x99,0x61 .byte 0x17,0x2b,0x04,0x7e,0xba,0x77,0xd6,0x26 .byte 0xe1,0x69,0x14,0x63,0x55,0x21,0x0c,0x7d #ifndef __APPLE__ .text .section .rodata .type L_AES_ARM64_NEON_shift_rows_invshuffle, %object .size L_AES_ARM64_NEON_shift_rows_invshuffle, 16 #else .section __DATA,__data #endif /* __APPLE__ */ # 8-byte aligned, 64-bit aligned #ifndef __APPLE__ .align 3 #else .p2align 3 #endif /* __APPLE__ */ L_AES_ARM64_NEON_shift_rows_invshuffle: .byte 0x04,0x09,0x0e,0x03,0x08,0x0d,0x02,0x07 .byte 0x0c,0x01,0x06,0x0b,0x00,0x05,0x0a,0x0f #if defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || defined(HAVE_AES_ECB) #ifndef __APPLE__ .text .globl AES_ECB_decrypt_NEON .type AES_ECB_decrypt_NEON,@function .align 2 AES_ECB_decrypt_NEON: #else .section __TEXT,__text .globl _AES_ECB_decrypt_NEON .p2align 2 _AES_ECB_decrypt_NEON: #endif /* __APPLE__ */ stp x29, x30, [sp, #-80]! add x29, sp, #0 stp d8, d9, [x29, #16] stp d10, d11, [x29, #32] stp d12, d13, [x29, #48] stp d14, d15, [x29, #64] #ifndef __APPLE__ adrp x5, L_AES_ARM64_NEON_td add x5, x5, :lo12:L_AES_ARM64_NEON_td #else adrp x5, L_AES_ARM64_NEON_td@PAGE add x5, x5, L_AES_ARM64_NEON_td@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x6, L_AES_ARM64_NEON_shift_rows_invshuffle add x6, x6, :lo12:L_AES_ARM64_NEON_shift_rows_invshuffle #else adrp x6, L_AES_ARM64_NEON_shift_rows_invshuffle@PAGE add x6, x6, L_AES_ARM64_NEON_shift_rows_invshuffle@PAGEOFF #endif /* __APPLE__ */ ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x5], #0x40 ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x5], #0x40 ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x5], #0x40 ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x5] cmp x2, #0x40 blt L_AES_ECB_decrypt_NEON_start_2 L_AES_ECB_decrypt_NEON_loop_4: mov x8, x3 ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 ld1 {v4.2d}, [x8], #16 rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b rev32 v2.16b, v2.16b rev32 v3.16b, v3.16b # Round: 0 - XOR in key schedule eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b sub w7, w4, #2 L_AES_ECB_decrypt_NEON_loop_nr_4: tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b eor v10.16b, v2.16b, v12.16b eor v11.16b, v3.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v13.16b eor v9.16b, v1.16b, v13.16b eor v10.16b, v2.16b, v13.16b eor v11.16b, v3.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b eor v10.16b, v2.16b, v14.16b eor v11.16b, v3.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b ld1 {v0.16b}, [x6] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b tbl v6.16b, {v6.16b}, v0.16b tbl v7.16b, {v7.16b}, v0.16b movi v28.16b, #27 sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 sshr v10.16b, v6.16b, #7 sshr v11.16b, v7.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 shl v14.16b, v6.16b, #1 shl v15.16b, v7.16b, #1 and v8.16b, v8.16b, v28.16b and v9.16b, v9.16b, v28.16b and v10.16b, v10.16b, v28.16b and v11.16b, v11.16b, v28.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b ushr v12.16b, v4.16b, #6 ushr v13.16b, v5.16b, #6 ushr v14.16b, v6.16b, #6 ushr v15.16b, v7.16b, #6 shl v0.16b, v4.16b, #2 shl v1.16b, v5.16b, #2 shl v2.16b, v6.16b, #2 shl v3.16b, v7.16b, #2 pmul v12.16b, v12.16b, v28.16b pmul v13.16b, v13.16b, v28.16b pmul v14.16b, v14.16b, v28.16b pmul v15.16b, v15.16b, v28.16b eor v12.16b, v12.16b, v0.16b eor v13.16b, v13.16b, v1.16b eor v14.16b, v14.16b, v2.16b eor v15.16b, v15.16b, v3.16b ushr v0.16b, v4.16b, #5 ushr v1.16b, v5.16b, #5 ushr v2.16b, v6.16b, #5 ushr v3.16b, v7.16b, #5 pmul v0.16b, v0.16b, v28.16b pmul v1.16b, v1.16b, v28.16b pmul v2.16b, v2.16b, v28.16b pmul v3.16b, v3.16b, v28.16b shl v28.16b, v4.16b, #3 shl v29.16b, v5.16b, #3 shl v30.16b, v6.16b, #3 shl v31.16b, v7.16b, #3 eor v0.16b, v0.16b, v28.16b eor v1.16b, v1.16b, v29.16b eor v2.16b, v2.16b, v30.16b eor v3.16b, v3.16b, v31.16b eor v28.16b, v8.16b, v0.16b eor v29.16b, v9.16b, v1.16b eor v30.16b, v10.16b, v2.16b eor v31.16b, v11.16b, v3.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b eor v8.16b, v12.16b, v0.16b eor v9.16b, v13.16b, v1.16b eor v10.16b, v14.16b, v2.16b eor v11.16b, v15.16b, v3.16b eor v12.16b, v12.16b, v28.16b eor v13.16b, v13.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v15.16b, v15.16b, v31.16b eor v28.16b, v28.16b, v4.16b eor v29.16b, v29.16b, v5.16b eor v30.16b, v30.16b, v6.16b eor v31.16b, v31.16b, v7.16b shl v4.4s, v28.4s, #8 shl v5.4s, v29.4s, #8 shl v6.4s, v30.4s, #8 shl v7.4s, v31.4s, #8 rev32 v8.8h, v8.8h rev32 v9.8h, v9.8h rev32 v10.8h, v10.8h rev32 v11.8h, v11.8h sri v4.4s, v28.4s, #24 sri v5.4s, v29.4s, #24 sri v6.4s, v30.4s, #24 sri v7.4s, v31.4s, #24 eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b eor v6.16b, v6.16b, v14.16b eor v7.16b, v7.16b, v15.16b shl v28.4s, v0.4s, #24 shl v29.4s, v1.4s, #24 shl v30.4s, v2.4s, #24 shl v31.4s, v3.4s, #24 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b sri v28.4s, v0.4s, #8 sri v29.4s, v1.4s, #8 sri v30.4s, v2.4s, #8 sri v31.4s, v3.4s, #8 eor v4.16b, v4.16b, v28.16b eor v5.16b, v5.16b, v29.16b eor v6.16b, v6.16b, v30.16b eor v7.16b, v7.16b, v31.16b ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x5] # XOR in Key Schedule ld1 {v0.2d}, [x8], #16 eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v6.16b, v6.16b, v0.16b eor v7.16b, v7.16b, v0.16b # Round Done tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b eor v10.16b, v6.16b, v12.16b eor v11.16b, v7.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v13.16b eor v9.16b, v5.16b, v13.16b eor v10.16b, v6.16b, v13.16b eor v11.16b, v7.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b eor v10.16b, v6.16b, v14.16b eor v11.16b, v7.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b ld1 {v4.16b}, [x6] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b tbl v2.16b, {v2.16b}, v4.16b tbl v3.16b, {v3.16b}, v4.16b movi v28.16b, #27 sshr v8.16b, v0.16b, #7 sshr v9.16b, v1.16b, #7 sshr v10.16b, v2.16b, #7 sshr v11.16b, v3.16b, #7 shl v12.16b, v0.16b, #1 shl v13.16b, v1.16b, #1 shl v14.16b, v2.16b, #1 shl v15.16b, v3.16b, #1 and v8.16b, v8.16b, v28.16b and v9.16b, v9.16b, v28.16b and v10.16b, v10.16b, v28.16b and v11.16b, v11.16b, v28.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b ushr v12.16b, v0.16b, #6 ushr v13.16b, v1.16b, #6 ushr v14.16b, v2.16b, #6 ushr v15.16b, v3.16b, #6 shl v4.16b, v0.16b, #2 shl v5.16b, v1.16b, #2 shl v6.16b, v2.16b, #2 shl v7.16b, v3.16b, #2 pmul v12.16b, v12.16b, v28.16b pmul v13.16b, v13.16b, v28.16b pmul v14.16b, v14.16b, v28.16b pmul v15.16b, v15.16b, v28.16b eor v12.16b, v12.16b, v4.16b eor v13.16b, v13.16b, v5.16b eor v14.16b, v14.16b, v6.16b eor v15.16b, v15.16b, v7.16b ushr v4.16b, v0.16b, #5 ushr v5.16b, v1.16b, #5 ushr v6.16b, v2.16b, #5 ushr v7.16b, v3.16b, #5 pmul v4.16b, v4.16b, v28.16b pmul v5.16b, v5.16b, v28.16b pmul v6.16b, v6.16b, v28.16b pmul v7.16b, v7.16b, v28.16b shl v28.16b, v0.16b, #3 shl v29.16b, v1.16b, #3 shl v30.16b, v2.16b, #3 shl v31.16b, v3.16b, #3 eor v4.16b, v4.16b, v28.16b eor v5.16b, v5.16b, v29.16b eor v6.16b, v6.16b, v30.16b eor v7.16b, v7.16b, v31.16b eor v28.16b, v8.16b, v4.16b eor v29.16b, v9.16b, v5.16b eor v30.16b, v10.16b, v6.16b eor v31.16b, v11.16b, v7.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b eor v6.16b, v6.16b, v2.16b eor v7.16b, v7.16b, v3.16b eor v8.16b, v12.16b, v4.16b eor v9.16b, v13.16b, v5.16b eor v10.16b, v14.16b, v6.16b eor v11.16b, v15.16b, v7.16b eor v12.16b, v12.16b, v28.16b eor v13.16b, v13.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v15.16b, v15.16b, v31.16b eor v28.16b, v28.16b, v0.16b eor v29.16b, v29.16b, v1.16b eor v30.16b, v30.16b, v2.16b eor v31.16b, v31.16b, v3.16b shl v0.4s, v28.4s, #8 shl v1.4s, v29.4s, #8 shl v2.4s, v30.4s, #8 shl v3.4s, v31.4s, #8 rev32 v8.8h, v8.8h rev32 v9.8h, v9.8h rev32 v10.8h, v10.8h rev32 v11.8h, v11.8h sri v0.4s, v28.4s, #24 sri v1.4s, v29.4s, #24 sri v2.4s, v30.4s, #24 sri v3.4s, v31.4s, #24 eor v0.16b, v0.16b, v12.16b eor v1.16b, v1.16b, v13.16b eor v2.16b, v2.16b, v14.16b eor v3.16b, v3.16b, v15.16b shl v28.4s, v4.4s, #24 shl v29.4s, v5.4s, #24 shl v30.4s, v6.4s, #24 shl v31.4s, v7.4s, #24 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b sri v28.4s, v4.4s, #8 sri v29.4s, v5.4s, #8 sri v30.4s, v6.4s, #8 sri v31.4s, v7.4s, #8 eor v0.16b, v0.16b, v28.16b eor v1.16b, v1.16b, v29.16b eor v2.16b, v2.16b, v30.16b eor v3.16b, v3.16b, v31.16b ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x5] # XOR in Key Schedule ld1 {v4.2d}, [x8], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b # Round Done subs w7, w7, #2 bne L_AES_ECB_decrypt_NEON_loop_nr_4 tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b eor v10.16b, v2.16b, v12.16b eor v11.16b, v3.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v13.16b eor v9.16b, v1.16b, v13.16b eor v10.16b, v2.16b, v13.16b eor v11.16b, v3.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b eor v10.16b, v2.16b, v14.16b eor v11.16b, v3.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b ld1 {v0.16b}, [x6] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b tbl v6.16b, {v6.16b}, v0.16b tbl v7.16b, {v7.16b}, v0.16b movi v28.16b, #27 sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 sshr v10.16b, v6.16b, #7 sshr v11.16b, v7.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 shl v14.16b, v6.16b, #1 shl v15.16b, v7.16b, #1 and v8.16b, v8.16b, v28.16b and v9.16b, v9.16b, v28.16b and v10.16b, v10.16b, v28.16b and v11.16b, v11.16b, v28.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b ushr v12.16b, v4.16b, #6 ushr v13.16b, v5.16b, #6 ushr v14.16b, v6.16b, #6 ushr v15.16b, v7.16b, #6 shl v0.16b, v4.16b, #2 shl v1.16b, v5.16b, #2 shl v2.16b, v6.16b, #2 shl v3.16b, v7.16b, #2 pmul v12.16b, v12.16b, v28.16b pmul v13.16b, v13.16b, v28.16b pmul v14.16b, v14.16b, v28.16b pmul v15.16b, v15.16b, v28.16b eor v12.16b, v12.16b, v0.16b eor v13.16b, v13.16b, v1.16b eor v14.16b, v14.16b, v2.16b eor v15.16b, v15.16b, v3.16b ushr v0.16b, v4.16b, #5 ushr v1.16b, v5.16b, #5 ushr v2.16b, v6.16b, #5 ushr v3.16b, v7.16b, #5 pmul v0.16b, v0.16b, v28.16b pmul v1.16b, v1.16b, v28.16b pmul v2.16b, v2.16b, v28.16b pmul v3.16b, v3.16b, v28.16b shl v28.16b, v4.16b, #3 shl v29.16b, v5.16b, #3 shl v30.16b, v6.16b, #3 shl v31.16b, v7.16b, #3 eor v0.16b, v0.16b, v28.16b eor v1.16b, v1.16b, v29.16b eor v2.16b, v2.16b, v30.16b eor v3.16b, v3.16b, v31.16b eor v28.16b, v8.16b, v0.16b eor v29.16b, v9.16b, v1.16b eor v30.16b, v10.16b, v2.16b eor v31.16b, v11.16b, v3.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b eor v8.16b, v12.16b, v0.16b eor v9.16b, v13.16b, v1.16b eor v10.16b, v14.16b, v2.16b eor v11.16b, v15.16b, v3.16b eor v12.16b, v12.16b, v28.16b eor v13.16b, v13.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v15.16b, v15.16b, v31.16b eor v28.16b, v28.16b, v4.16b eor v29.16b, v29.16b, v5.16b eor v30.16b, v30.16b, v6.16b eor v31.16b, v31.16b, v7.16b shl v4.4s, v28.4s, #8 shl v5.4s, v29.4s, #8 shl v6.4s, v30.4s, #8 shl v7.4s, v31.4s, #8 rev32 v8.8h, v8.8h rev32 v9.8h, v9.8h rev32 v10.8h, v10.8h rev32 v11.8h, v11.8h sri v4.4s, v28.4s, #24 sri v5.4s, v29.4s, #24 sri v6.4s, v30.4s, #24 sri v7.4s, v31.4s, #24 eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b eor v6.16b, v6.16b, v14.16b eor v7.16b, v7.16b, v15.16b shl v28.4s, v0.4s, #24 shl v29.4s, v1.4s, #24 shl v30.4s, v2.4s, #24 shl v31.4s, v3.4s, #24 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b sri v28.4s, v0.4s, #8 sri v29.4s, v1.4s, #8 sri v30.4s, v2.4s, #8 sri v31.4s, v3.4s, #8 eor v4.16b, v4.16b, v28.16b eor v5.16b, v5.16b, v29.16b eor v6.16b, v6.16b, v30.16b eor v7.16b, v7.16b, v31.16b ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x5] # XOR in Key Schedule ld1 {v0.2d}, [x8], #16 eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v6.16b, v6.16b, v0.16b eor v7.16b, v7.16b, v0.16b # Round Done tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b eor v10.16b, v6.16b, v12.16b eor v11.16b, v7.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v13.16b eor v9.16b, v5.16b, v13.16b eor v10.16b, v6.16b, v13.16b eor v11.16b, v7.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b eor v10.16b, v6.16b, v14.16b eor v11.16b, v7.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b ld1 {v4.16b}, [x6] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b tbl v2.16b, {v2.16b}, v4.16b tbl v3.16b, {v3.16b}, v4.16b # XOR in Key Schedule ld1 {v4.2d}, [x8], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b # Round Done rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b rev32 v2.16b, v2.16b rev32 v3.16b, v3.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 sub x2, x2, #0x40 cmp x2, #0x40 bge L_AES_ECB_decrypt_NEON_loop_4 L_AES_ECB_decrypt_NEON_start_2: cmp x2, #16 beq L_AES_ECB_decrypt_NEON_start_1 blt L_AES_ECB_decrypt_NEON_data_done L_AES_ECB_decrypt_NEON_loop_2: mov x8, x3 ld1 {v0.16b, v1.16b}, [x0], #32 ld1 {v4.2d}, [x8], #16 rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b # Round: 0 - XOR in key schedule eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b sub w7, w4, #2 L_AES_ECB_decrypt_NEON_loop_nr_2: movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v0.16b, v13.16b eor v11.16b, v1.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b orr v4.16b, v4.16b, v10.16b orr v5.16b, v5.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b ld1 {v0.16b}, [x6] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b movi v10.16b, #27 sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 and v8.16b, v8.16b, v10.16b and v9.16b, v9.16b, v10.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b ushr v12.16b, v4.16b, #6 ushr v13.16b, v5.16b, #6 shl v0.16b, v4.16b, #2 shl v1.16b, v5.16b, #2 pmul v12.16b, v12.16b, v10.16b pmul v13.16b, v13.16b, v10.16b eor v12.16b, v12.16b, v0.16b eor v13.16b, v13.16b, v1.16b ushr v0.16b, v4.16b, #5 ushr v1.16b, v5.16b, #5 pmul v0.16b, v0.16b, v10.16b pmul v1.16b, v1.16b, v10.16b shl v10.16b, v4.16b, #3 shl v11.16b, v5.16b, #3 eor v0.16b, v0.16b, v10.16b eor v1.16b, v1.16b, v11.16b eor v10.16b, v8.16b, v0.16b eor v11.16b, v9.16b, v1.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v8.16b, v12.16b, v0.16b eor v9.16b, v13.16b, v1.16b eor v12.16b, v12.16b, v10.16b eor v13.16b, v13.16b, v11.16b eor v10.16b, v10.16b, v4.16b eor v11.16b, v11.16b, v5.16b shl v4.4s, v10.4s, #8 shl v5.4s, v11.4s, #8 rev32 v8.8h, v8.8h rev32 v9.8h, v9.8h sri v4.4s, v10.4s, #24 sri v5.4s, v11.4s, #24 eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b shl v10.4s, v0.4s, #24 shl v11.4s, v1.4s, #24 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b sri v10.4s, v0.4s, #8 sri v11.4s, v1.4s, #8 eor v4.16b, v4.16b, v10.16b eor v5.16b, v5.16b, v11.16b # XOR in Key Schedule ld1 {v0.2d}, [x8], #16 eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b # Round Done movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v4.16b, v13.16b eor v11.16b, v5.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b orr v0.16b, v0.16b, v10.16b orr v1.16b, v1.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b ld1 {v4.16b}, [x6] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b movi v10.16b, #27 sshr v8.16b, v0.16b, #7 sshr v9.16b, v1.16b, #7 shl v12.16b, v0.16b, #1 shl v13.16b, v1.16b, #1 and v8.16b, v8.16b, v10.16b and v9.16b, v9.16b, v10.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b ushr v12.16b, v0.16b, #6 ushr v13.16b, v1.16b, #6 shl v4.16b, v0.16b, #2 shl v5.16b, v1.16b, #2 pmul v12.16b, v12.16b, v10.16b pmul v13.16b, v13.16b, v10.16b eor v12.16b, v12.16b, v4.16b eor v13.16b, v13.16b, v5.16b ushr v4.16b, v0.16b, #5 ushr v5.16b, v1.16b, #5 pmul v4.16b, v4.16b, v10.16b pmul v5.16b, v5.16b, v10.16b shl v10.16b, v0.16b, #3 shl v11.16b, v1.16b, #3 eor v4.16b, v4.16b, v10.16b eor v5.16b, v5.16b, v11.16b eor v10.16b, v8.16b, v4.16b eor v11.16b, v9.16b, v5.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b eor v8.16b, v12.16b, v4.16b eor v9.16b, v13.16b, v5.16b eor v12.16b, v12.16b, v10.16b eor v13.16b, v13.16b, v11.16b eor v10.16b, v10.16b, v0.16b eor v11.16b, v11.16b, v1.16b shl v0.4s, v10.4s, #8 shl v1.4s, v11.4s, #8 rev32 v8.8h, v8.8h rev32 v9.8h, v9.8h sri v0.4s, v10.4s, #24 sri v1.4s, v11.4s, #24 eor v0.16b, v0.16b, v12.16b eor v1.16b, v1.16b, v13.16b shl v10.4s, v4.4s, #24 shl v11.4s, v5.4s, #24 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b sri v10.4s, v4.4s, #8 sri v11.4s, v5.4s, #8 eor v0.16b, v0.16b, v10.16b eor v1.16b, v1.16b, v11.16b # XOR in Key Schedule ld1 {v4.2d}, [x8], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b # Round Done subs w7, w7, #2 bne L_AES_ECB_decrypt_NEON_loop_nr_2 movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v0.16b, v13.16b eor v11.16b, v1.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b orr v4.16b, v4.16b, v10.16b orr v5.16b, v5.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b ld1 {v0.16b}, [x6] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b movi v10.16b, #27 sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 and v8.16b, v8.16b, v10.16b and v9.16b, v9.16b, v10.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b ushr v12.16b, v4.16b, #6 ushr v13.16b, v5.16b, #6 shl v0.16b, v4.16b, #2 shl v1.16b, v5.16b, #2 pmul v12.16b, v12.16b, v10.16b pmul v13.16b, v13.16b, v10.16b eor v12.16b, v12.16b, v0.16b eor v13.16b, v13.16b, v1.16b ushr v0.16b, v4.16b, #5 ushr v1.16b, v5.16b, #5 pmul v0.16b, v0.16b, v10.16b pmul v1.16b, v1.16b, v10.16b shl v10.16b, v4.16b, #3 shl v11.16b, v5.16b, #3 eor v0.16b, v0.16b, v10.16b eor v1.16b, v1.16b, v11.16b eor v10.16b, v8.16b, v0.16b eor v11.16b, v9.16b, v1.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v8.16b, v12.16b, v0.16b eor v9.16b, v13.16b, v1.16b eor v12.16b, v12.16b, v10.16b eor v13.16b, v13.16b, v11.16b eor v10.16b, v10.16b, v4.16b eor v11.16b, v11.16b, v5.16b shl v4.4s, v10.4s, #8 shl v5.4s, v11.4s, #8 rev32 v8.8h, v8.8h rev32 v9.8h, v9.8h sri v4.4s, v10.4s, #24 sri v5.4s, v11.4s, #24 eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b shl v10.4s, v0.4s, #24 shl v11.4s, v1.4s, #24 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b sri v10.4s, v0.4s, #8 sri v11.4s, v1.4s, #8 eor v4.16b, v4.16b, v10.16b eor v5.16b, v5.16b, v11.16b # XOR in Key Schedule ld1 {v0.2d}, [x8], #16 eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b # Round Done movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v4.16b, v13.16b eor v11.16b, v5.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b orr v0.16b, v0.16b, v10.16b orr v1.16b, v1.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b ld1 {v4.16b}, [x6] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b # XOR in Key Schedule ld1 {v4.2d}, [x8], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b # Round Done rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b st1 {v0.16b, v1.16b}, [x1], #32 sub x2, x2, #32 cmp x2, #0 beq L_AES_ECB_decrypt_NEON_data_done L_AES_ECB_decrypt_NEON_start_1: movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 movi v15.16b, #27 ld1 {v3.2d}, [x6] mov x8, x3 ld1 {v0.16b}, [x0], #16 ld1 {v4.2d}, [x8], #16 rev32 v0.16b, v0.16b # Round: 0 - XOR in key schedule eor v0.16b, v0.16b, v4.16b sub w7, w4, #2 L_AES_ECB_decrypt_NEON_loop_nr_1: eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b sshr v10.16b, v4.16b, #7 ushr v11.16b, v4.16b, #6 ushr v8.16b, v4.16b, #5 and v10.16b, v10.16b, v15.16b pmul v11.16b, v11.16b, v15.16b pmul v8.16b, v8.16b, v15.16b shl v9.16b, v4.16b, #1 eor v10.16b, v10.16b, v9.16b shl v9.16b, v4.16b, #3 eor v8.16b, v8.16b, v9.16b shl v9.16b, v4.16b, #2 eor v11.16b, v11.16b, v9.16b eor v9.16b, v10.16b, v8.16b eor v8.16b, v8.16b, v4.16b eor v10.16b, v11.16b, v8.16b eor v11.16b, v11.16b, v9.16b eor v9.16b, v9.16b, v4.16b shl v4.4s, v9.4s, #8 rev32 v10.8h, v10.8h sri v4.4s, v9.4s, #24 eor v4.16b, v4.16b, v11.16b shl v9.4s, v8.4s, #24 eor v4.16b, v4.16b, v10.16b sri v9.4s, v8.4s, #8 eor v4.16b, v4.16b, v9.16b ld1 {v0.2d}, [x8], #16 # XOR in Key Schedule eor v4.16b, v4.16b, v0.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b sshr v10.16b, v0.16b, #7 ushr v11.16b, v0.16b, #6 ushr v8.16b, v0.16b, #5 and v10.16b, v10.16b, v15.16b pmul v11.16b, v11.16b, v15.16b pmul v8.16b, v8.16b, v15.16b shl v9.16b, v0.16b, #1 eor v10.16b, v10.16b, v9.16b shl v9.16b, v0.16b, #3 eor v8.16b, v8.16b, v9.16b shl v9.16b, v0.16b, #2 eor v11.16b, v11.16b, v9.16b eor v9.16b, v10.16b, v8.16b eor v8.16b, v8.16b, v0.16b eor v10.16b, v11.16b, v8.16b eor v11.16b, v11.16b, v9.16b eor v9.16b, v9.16b, v0.16b shl v0.4s, v9.4s, #8 rev32 v10.8h, v10.8h sri v0.4s, v9.4s, #24 eor v0.16b, v0.16b, v11.16b shl v9.4s, v8.4s, #24 eor v0.16b, v0.16b, v10.16b sri v9.4s, v8.4s, #8 eor v0.16b, v0.16b, v9.16b ld1 {v4.2d}, [x8], #16 # XOR in Key Schedule eor v0.16b, v0.16b, v4.16b subs w7, w7, #2 bne L_AES_ECB_decrypt_NEON_loop_nr_1 eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b sshr v10.16b, v4.16b, #7 ushr v11.16b, v4.16b, #6 ushr v8.16b, v4.16b, #5 and v10.16b, v10.16b, v15.16b pmul v11.16b, v11.16b, v15.16b pmul v8.16b, v8.16b, v15.16b shl v9.16b, v4.16b, #1 eor v10.16b, v10.16b, v9.16b shl v9.16b, v4.16b, #3 eor v8.16b, v8.16b, v9.16b shl v9.16b, v4.16b, #2 eor v11.16b, v11.16b, v9.16b eor v9.16b, v10.16b, v8.16b eor v8.16b, v8.16b, v4.16b eor v10.16b, v11.16b, v8.16b eor v11.16b, v11.16b, v9.16b eor v9.16b, v9.16b, v4.16b shl v4.4s, v9.4s, #8 rev32 v10.8h, v10.8h sri v4.4s, v9.4s, #24 eor v4.16b, v4.16b, v11.16b shl v9.4s, v8.4s, #24 eor v4.16b, v4.16b, v10.16b sri v9.4s, v8.4s, #8 eor v4.16b, v4.16b, v9.16b ld1 {v0.2d}, [x8], #16 # XOR in Key Schedule eor v4.16b, v4.16b, v0.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b ld1 {v4.2d}, [x8], #16 # XOR in Key Schedule eor v0.16b, v0.16b, v4.16b rev32 v0.16b, v0.16b st1 {v0.16b}, [x1], #16 L_AES_ECB_decrypt_NEON_data_done: ldp d8, d9, [x29, #16] ldp d10, d11, [x29, #32] ldp d12, d13, [x29, #48] ldp d14, d15, [x29, #64] ldp x29, x30, [sp], #0x50 ret #ifndef __APPLE__ .size AES_ECB_decrypt_NEON,.-AES_ECB_decrypt_NEON #endif /* __APPLE__ */ #endif /* WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER || defined(HAVE_AES_ECB) */ #ifdef HAVE_AES_CBC #ifndef __APPLE__ .text .globl AES_CBC_decrypt_NEON .type AES_CBC_decrypt_NEON,@function .align 2 AES_CBC_decrypt_NEON: #else .section __TEXT,__text .globl _AES_CBC_decrypt_NEON .p2align 2 _AES_CBC_decrypt_NEON: #endif /* __APPLE__ */ stp x29, x30, [sp, #-160]! add x29, sp, #0 stp d8, d9, [x29, #96] stp d10, d11, [x29, #112] stp d12, d13, [x29, #128] stp d14, d15, [x29, #144] #ifndef __APPLE__ adrp x6, L_AES_ARM64_NEON_td add x6, x6, :lo12:L_AES_ARM64_NEON_td #else adrp x6, L_AES_ARM64_NEON_td@PAGE add x6, x6, L_AES_ARM64_NEON_td@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x7, L_AES_ARM64_NEON_shift_rows_invshuffle add x7, x7, :lo12:L_AES_ARM64_NEON_shift_rows_invshuffle #else adrp x7, L_AES_ARM64_NEON_shift_rows_invshuffle@PAGE add x7, x7, L_AES_ARM64_NEON_shift_rows_invshuffle@PAGEOFF #endif /* __APPLE__ */ ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x6], #0x40 ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x6], #0x40 ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x6], #0x40 ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x6] ld1 {v3.2d}, [x5] add x10, x29, #16 cmp x2, #0x40 blt L_AES_CBC_decrypt_NEON_start_2 L_AES_CBC_decrypt_NEON_loop_4: mov x9, x3 ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 st1 {v3.2d, v4.2d, v5.2d, v6.2d}, [x10] str q7, [x10, #64] ld1 {v8.2d}, [x9], #16 rev32 v4.16b, v4.16b rev32 v5.16b, v5.16b rev32 v6.16b, v6.16b rev32 v7.16b, v7.16b # Round: 0 - XOR in key schedule eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v8.16b eor v6.16b, v6.16b, v8.16b eor v7.16b, v7.16b, v8.16b sub w8, w4, #2 L_AES_CBC_decrypt_NEON_loop_nr_4: tbl v8.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v9.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v10.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b tbl v11.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v0.16b, v4.16b, v12.16b eor v1.16b, v5.16b, v12.16b eor v2.16b, v6.16b, v12.16b eor v3.16b, v7.16b, v12.16b tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b tbl v2.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v2.16b tbl v3.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v3.16b orr v8.16b, v8.16b, v0.16b orr v9.16b, v9.16b, v1.16b orr v10.16b, v10.16b, v2.16b orr v11.16b, v11.16b, v3.16b eor v0.16b, v4.16b, v13.16b eor v1.16b, v5.16b, v13.16b eor v2.16b, v6.16b, v13.16b eor v3.16b, v7.16b, v13.16b tbl v0.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v0.16b tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b orr v8.16b, v8.16b, v0.16b orr v9.16b, v9.16b, v1.16b orr v10.16b, v10.16b, v2.16b orr v11.16b, v11.16b, v3.16b eor v0.16b, v4.16b, v14.16b eor v1.16b, v5.16b, v14.16b eor v2.16b, v6.16b, v14.16b eor v3.16b, v7.16b, v14.16b tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b tbl v3.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v3.16b orr v8.16b, v8.16b, v0.16b orr v9.16b, v9.16b, v1.16b orr v10.16b, v10.16b, v2.16b orr v11.16b, v11.16b, v3.16b ld1 {v4.16b}, [x7] tbl v8.16b, {v8.16b}, v4.16b tbl v9.16b, {v9.16b}, v4.16b tbl v10.16b, {v10.16b}, v4.16b tbl v11.16b, {v11.16b}, v4.16b movi v28.16b, #27 sshr v0.16b, v8.16b, #7 sshr v1.16b, v9.16b, #7 sshr v2.16b, v10.16b, #7 sshr v3.16b, v11.16b, #7 shl v12.16b, v8.16b, #1 shl v13.16b, v9.16b, #1 shl v14.16b, v10.16b, #1 shl v15.16b, v11.16b, #1 and v0.16b, v0.16b, v28.16b and v1.16b, v1.16b, v28.16b and v2.16b, v2.16b, v28.16b and v3.16b, v3.16b, v28.16b eor v0.16b, v0.16b, v12.16b eor v1.16b, v1.16b, v13.16b eor v2.16b, v2.16b, v14.16b eor v3.16b, v3.16b, v15.16b ushr v12.16b, v8.16b, #6 ushr v13.16b, v9.16b, #6 ushr v14.16b, v10.16b, #6 ushr v15.16b, v11.16b, #6 shl v4.16b, v8.16b, #2 shl v5.16b, v9.16b, #2 shl v6.16b, v10.16b, #2 shl v7.16b, v11.16b, #2 pmul v12.16b, v12.16b, v28.16b pmul v13.16b, v13.16b, v28.16b pmul v14.16b, v14.16b, v28.16b pmul v15.16b, v15.16b, v28.16b eor v12.16b, v12.16b, v4.16b eor v13.16b, v13.16b, v5.16b eor v14.16b, v14.16b, v6.16b eor v15.16b, v15.16b, v7.16b ushr v4.16b, v8.16b, #5 ushr v5.16b, v9.16b, #5 ushr v6.16b, v10.16b, #5 ushr v7.16b, v11.16b, #5 pmul v4.16b, v4.16b, v28.16b pmul v5.16b, v5.16b, v28.16b pmul v6.16b, v6.16b, v28.16b pmul v7.16b, v7.16b, v28.16b shl v28.16b, v8.16b, #3 shl v29.16b, v9.16b, #3 shl v30.16b, v10.16b, #3 shl v31.16b, v11.16b, #3 eor v4.16b, v4.16b, v28.16b eor v5.16b, v5.16b, v29.16b eor v6.16b, v6.16b, v30.16b eor v7.16b, v7.16b, v31.16b eor v28.16b, v0.16b, v4.16b eor v29.16b, v1.16b, v5.16b eor v30.16b, v2.16b, v6.16b eor v31.16b, v3.16b, v7.16b eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b eor v0.16b, v12.16b, v4.16b eor v1.16b, v13.16b, v5.16b eor v2.16b, v14.16b, v6.16b eor v3.16b, v15.16b, v7.16b eor v12.16b, v12.16b, v28.16b eor v13.16b, v13.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v15.16b, v15.16b, v31.16b eor v28.16b, v28.16b, v8.16b eor v29.16b, v29.16b, v9.16b eor v30.16b, v30.16b, v10.16b eor v31.16b, v31.16b, v11.16b shl v8.4s, v28.4s, #8 shl v9.4s, v29.4s, #8 shl v10.4s, v30.4s, #8 shl v11.4s, v31.4s, #8 rev32 v0.8h, v0.8h rev32 v1.8h, v1.8h rev32 v2.8h, v2.8h rev32 v3.8h, v3.8h sri v8.4s, v28.4s, #24 sri v9.4s, v29.4s, #24 sri v10.4s, v30.4s, #24 sri v11.4s, v31.4s, #24 eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b shl v28.4s, v4.4s, #24 shl v29.4s, v5.4s, #24 shl v30.4s, v6.4s, #24 shl v31.4s, v7.4s, #24 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b sri v28.4s, v4.4s, #8 sri v29.4s, v5.4s, #8 sri v30.4s, v6.4s, #8 sri v31.4s, v7.4s, #8 eor v8.16b, v8.16b, v28.16b eor v9.16b, v9.16b, v29.16b eor v10.16b, v10.16b, v30.16b eor v11.16b, v11.16b, v31.16b ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x6] # XOR in Key Schedule ld1 {v4.2d}, [x9], #16 eor v8.16b, v8.16b, v4.16b eor v9.16b, v9.16b, v4.16b eor v10.16b, v10.16b, v4.16b eor v11.16b, v11.16b, v4.16b # Round Done tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v8.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v9.16b tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v10.16b tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v11.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v0.16b, v8.16b, v12.16b eor v1.16b, v9.16b, v12.16b eor v2.16b, v10.16b, v12.16b eor v3.16b, v11.16b, v12.16b tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b tbl v2.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v2.16b tbl v3.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v3.16b orr v4.16b, v4.16b, v0.16b orr v5.16b, v5.16b, v1.16b orr v6.16b, v6.16b, v2.16b orr v7.16b, v7.16b, v3.16b eor v0.16b, v8.16b, v13.16b eor v1.16b, v9.16b, v13.16b eor v2.16b, v10.16b, v13.16b eor v3.16b, v11.16b, v13.16b tbl v0.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v0.16b tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b orr v4.16b, v4.16b, v0.16b orr v5.16b, v5.16b, v1.16b orr v6.16b, v6.16b, v2.16b orr v7.16b, v7.16b, v3.16b eor v0.16b, v8.16b, v14.16b eor v1.16b, v9.16b, v14.16b eor v2.16b, v10.16b, v14.16b eor v3.16b, v11.16b, v14.16b tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b tbl v3.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v3.16b orr v4.16b, v4.16b, v0.16b orr v5.16b, v5.16b, v1.16b orr v6.16b, v6.16b, v2.16b orr v7.16b, v7.16b, v3.16b ld1 {v8.16b}, [x7] tbl v4.16b, {v4.16b}, v8.16b tbl v5.16b, {v5.16b}, v8.16b tbl v6.16b, {v6.16b}, v8.16b tbl v7.16b, {v7.16b}, v8.16b movi v28.16b, #27 sshr v0.16b, v4.16b, #7 sshr v1.16b, v5.16b, #7 sshr v2.16b, v6.16b, #7 sshr v3.16b, v7.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 shl v14.16b, v6.16b, #1 shl v15.16b, v7.16b, #1 and v0.16b, v0.16b, v28.16b and v1.16b, v1.16b, v28.16b and v2.16b, v2.16b, v28.16b and v3.16b, v3.16b, v28.16b eor v0.16b, v0.16b, v12.16b eor v1.16b, v1.16b, v13.16b eor v2.16b, v2.16b, v14.16b eor v3.16b, v3.16b, v15.16b ushr v12.16b, v4.16b, #6 ushr v13.16b, v5.16b, #6 ushr v14.16b, v6.16b, #6 ushr v15.16b, v7.16b, #6 shl v8.16b, v4.16b, #2 shl v9.16b, v5.16b, #2 shl v10.16b, v6.16b, #2 shl v11.16b, v7.16b, #2 pmul v12.16b, v12.16b, v28.16b pmul v13.16b, v13.16b, v28.16b pmul v14.16b, v14.16b, v28.16b pmul v15.16b, v15.16b, v28.16b eor v12.16b, v12.16b, v8.16b eor v13.16b, v13.16b, v9.16b eor v14.16b, v14.16b, v10.16b eor v15.16b, v15.16b, v11.16b ushr v8.16b, v4.16b, #5 ushr v9.16b, v5.16b, #5 ushr v10.16b, v6.16b, #5 ushr v11.16b, v7.16b, #5 pmul v8.16b, v8.16b, v28.16b pmul v9.16b, v9.16b, v28.16b pmul v10.16b, v10.16b, v28.16b pmul v11.16b, v11.16b, v28.16b shl v28.16b, v4.16b, #3 shl v29.16b, v5.16b, #3 shl v30.16b, v6.16b, #3 shl v31.16b, v7.16b, #3 eor v8.16b, v8.16b, v28.16b eor v9.16b, v9.16b, v29.16b eor v10.16b, v10.16b, v30.16b eor v11.16b, v11.16b, v31.16b eor v28.16b, v0.16b, v8.16b eor v29.16b, v1.16b, v9.16b eor v30.16b, v2.16b, v10.16b eor v31.16b, v3.16b, v11.16b eor v8.16b, v8.16b, v4.16b eor v9.16b, v9.16b, v5.16b eor v10.16b, v10.16b, v6.16b eor v11.16b, v11.16b, v7.16b eor v0.16b, v12.16b, v8.16b eor v1.16b, v13.16b, v9.16b eor v2.16b, v14.16b, v10.16b eor v3.16b, v15.16b, v11.16b eor v12.16b, v12.16b, v28.16b eor v13.16b, v13.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v15.16b, v15.16b, v31.16b eor v28.16b, v28.16b, v4.16b eor v29.16b, v29.16b, v5.16b eor v30.16b, v30.16b, v6.16b eor v31.16b, v31.16b, v7.16b shl v4.4s, v28.4s, #8 shl v5.4s, v29.4s, #8 shl v6.4s, v30.4s, #8 shl v7.4s, v31.4s, #8 rev32 v0.8h, v0.8h rev32 v1.8h, v1.8h rev32 v2.8h, v2.8h rev32 v3.8h, v3.8h sri v4.4s, v28.4s, #24 sri v5.4s, v29.4s, #24 sri v6.4s, v30.4s, #24 sri v7.4s, v31.4s, #24 eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b eor v6.16b, v6.16b, v14.16b eor v7.16b, v7.16b, v15.16b shl v28.4s, v8.4s, #24 shl v29.4s, v9.4s, #24 shl v30.4s, v10.4s, #24 shl v31.4s, v11.4s, #24 eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b eor v6.16b, v6.16b, v2.16b eor v7.16b, v7.16b, v3.16b sri v28.4s, v8.4s, #8 sri v29.4s, v9.4s, #8 sri v30.4s, v10.4s, #8 sri v31.4s, v11.4s, #8 eor v4.16b, v4.16b, v28.16b eor v5.16b, v5.16b, v29.16b eor v6.16b, v6.16b, v30.16b eor v7.16b, v7.16b, v31.16b ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x6] # XOR in Key Schedule ld1 {v8.2d}, [x9], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v8.16b eor v6.16b, v6.16b, v8.16b eor v7.16b, v7.16b, v8.16b # Round Done subs w8, w8, #2 bne L_AES_CBC_decrypt_NEON_loop_nr_4 tbl v8.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v9.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v10.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b tbl v11.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v0.16b, v4.16b, v12.16b eor v1.16b, v5.16b, v12.16b eor v2.16b, v6.16b, v12.16b eor v3.16b, v7.16b, v12.16b tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b tbl v2.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v2.16b tbl v3.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v3.16b orr v8.16b, v8.16b, v0.16b orr v9.16b, v9.16b, v1.16b orr v10.16b, v10.16b, v2.16b orr v11.16b, v11.16b, v3.16b eor v0.16b, v4.16b, v13.16b eor v1.16b, v5.16b, v13.16b eor v2.16b, v6.16b, v13.16b eor v3.16b, v7.16b, v13.16b tbl v0.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v0.16b tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b orr v8.16b, v8.16b, v0.16b orr v9.16b, v9.16b, v1.16b orr v10.16b, v10.16b, v2.16b orr v11.16b, v11.16b, v3.16b eor v0.16b, v4.16b, v14.16b eor v1.16b, v5.16b, v14.16b eor v2.16b, v6.16b, v14.16b eor v3.16b, v7.16b, v14.16b tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b tbl v3.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v3.16b orr v8.16b, v8.16b, v0.16b orr v9.16b, v9.16b, v1.16b orr v10.16b, v10.16b, v2.16b orr v11.16b, v11.16b, v3.16b ld1 {v4.16b}, [x7] tbl v8.16b, {v8.16b}, v4.16b tbl v9.16b, {v9.16b}, v4.16b tbl v10.16b, {v10.16b}, v4.16b tbl v11.16b, {v11.16b}, v4.16b movi v28.16b, #27 sshr v0.16b, v8.16b, #7 sshr v1.16b, v9.16b, #7 sshr v2.16b, v10.16b, #7 sshr v3.16b, v11.16b, #7 shl v12.16b, v8.16b, #1 shl v13.16b, v9.16b, #1 shl v14.16b, v10.16b, #1 shl v15.16b, v11.16b, #1 and v0.16b, v0.16b, v28.16b and v1.16b, v1.16b, v28.16b and v2.16b, v2.16b, v28.16b and v3.16b, v3.16b, v28.16b eor v0.16b, v0.16b, v12.16b eor v1.16b, v1.16b, v13.16b eor v2.16b, v2.16b, v14.16b eor v3.16b, v3.16b, v15.16b ushr v12.16b, v8.16b, #6 ushr v13.16b, v9.16b, #6 ushr v14.16b, v10.16b, #6 ushr v15.16b, v11.16b, #6 shl v4.16b, v8.16b, #2 shl v5.16b, v9.16b, #2 shl v6.16b, v10.16b, #2 shl v7.16b, v11.16b, #2 pmul v12.16b, v12.16b, v28.16b pmul v13.16b, v13.16b, v28.16b pmul v14.16b, v14.16b, v28.16b pmul v15.16b, v15.16b, v28.16b eor v12.16b, v12.16b, v4.16b eor v13.16b, v13.16b, v5.16b eor v14.16b, v14.16b, v6.16b eor v15.16b, v15.16b, v7.16b ushr v4.16b, v8.16b, #5 ushr v5.16b, v9.16b, #5 ushr v6.16b, v10.16b, #5 ushr v7.16b, v11.16b, #5 pmul v4.16b, v4.16b, v28.16b pmul v5.16b, v5.16b, v28.16b pmul v6.16b, v6.16b, v28.16b pmul v7.16b, v7.16b, v28.16b shl v28.16b, v8.16b, #3 shl v29.16b, v9.16b, #3 shl v30.16b, v10.16b, #3 shl v31.16b, v11.16b, #3 eor v4.16b, v4.16b, v28.16b eor v5.16b, v5.16b, v29.16b eor v6.16b, v6.16b, v30.16b eor v7.16b, v7.16b, v31.16b eor v28.16b, v0.16b, v4.16b eor v29.16b, v1.16b, v5.16b eor v30.16b, v2.16b, v6.16b eor v31.16b, v3.16b, v7.16b eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b eor v0.16b, v12.16b, v4.16b eor v1.16b, v13.16b, v5.16b eor v2.16b, v14.16b, v6.16b eor v3.16b, v15.16b, v7.16b eor v12.16b, v12.16b, v28.16b eor v13.16b, v13.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v15.16b, v15.16b, v31.16b eor v28.16b, v28.16b, v8.16b eor v29.16b, v29.16b, v9.16b eor v30.16b, v30.16b, v10.16b eor v31.16b, v31.16b, v11.16b shl v8.4s, v28.4s, #8 shl v9.4s, v29.4s, #8 shl v10.4s, v30.4s, #8 shl v11.4s, v31.4s, #8 rev32 v0.8h, v0.8h rev32 v1.8h, v1.8h rev32 v2.8h, v2.8h rev32 v3.8h, v3.8h sri v8.4s, v28.4s, #24 sri v9.4s, v29.4s, #24 sri v10.4s, v30.4s, #24 sri v11.4s, v31.4s, #24 eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b shl v28.4s, v4.4s, #24 shl v29.4s, v5.4s, #24 shl v30.4s, v6.4s, #24 shl v31.4s, v7.4s, #24 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v2.16b eor v11.16b, v11.16b, v3.16b sri v28.4s, v4.4s, #8 sri v29.4s, v5.4s, #8 sri v30.4s, v6.4s, #8 sri v31.4s, v7.4s, #8 eor v8.16b, v8.16b, v28.16b eor v9.16b, v9.16b, v29.16b eor v10.16b, v10.16b, v30.16b eor v11.16b, v11.16b, v31.16b ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x6] # XOR in Key Schedule ld1 {v4.2d}, [x9], #16 eor v8.16b, v8.16b, v4.16b eor v9.16b, v9.16b, v4.16b eor v10.16b, v10.16b, v4.16b eor v11.16b, v11.16b, v4.16b # Round Done tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v8.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v9.16b tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v10.16b tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v11.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v0.16b, v8.16b, v12.16b eor v1.16b, v9.16b, v12.16b eor v2.16b, v10.16b, v12.16b eor v3.16b, v11.16b, v12.16b tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b tbl v2.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v2.16b tbl v3.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v3.16b orr v4.16b, v4.16b, v0.16b orr v5.16b, v5.16b, v1.16b orr v6.16b, v6.16b, v2.16b orr v7.16b, v7.16b, v3.16b eor v0.16b, v8.16b, v13.16b eor v1.16b, v9.16b, v13.16b eor v2.16b, v10.16b, v13.16b eor v3.16b, v11.16b, v13.16b tbl v0.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v0.16b tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b orr v4.16b, v4.16b, v0.16b orr v5.16b, v5.16b, v1.16b orr v6.16b, v6.16b, v2.16b orr v7.16b, v7.16b, v3.16b eor v0.16b, v8.16b, v14.16b eor v1.16b, v9.16b, v14.16b eor v2.16b, v10.16b, v14.16b eor v3.16b, v11.16b, v14.16b tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b tbl v3.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v3.16b orr v4.16b, v4.16b, v0.16b orr v5.16b, v5.16b, v1.16b orr v6.16b, v6.16b, v2.16b orr v7.16b, v7.16b, v3.16b ld1 {v8.16b}, [x7] tbl v4.16b, {v4.16b}, v8.16b tbl v5.16b, {v5.16b}, v8.16b tbl v6.16b, {v6.16b}, v8.16b tbl v7.16b, {v7.16b}, v8.16b # XOR in Key Schedule ld1 {v8.2d}, [x9], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v8.16b eor v6.16b, v6.16b, v8.16b eor v7.16b, v7.16b, v8.16b # Round Done rev32 v4.16b, v4.16b rev32 v5.16b, v5.16b rev32 v6.16b, v6.16b rev32 v7.16b, v7.16b ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x10] ldr q3, [x10, #64] eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 sub x2, x2, #0x40 cmp x2, #0x40 bge L_AES_CBC_decrypt_NEON_loop_4 L_AES_CBC_decrypt_NEON_start_2: cmp x2, #16 beq L_AES_CBC_decrypt_NEON_start_1 blt L_AES_CBC_decrypt_NEON_data_done L_AES_CBC_decrypt_NEON_loop_2: mov x9, x3 ld1 {v4.16b, v5.16b}, [x0], #32 st1 {v3.2d, v4.2d, v5.2d}, [x10] ld1 {v8.2d}, [x9], #16 rev32 v4.16b, v4.16b rev32 v5.16b, v5.16b # Round: 0 - XOR in key schedule eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v8.16b sub w8, w4, #2 L_AES_CBC_decrypt_NEON_loop_nr_2: movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v0.16b, v4.16b, v12.16b eor v1.16b, v5.16b, v12.16b tbl v8.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v9.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b eor v2.16b, v4.16b, v13.16b eor v3.16b, v5.16b, v13.16b tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b orr v8.16b, v8.16b, v0.16b orr v9.16b, v9.16b, v1.16b eor v0.16b, v4.16b, v14.16b eor v1.16b, v5.16b, v14.16b orr v8.16b, v8.16b, v2.16b orr v9.16b, v9.16b, v3.16b tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b orr v8.16b, v8.16b, v0.16b orr v9.16b, v9.16b, v1.16b ld1 {v4.16b}, [x7] tbl v8.16b, {v8.16b}, v4.16b tbl v9.16b, {v9.16b}, v4.16b movi v2.16b, #27 sshr v0.16b, v8.16b, #7 sshr v1.16b, v9.16b, #7 shl v12.16b, v8.16b, #1 shl v13.16b, v9.16b, #1 and v0.16b, v0.16b, v2.16b and v1.16b, v1.16b, v2.16b eor v0.16b, v0.16b, v12.16b eor v1.16b, v1.16b, v13.16b ushr v12.16b, v8.16b, #6 ushr v13.16b, v9.16b, #6 shl v4.16b, v8.16b, #2 shl v5.16b, v9.16b, #2 pmul v12.16b, v12.16b, v2.16b pmul v13.16b, v13.16b, v2.16b eor v12.16b, v12.16b, v4.16b eor v13.16b, v13.16b, v5.16b ushr v4.16b, v8.16b, #5 ushr v5.16b, v9.16b, #5 pmul v4.16b, v4.16b, v2.16b pmul v5.16b, v5.16b, v2.16b shl v2.16b, v8.16b, #3 shl v3.16b, v9.16b, #3 eor v4.16b, v4.16b, v2.16b eor v5.16b, v5.16b, v3.16b eor v2.16b, v0.16b, v4.16b eor v3.16b, v1.16b, v5.16b eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v0.16b, v12.16b, v4.16b eor v1.16b, v13.16b, v5.16b eor v12.16b, v12.16b, v2.16b eor v13.16b, v13.16b, v3.16b eor v2.16b, v2.16b, v8.16b eor v3.16b, v3.16b, v9.16b shl v8.4s, v2.4s, #8 shl v9.4s, v3.4s, #8 rev32 v0.8h, v0.8h rev32 v1.8h, v1.8h sri v8.4s, v2.4s, #24 sri v9.4s, v3.4s, #24 eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b shl v2.4s, v4.4s, #24 shl v3.4s, v5.4s, #24 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b sri v2.4s, v4.4s, #8 sri v3.4s, v5.4s, #8 eor v8.16b, v8.16b, v2.16b eor v9.16b, v9.16b, v3.16b # XOR in Key Schedule ld1 {v4.2d}, [x9], #16 eor v8.16b, v8.16b, v4.16b eor v9.16b, v9.16b, v4.16b # Round Done movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v0.16b, v8.16b, v12.16b eor v1.16b, v9.16b, v12.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v8.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v9.16b tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b eor v2.16b, v8.16b, v13.16b eor v3.16b, v9.16b, v13.16b tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b orr v4.16b, v4.16b, v0.16b orr v5.16b, v5.16b, v1.16b eor v0.16b, v8.16b, v14.16b eor v1.16b, v9.16b, v14.16b orr v4.16b, v4.16b, v2.16b orr v5.16b, v5.16b, v3.16b tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b orr v4.16b, v4.16b, v0.16b orr v5.16b, v5.16b, v1.16b ld1 {v8.16b}, [x7] tbl v4.16b, {v4.16b}, v8.16b tbl v5.16b, {v5.16b}, v8.16b movi v2.16b, #27 sshr v0.16b, v4.16b, #7 sshr v1.16b, v5.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 and v0.16b, v0.16b, v2.16b and v1.16b, v1.16b, v2.16b eor v0.16b, v0.16b, v12.16b eor v1.16b, v1.16b, v13.16b ushr v12.16b, v4.16b, #6 ushr v13.16b, v5.16b, #6 shl v8.16b, v4.16b, #2 shl v9.16b, v5.16b, #2 pmul v12.16b, v12.16b, v2.16b pmul v13.16b, v13.16b, v2.16b eor v12.16b, v12.16b, v8.16b eor v13.16b, v13.16b, v9.16b ushr v8.16b, v4.16b, #5 ushr v9.16b, v5.16b, #5 pmul v8.16b, v8.16b, v2.16b pmul v9.16b, v9.16b, v2.16b shl v2.16b, v4.16b, #3 shl v3.16b, v5.16b, #3 eor v8.16b, v8.16b, v2.16b eor v9.16b, v9.16b, v3.16b eor v2.16b, v0.16b, v8.16b eor v3.16b, v1.16b, v9.16b eor v8.16b, v8.16b, v4.16b eor v9.16b, v9.16b, v5.16b eor v0.16b, v12.16b, v8.16b eor v1.16b, v13.16b, v9.16b eor v12.16b, v12.16b, v2.16b eor v13.16b, v13.16b, v3.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v5.16b shl v4.4s, v2.4s, #8 shl v5.4s, v3.4s, #8 rev32 v0.8h, v0.8h rev32 v1.8h, v1.8h sri v4.4s, v2.4s, #24 sri v5.4s, v3.4s, #24 eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b shl v2.4s, v8.4s, #24 shl v3.4s, v9.4s, #24 eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b sri v2.4s, v8.4s, #8 sri v3.4s, v9.4s, #8 eor v4.16b, v4.16b, v2.16b eor v5.16b, v5.16b, v3.16b # XOR in Key Schedule ld1 {v8.2d}, [x9], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v8.16b # Round Done subs w8, w8, #2 bne L_AES_CBC_decrypt_NEON_loop_nr_2 movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v0.16b, v4.16b, v12.16b eor v1.16b, v5.16b, v12.16b tbl v8.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v9.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b eor v2.16b, v4.16b, v13.16b eor v3.16b, v5.16b, v13.16b tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b orr v8.16b, v8.16b, v0.16b orr v9.16b, v9.16b, v1.16b eor v0.16b, v4.16b, v14.16b eor v1.16b, v5.16b, v14.16b orr v8.16b, v8.16b, v2.16b orr v9.16b, v9.16b, v3.16b tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b orr v8.16b, v8.16b, v0.16b orr v9.16b, v9.16b, v1.16b ld1 {v4.16b}, [x7] tbl v8.16b, {v8.16b}, v4.16b tbl v9.16b, {v9.16b}, v4.16b movi v2.16b, #27 sshr v0.16b, v8.16b, #7 sshr v1.16b, v9.16b, #7 shl v12.16b, v8.16b, #1 shl v13.16b, v9.16b, #1 and v0.16b, v0.16b, v2.16b and v1.16b, v1.16b, v2.16b eor v0.16b, v0.16b, v12.16b eor v1.16b, v1.16b, v13.16b ushr v12.16b, v8.16b, #6 ushr v13.16b, v9.16b, #6 shl v4.16b, v8.16b, #2 shl v5.16b, v9.16b, #2 pmul v12.16b, v12.16b, v2.16b pmul v13.16b, v13.16b, v2.16b eor v12.16b, v12.16b, v4.16b eor v13.16b, v13.16b, v5.16b ushr v4.16b, v8.16b, #5 ushr v5.16b, v9.16b, #5 pmul v4.16b, v4.16b, v2.16b pmul v5.16b, v5.16b, v2.16b shl v2.16b, v8.16b, #3 shl v3.16b, v9.16b, #3 eor v4.16b, v4.16b, v2.16b eor v5.16b, v5.16b, v3.16b eor v2.16b, v0.16b, v4.16b eor v3.16b, v1.16b, v5.16b eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v0.16b, v12.16b, v4.16b eor v1.16b, v13.16b, v5.16b eor v12.16b, v12.16b, v2.16b eor v13.16b, v13.16b, v3.16b eor v2.16b, v2.16b, v8.16b eor v3.16b, v3.16b, v9.16b shl v8.4s, v2.4s, #8 shl v9.4s, v3.4s, #8 rev32 v0.8h, v0.8h rev32 v1.8h, v1.8h sri v8.4s, v2.4s, #24 sri v9.4s, v3.4s, #24 eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b shl v2.4s, v4.4s, #24 shl v3.4s, v5.4s, #24 eor v8.16b, v8.16b, v0.16b eor v9.16b, v9.16b, v1.16b sri v2.4s, v4.4s, #8 sri v3.4s, v5.4s, #8 eor v8.16b, v8.16b, v2.16b eor v9.16b, v9.16b, v3.16b # XOR in Key Schedule ld1 {v4.2d}, [x9], #16 eor v8.16b, v8.16b, v4.16b eor v9.16b, v9.16b, v4.16b # Round Done movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v0.16b, v8.16b, v12.16b eor v1.16b, v9.16b, v12.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v8.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v9.16b tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b eor v2.16b, v8.16b, v13.16b eor v3.16b, v9.16b, v13.16b tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b orr v4.16b, v4.16b, v0.16b orr v5.16b, v5.16b, v1.16b eor v0.16b, v8.16b, v14.16b eor v1.16b, v9.16b, v14.16b orr v4.16b, v4.16b, v2.16b orr v5.16b, v5.16b, v3.16b tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b orr v4.16b, v4.16b, v0.16b orr v5.16b, v5.16b, v1.16b ld1 {v8.16b}, [x7] tbl v4.16b, {v4.16b}, v8.16b tbl v5.16b, {v5.16b}, v8.16b # XOR in Key Schedule ld1 {v8.2d}, [x9], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v8.16b # Round Done rev32 v4.16b, v4.16b rev32 v5.16b, v5.16b ld1 {v1.16b, v2.16b, v3.16b}, [x10] eor v4.16b, v4.16b, v1.16b eor v5.16b, v5.16b, v2.16b st1 {v4.16b, v5.16b}, [x1], #32 sub x2, x2, #32 cmp x2, #32 bge L_AES_CBC_decrypt_NEON_loop_2 cmp x2, #0 beq L_AES_CBC_decrypt_NEON_data_done L_AES_CBC_decrypt_NEON_start_1: movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 movi v15.16b, #27 ld1 {v7.2d}, [x7] mov x9, x3 ld1 {v4.16b}, [x0], #16 mov v10.16b, v3.16b mov v11.16b, v4.16b ld1 {v8.16b}, [x9], #16 rev32 v4.16b, v4.16b # Round: 0 - XOR in key schedule eor v4.16b, v4.16b, v8.16b sub w8, w4, #2 L_AES_CBC_decrypt_NEON_loop_nr_1: eor v0.16b, v4.16b, v12.16b eor v1.16b, v4.16b, v13.16b eor v2.16b, v4.16b, v14.16b tbl v8.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b orr v8.16b, v8.16b, v0.16b orr v1.16b, v1.16b, v2.16b orr v8.16b, v8.16b, v1.16b tbl v8.16b, {v8.16b}, v7.16b sshr v2.16b, v8.16b, #7 ushr v3.16b, v8.16b, #6 ushr v0.16b, v8.16b, #5 and v2.16b, v2.16b, v15.16b pmul v3.16b, v3.16b, v15.16b pmul v0.16b, v0.16b, v15.16b shl v1.16b, v8.16b, #1 eor v2.16b, v2.16b, v1.16b shl v1.16b, v8.16b, #3 eor v0.16b, v0.16b, v1.16b shl v1.16b, v8.16b, #2 eor v3.16b, v3.16b, v1.16b eor v1.16b, v2.16b, v0.16b eor v0.16b, v0.16b, v8.16b eor v2.16b, v3.16b, v0.16b eor v3.16b, v3.16b, v1.16b eor v1.16b, v1.16b, v8.16b shl v8.4s, v1.4s, #8 rev32 v2.8h, v2.8h sri v8.4s, v1.4s, #24 eor v8.16b, v8.16b, v3.16b shl v1.4s, v0.4s, #24 eor v8.16b, v8.16b, v2.16b sri v1.4s, v0.4s, #8 eor v8.16b, v8.16b, v1.16b ld1 {v4.2d}, [x9], #16 # XOR in Key Schedule eor v8.16b, v8.16b, v4.16b eor v0.16b, v8.16b, v12.16b eor v1.16b, v8.16b, v13.16b eor v2.16b, v8.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v8.16b tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b orr v4.16b, v4.16b, v0.16b orr v1.16b, v1.16b, v2.16b orr v4.16b, v4.16b, v1.16b tbl v4.16b, {v4.16b}, v7.16b sshr v2.16b, v4.16b, #7 ushr v3.16b, v4.16b, #6 ushr v0.16b, v4.16b, #5 and v2.16b, v2.16b, v15.16b pmul v3.16b, v3.16b, v15.16b pmul v0.16b, v0.16b, v15.16b shl v1.16b, v4.16b, #1 eor v2.16b, v2.16b, v1.16b shl v1.16b, v4.16b, #3 eor v0.16b, v0.16b, v1.16b shl v1.16b, v4.16b, #2 eor v3.16b, v3.16b, v1.16b eor v1.16b, v2.16b, v0.16b eor v0.16b, v0.16b, v4.16b eor v2.16b, v3.16b, v0.16b eor v3.16b, v3.16b, v1.16b eor v1.16b, v1.16b, v4.16b shl v4.4s, v1.4s, #8 rev32 v2.8h, v2.8h sri v4.4s, v1.4s, #24 eor v4.16b, v4.16b, v3.16b shl v1.4s, v0.4s, #24 eor v4.16b, v4.16b, v2.16b sri v1.4s, v0.4s, #8 eor v4.16b, v4.16b, v1.16b ld1 {v8.2d}, [x9], #16 # XOR in Key Schedule eor v4.16b, v4.16b, v8.16b subs w8, w8, #2 bne L_AES_CBC_decrypt_NEON_loop_nr_1 eor v0.16b, v4.16b, v12.16b eor v1.16b, v4.16b, v13.16b eor v2.16b, v4.16b, v14.16b tbl v8.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b orr v8.16b, v8.16b, v0.16b orr v1.16b, v1.16b, v2.16b orr v8.16b, v8.16b, v1.16b tbl v8.16b, {v8.16b}, v7.16b sshr v2.16b, v8.16b, #7 ushr v3.16b, v8.16b, #6 ushr v0.16b, v8.16b, #5 and v2.16b, v2.16b, v15.16b pmul v3.16b, v3.16b, v15.16b pmul v0.16b, v0.16b, v15.16b shl v1.16b, v8.16b, #1 eor v2.16b, v2.16b, v1.16b shl v1.16b, v8.16b, #3 eor v0.16b, v0.16b, v1.16b shl v1.16b, v8.16b, #2 eor v3.16b, v3.16b, v1.16b eor v1.16b, v2.16b, v0.16b eor v0.16b, v0.16b, v8.16b eor v2.16b, v3.16b, v0.16b eor v3.16b, v3.16b, v1.16b eor v1.16b, v1.16b, v8.16b shl v8.4s, v1.4s, #8 rev32 v2.8h, v2.8h sri v8.4s, v1.4s, #24 eor v8.16b, v8.16b, v3.16b shl v1.4s, v0.4s, #24 eor v8.16b, v8.16b, v2.16b sri v1.4s, v0.4s, #8 eor v8.16b, v8.16b, v1.16b ld1 {v4.2d}, [x9], #16 # XOR in Key Schedule eor v8.16b, v8.16b, v4.16b eor v0.16b, v8.16b, v12.16b eor v1.16b, v8.16b, v13.16b eor v2.16b, v8.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v8.16b tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b orr v4.16b, v4.16b, v0.16b orr v1.16b, v1.16b, v2.16b orr v4.16b, v4.16b, v1.16b tbl v4.16b, {v4.16b}, v7.16b ld1 {v8.2d}, [x9], #16 # XOR in Key Schedule eor v4.16b, v4.16b, v8.16b rev32 v4.16b, v4.16b mov v3.16b, v11.16b eor v4.16b, v4.16b, v10.16b st1 {v4.16b}, [x1], #16 L_AES_CBC_decrypt_NEON_data_done: st1 {v3.2d}, [x5] ldp d8, d9, [x29, #96] ldp d10, d11, [x29, #112] ldp d12, d13, [x29, #128] ldp d14, d15, [x29, #144] ldp x29, x30, [sp], #0xa0 ret #ifndef __APPLE__ .size AES_CBC_decrypt_NEON,.-AES_CBC_decrypt_NEON #endif /* __APPLE__ */ #endif /* HAVE_AES_CBC */ #endif /* WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER || HAVE_AES_CBC * HAVE_AES_ECB */ #endif /* HAVE_AES_DECRYPT */ #ifdef HAVE_AESGCM #ifndef __APPLE__ .text .globl GCM_gmult_len_NEON .type GCM_gmult_len_NEON,@function .align 2 GCM_gmult_len_NEON: #else .section __TEXT,__text .globl _GCM_gmult_len_NEON .p2align 2 _GCM_gmult_len_NEON: #endif /* __APPLE__ */ stp x29, x30, [sp, #-80]! add x29, sp, #0 stp d8, d9, [x29, #16] stp d10, d11, [x29, #32] stp d12, d13, [x29, #48] stp d14, d15, [x29, #64] ld1 {v18.2d}, [x0] ld1 {v10.2d}, [x1] movi v19.16b, #15 eor v20.16b, v20.16b, v20.16b rbit v18.16b, v18.16b rbit v10.16b, v10.16b and v12.16b, v10.16b, v19.16b ushr v13.16b, v10.16b, #4 eor v14.16b, v12.16b, v13.16b L_GCM_gmult_len_NEON_start_block: ld1 {v0.16b}, [x2], #16 rbit v0.16b, v0.16b eor v18.16b, v18.16b, v0.16b # Mul 128x128 and v15.16b, v18.16b, v19.16b ushr v16.16b, v18.16b, #4 eor v17.16b, v15.16b, v16.16b dup v0.16b, v12.b[0] dup v2.16b, v14.b[0] dup v1.16b, v13.b[0] pmul v8.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v8.16b eor v5.16b, v5.16b, v4.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v8.16b, v8.16b, v6.16b eor v11.16b, v4.16b, v7.16b dup v0.16b, v12.b[1] dup v2.16b, v14.b[1] dup v1.16b, v13.b[1] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v6.16b, v20.16b, v3.16b, #15 ext v9.16b, v3.16b, v20.16b, #15 eor v8.16b, v8.16b, v6.16b dup v0.16b, v12.b[2] dup v2.16b, v14.b[2] dup v1.16b, v13.b[2] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v7.16b, v3.16b, v20.16b, #14 ext v6.16b, v20.16b, v3.16b, #14 eor v9.16b, v9.16b, v7.16b eor v8.16b, v8.16b, v6.16b dup v0.16b, v12.b[3] dup v2.16b, v14.b[3] dup v1.16b, v13.b[3] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v7.16b, v3.16b, v20.16b, #13 ext v6.16b, v20.16b, v3.16b, #13 eor v9.16b, v9.16b, v7.16b eor v8.16b, v8.16b, v6.16b dup v0.16b, v12.b[4] dup v2.16b, v14.b[4] dup v1.16b, v13.b[4] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v7.16b, v3.16b, v20.16b, #12 ext v6.16b, v20.16b, v3.16b, #12 eor v9.16b, v9.16b, v7.16b eor v8.16b, v8.16b, v6.16b dup v0.16b, v12.b[5] dup v2.16b, v14.b[5] dup v1.16b, v13.b[5] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v7.16b, v3.16b, v20.16b, #11 ext v6.16b, v20.16b, v3.16b, #11 eor v9.16b, v9.16b, v7.16b eor v8.16b, v8.16b, v6.16b dup v0.16b, v12.b[6] dup v2.16b, v14.b[6] dup v1.16b, v13.b[6] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v7.16b, v3.16b, v20.16b, #10 ext v6.16b, v20.16b, v3.16b, #10 eor v9.16b, v9.16b, v7.16b eor v8.16b, v8.16b, v6.16b dup v0.16b, v12.b[7] dup v2.16b, v14.b[7] dup v1.16b, v13.b[7] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v7.16b, v3.16b, v20.16b, #9 ext v6.16b, v20.16b, v3.16b, #9 eor v9.16b, v9.16b, v7.16b eor v8.16b, v8.16b, v6.16b dup v0.16b, v12.b[8] dup v2.16b, v14.b[8] dup v1.16b, v13.b[8] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v7.16b, v3.16b, v20.16b, #8 ext v6.16b, v20.16b, v3.16b, #8 eor v9.16b, v9.16b, v7.16b eor v8.16b, v8.16b, v6.16b dup v0.16b, v12.b[9] dup v2.16b, v14.b[9] dup v1.16b, v13.b[9] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v7.16b, v3.16b, v20.16b, #7 ext v6.16b, v20.16b, v3.16b, #7 eor v9.16b, v9.16b, v7.16b eor v8.16b, v8.16b, v6.16b dup v0.16b, v12.b[10] dup v2.16b, v14.b[10] dup v1.16b, v13.b[10] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v7.16b, v3.16b, v20.16b, #6 ext v6.16b, v20.16b, v3.16b, #6 eor v9.16b, v9.16b, v7.16b eor v8.16b, v8.16b, v6.16b dup v0.16b, v12.b[11] dup v2.16b, v14.b[11] dup v1.16b, v13.b[11] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v7.16b, v3.16b, v20.16b, #5 ext v6.16b, v20.16b, v3.16b, #5 eor v9.16b, v9.16b, v7.16b eor v8.16b, v8.16b, v6.16b dup v0.16b, v12.b[12] dup v2.16b, v14.b[12] dup v1.16b, v13.b[12] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v7.16b, v3.16b, v20.16b, #4 ext v6.16b, v20.16b, v3.16b, #4 eor v9.16b, v9.16b, v7.16b eor v8.16b, v8.16b, v6.16b dup v0.16b, v12.b[13] dup v2.16b, v14.b[13] dup v1.16b, v13.b[13] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v7.16b, v3.16b, v20.16b, #3 ext v6.16b, v20.16b, v3.16b, #3 eor v9.16b, v9.16b, v7.16b eor v8.16b, v8.16b, v6.16b dup v0.16b, v12.b[14] dup v2.16b, v14.b[14] dup v1.16b, v13.b[14] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v7.16b, v3.16b, v20.16b, #2 ext v6.16b, v20.16b, v3.16b, #2 eor v9.16b, v9.16b, v7.16b eor v8.16b, v8.16b, v6.16b dup v0.16b, v12.b[15] dup v2.16b, v14.b[15] dup v1.16b, v13.b[15] pmul v3.16b, v15.16b, v0.16b pmul v5.16b, v17.16b, v2.16b pmul v4.16b, v16.16b, v1.16b eor v5.16b, v5.16b, v3.16b eor v5.16b, v5.16b, v4.16b eor v3.16b, v3.16b, v11.16b shl v6.16b, v5.16b, #4 ushr v7.16b, v5.16b, #4 eor v3.16b, v3.16b, v6.16b eor v11.16b, v4.16b, v7.16b ext v7.16b, v3.16b, v20.16b, #1 ext v6.16b, v20.16b, v3.16b, #1 eor v9.16b, v9.16b, v7.16b eor v8.16b, v8.16b, v6.16b eor v9.16b, v9.16b, v11.16b # Reduce 254-bit number shl v0.16b, v9.16b, #1 shl v1.16b, v9.16b, #2 shl v2.16b, v9.16b, #7 ushr v3.16b, v9.16b, #7 ushr v4.16b, v9.16b, #6 ushr v5.16b, v9.16b, #1 eor v0.16b, v0.16b, v9.16b eor v1.16b, v1.16b, v2.16b eor v0.16b, v0.16b, v1.16b eor v8.16b, v8.16b, v0.16b ext v0.16b, v20.16b, v3.16b, #15 ext v1.16b, v20.16b, v4.16b, #15 ext v2.16b, v20.16b, v5.16b, #15 ext v4.16b, v4.16b, v20.16b, #15 ext v5.16b, v5.16b, v20.16b, #15 eor v0.16b, v0.16b, v1.16b eor v8.16b, v8.16b, v2.16b eor v8.16b, v8.16b, v0.16b eor v3.16b, v4.16b, v5.16b shl v0.2d, v3.2d, #1 shl v1.2d, v3.2d, #2 shl v2.2d, v3.2d, #7 eor v3.16b, v3.16b, v0.16b eor v1.16b, v1.16b, v2.16b eor v8.16b, v8.16b, v3.16b eor v18.16b, v8.16b, v1.16b subs x3, x3, #16 bne L_GCM_gmult_len_NEON_start_block rbit v18.16b, v18.16b st1 {v18.2d}, [x0] ldp d8, d9, [x29, #16] ldp d10, d11, [x29, #32] ldp d12, d13, [x29, #48] ldp d14, d15, [x29, #64] ldp x29, x30, [sp], #0x50 ret #ifndef __APPLE__ .size GCM_gmult_len_NEON,.-GCM_gmult_len_NEON #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_encrypt_NEON .type AES_GCM_encrypt_NEON,@function .align 2 AES_GCM_encrypt_NEON: #else .section __TEXT,__text .globl _AES_GCM_encrypt_NEON .p2align 2 _AES_GCM_encrypt_NEON: #endif /* __APPLE__ */ stp x29, x30, [sp, #-80]! add x29, sp, #0 stp d8, d9, [x29, #16] stp d10, d11, [x29, #32] stp d12, d13, [x29, #48] stp d14, d15, [x29, #64] #ifndef __APPLE__ adrp x9, L_AES_ARM64_NEON_te add x9, x9, :lo12:L_AES_ARM64_NEON_te #else adrp x9, L_AES_ARM64_NEON_te@PAGE add x9, x9, L_AES_ARM64_NEON_te@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x10, L_AES_ARM64_NEON_shift_rows_shuffle add x10, x10, :lo12:L_AES_ARM64_NEON_shift_rows_shuffle #else adrp x10, L_AES_ARM64_NEON_shift_rows_shuffle@PAGE add x10, x10, L_AES_ARM64_NEON_shift_rows_shuffle@PAGEOFF #endif /* __APPLE__ */ ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x9], #0x40 ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x9], #0x40 ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x9], #0x40 ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x9] ld1 {v2.2d}, [x5] rev32 v2.16b, v2.16b mov w6, v2.s[3] cmp x2, #0x40 blt L_AES_GCM_encrypt_NEON_start_2 mov x7, v2.d[0] mov x8, v2.d[1] L_AES_GCM_encrypt_NEON_loop_4: mov x12, x3 ld1 {v4.2d}, [x12], #16 mov v8.d[0], x7 mov v8.d[1], x8 # Round: 0 - XOR in key schedule add w6, w6, #1 mov v8.s[3], w6 eor v0.16b, v8.16b, v4.16b add w6, w6, #1 mov v8.s[3], w6 eor v1.16b, v8.16b, v4.16b add w6, w6, #1 mov v8.s[3], w6 eor v2.16b, v8.16b, v4.16b add w6, w6, #1 mov v8.s[3], w6 eor v3.16b, v8.16b, v4.16b sub w11, w4, #2 L_AES_GCM_encrypt_NEON_loop_nr_4: tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b eor v10.16b, v2.16b, v12.16b eor v11.16b, v3.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v13.16b eor v9.16b, v1.16b, v13.16b eor v10.16b, v2.16b, v13.16b eor v11.16b, v3.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b eor v10.16b, v2.16b, v14.16b eor v11.16b, v3.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b ld1 {v0.16b}, [x10] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b tbl v6.16b, {v6.16b}, v0.16b tbl v7.16b, {v7.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 sshr v10.16b, v6.16b, #7 sshr v11.16b, v7.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 shl v14.16b, v6.16b, #1 shl v15.16b, v7.16b, #1 movi v0.16b, #27 and v8.16b, v8.16b, v0.16b and v9.16b, v9.16b, v0.16b and v10.16b, v10.16b, v0.16b and v11.16b, v11.16b, v0.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b eor v2.16b, v10.16b, v6.16b eor v3.16b, v11.16b, v7.16b shl v12.4s, v0.4s, #8 shl v13.4s, v1.4s, #8 shl v14.4s, v2.4s, #8 shl v15.4s, v3.4s, #8 sri v12.4s, v0.4s, #24 sri v13.4s, v1.4s, #24 sri v14.4s, v2.4s, #24 sri v15.4s, v3.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 shl v2.4s, v6.4s, #24 shl v3.4s, v7.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 sri v2.4s, v6.4s, #8 sri v3.4s, v7.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h rev32 v6.8h, v6.8h rev32 v7.8h, v7.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b eor v6.16b, v6.16b, v2.16b eor v7.16b, v7.16b, v3.16b # XOR in Key Schedule ld1 {v0.2d}, [x12], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v6.16b, v6.16b, v0.16b eor v7.16b, v7.16b, v0.16b eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b eor v6.16b, v6.16b, v14.16b eor v7.16b, v7.16b, v15.16b # Round Done tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b eor v10.16b, v6.16b, v12.16b eor v11.16b, v7.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v13.16b eor v9.16b, v5.16b, v13.16b eor v10.16b, v6.16b, v13.16b eor v11.16b, v7.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b eor v10.16b, v6.16b, v14.16b eor v11.16b, v7.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b ld1 {v4.16b}, [x10] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b tbl v2.16b, {v2.16b}, v4.16b tbl v3.16b, {v3.16b}, v4.16b sshr v8.16b, v0.16b, #7 sshr v9.16b, v1.16b, #7 sshr v10.16b, v2.16b, #7 sshr v11.16b, v3.16b, #7 shl v12.16b, v0.16b, #1 shl v13.16b, v1.16b, #1 shl v14.16b, v2.16b, #1 shl v15.16b, v3.16b, #1 movi v4.16b, #27 and v8.16b, v8.16b, v4.16b and v9.16b, v9.16b, v4.16b and v10.16b, v10.16b, v4.16b and v11.16b, v11.16b, v4.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b eor v4.16b, v8.16b, v0.16b eor v5.16b, v9.16b, v1.16b eor v6.16b, v10.16b, v2.16b eor v7.16b, v11.16b, v3.16b shl v12.4s, v4.4s, #8 shl v13.4s, v5.4s, #8 shl v14.4s, v6.4s, #8 shl v15.4s, v7.4s, #8 sri v12.4s, v4.4s, #24 sri v13.4s, v5.4s, #24 sri v14.4s, v6.4s, #24 sri v15.4s, v7.4s, #24 shl v4.4s, v0.4s, #24 shl v5.4s, v1.4s, #24 shl v6.4s, v2.4s, #24 shl v7.4s, v3.4s, #24 sri v4.4s, v0.4s, #8 sri v5.4s, v1.4s, #8 sri v6.4s, v2.4s, #8 sri v7.4s, v3.4s, #8 rev32 v0.8h, v0.8h rev32 v1.8h, v1.8h rev32 v2.8h, v2.8h rev32 v3.8h, v3.8h eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b # XOR in Key Schedule ld1 {v4.2d}, [x12], #16 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b eor v0.16b, v0.16b, v12.16b eor v1.16b, v1.16b, v13.16b eor v2.16b, v2.16b, v14.16b eor v3.16b, v3.16b, v15.16b # Round Done subs w11, w11, #2 bne L_AES_GCM_encrypt_NEON_loop_nr_4 tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b eor v10.16b, v2.16b, v12.16b eor v11.16b, v3.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v13.16b eor v9.16b, v1.16b, v13.16b eor v10.16b, v2.16b, v13.16b eor v11.16b, v3.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b eor v10.16b, v2.16b, v14.16b eor v11.16b, v3.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b ld1 {v0.16b}, [x10] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b tbl v6.16b, {v6.16b}, v0.16b tbl v7.16b, {v7.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 sshr v10.16b, v6.16b, #7 sshr v11.16b, v7.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 shl v14.16b, v6.16b, #1 shl v15.16b, v7.16b, #1 movi v0.16b, #27 and v8.16b, v8.16b, v0.16b and v9.16b, v9.16b, v0.16b and v10.16b, v10.16b, v0.16b and v11.16b, v11.16b, v0.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b eor v2.16b, v10.16b, v6.16b eor v3.16b, v11.16b, v7.16b shl v12.4s, v0.4s, #8 shl v13.4s, v1.4s, #8 shl v14.4s, v2.4s, #8 shl v15.4s, v3.4s, #8 sri v12.4s, v0.4s, #24 sri v13.4s, v1.4s, #24 sri v14.4s, v2.4s, #24 sri v15.4s, v3.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 shl v2.4s, v6.4s, #24 shl v3.4s, v7.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 sri v2.4s, v6.4s, #8 sri v3.4s, v7.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h rev32 v6.8h, v6.8h rev32 v7.8h, v7.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b eor v6.16b, v6.16b, v2.16b eor v7.16b, v7.16b, v3.16b # XOR in Key Schedule ld1 {v0.2d}, [x12], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v6.16b, v6.16b, v0.16b eor v7.16b, v7.16b, v0.16b eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b eor v6.16b, v6.16b, v14.16b eor v7.16b, v7.16b, v15.16b # Round Done tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b eor v10.16b, v6.16b, v12.16b eor v11.16b, v7.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v13.16b eor v9.16b, v5.16b, v13.16b eor v10.16b, v6.16b, v13.16b eor v11.16b, v7.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b eor v10.16b, v6.16b, v14.16b eor v11.16b, v7.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b ld1 {v4.16b}, [x10] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b tbl v2.16b, {v2.16b}, v4.16b tbl v3.16b, {v3.16b}, v4.16b # XOR in Key Schedule ld1 {v4.2d}, [x12], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b # Round Done rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b rev32 v2.16b, v2.16b rev32 v3.16b, v3.16b ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 sub x2, x2, #0x40 cmp x2, #0x40 bge L_AES_GCM_encrypt_NEON_loop_4 mov v2.d[0], x7 mov v2.d[1], x8 mov v2.s[3], w6 L_AES_GCM_encrypt_NEON_start_2: movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 movi v15.16b, #27 cmp x2, #16 beq L_AES_GCM_encrypt_NEON_start_1 blt L_AES_GCM_encrypt_NEON_data_done L_AES_GCM_encrypt_NEON_loop_2: mov x12, x3 ld1 {v4.2d}, [x12], #16 # Round: 0 - XOR in key schedule add w6, w6, #1 mov v2.s[3], w6 eor v0.16b, v2.16b, v4.16b add w6, w6, #1 mov v2.s[3], w6 eor v1.16b, v2.16b, v4.16b sub w11, w4, #2 L_AES_GCM_encrypt_NEON_loop_nr_2: eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v0.16b, v13.16b eor v11.16b, v1.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b orr v4.16b, v4.16b, v10.16b orr v5.16b, v5.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b ld1 {v0.16b}, [x10] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 shl v10.16b, v4.16b, #1 shl v11.16b, v5.16b, #1 and v8.16b, v8.16b, v15.16b and v9.16b, v9.16b, v15.16b eor v8.16b, v8.16b, v10.16b eor v9.16b, v9.16b, v11.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b shl v10.4s, v0.4s, #8 shl v11.4s, v1.4s, #8 sri v10.4s, v0.4s, #24 sri v11.4s, v1.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b # XOR in Key Schedule ld1 {v0.2d}, [x12], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v4.16b, v4.16b, v10.16b eor v5.16b, v5.16b, v11.16b # Round Done eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v4.16b, v13.16b eor v11.16b, v5.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b orr v0.16b, v0.16b, v10.16b orr v1.16b, v1.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b ld1 {v4.16b}, [x10] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b sshr v8.16b, v0.16b, #7 sshr v9.16b, v1.16b, #7 shl v10.16b, v0.16b, #1 shl v11.16b, v1.16b, #1 and v8.16b, v8.16b, v15.16b and v9.16b, v9.16b, v15.16b eor v8.16b, v8.16b, v10.16b eor v9.16b, v9.16b, v11.16b eor v4.16b, v8.16b, v0.16b eor v5.16b, v9.16b, v1.16b shl v10.4s, v4.4s, #8 shl v11.4s, v5.4s, #8 sri v10.4s, v4.4s, #24 sri v11.4s, v5.4s, #24 shl v4.4s, v0.4s, #24 shl v5.4s, v1.4s, #24 sri v4.4s, v0.4s, #8 sri v5.4s, v1.4s, #8 rev32 v0.8h, v0.8h rev32 v1.8h, v1.8h eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b # XOR in Key Schedule ld1 {v4.2d}, [x12], #16 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v0.16b, v0.16b, v10.16b eor v1.16b, v1.16b, v11.16b # Round Done subs w11, w11, #2 bne L_AES_GCM_encrypt_NEON_loop_nr_2 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v0.16b, v13.16b eor v11.16b, v1.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b orr v4.16b, v4.16b, v10.16b orr v5.16b, v5.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b ld1 {v0.16b}, [x10] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 shl v10.16b, v4.16b, #1 shl v11.16b, v5.16b, #1 and v8.16b, v8.16b, v15.16b and v9.16b, v9.16b, v15.16b eor v8.16b, v8.16b, v10.16b eor v9.16b, v9.16b, v11.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b shl v10.4s, v0.4s, #8 shl v11.4s, v1.4s, #8 sri v10.4s, v0.4s, #24 sri v11.4s, v1.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b # XOR in Key Schedule ld1 {v0.2d}, [x12], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v4.16b, v4.16b, v10.16b eor v5.16b, v5.16b, v11.16b # Round Done eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v4.16b, v13.16b eor v11.16b, v5.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b orr v0.16b, v0.16b, v10.16b orr v1.16b, v1.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b ld1 {v4.16b}, [x10] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b # XOR in Key Schedule ld1 {v4.2d}, [x12], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b # Round Done rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b ld1 {v4.16b, v5.16b}, [x0], #32 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b st1 {v0.16b, v1.16b}, [x1], #32 sub x2, x2, #32 cmp x2, #0 beq L_AES_GCM_encrypt_NEON_data_done L_AES_GCM_encrypt_NEON_start_1: ld1 {v3.2d}, [x10] mov x12, x3 add w6, w6, #1 ld1 {v4.2d}, [x12], #16 mov v2.s[3], w6 # Round: 0 - XOR in key schedule eor v0.16b, v2.16b, v4.16b sub w11, w4, #2 L_AES_GCM_encrypt_NEON_loop_nr_1: eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b ld1 {v0.2d}, [x12], #16 sshr v10.16b, v4.16b, #7 shl v9.16b, v4.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v4.8h eor v11.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v4.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v0.16b sri v9.4s, v4.4s, #8 sri v8.4s, v11.4s, #24 eor v4.16b, v10.16b, v9.16b eor v4.16b, v4.16b, v8.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b ld1 {v4.2d}, [x12], #16 sshr v10.16b, v0.16b, #7 shl v9.16b, v0.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v0.8h eor v11.16b, v10.16b, v0.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v0.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v4.16b sri v9.4s, v0.4s, #8 sri v8.4s, v11.4s, #24 eor v0.16b, v10.16b, v9.16b eor v0.16b, v0.16b, v8.16b subs w11, w11, #2 bne L_AES_GCM_encrypt_NEON_loop_nr_1 eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b ld1 {v0.2d}, [x12], #16 sshr v10.16b, v4.16b, #7 shl v9.16b, v4.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v4.8h eor v11.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v4.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v0.16b sri v9.4s, v4.4s, #8 sri v8.4s, v11.4s, #24 eor v4.16b, v10.16b, v9.16b eor v4.16b, v4.16b, v8.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b ld1 {v4.2d}, [x12], #16 # XOR in Key Schedule eor v0.16b, v0.16b, v4.16b rev32 v0.16b, v0.16b ld1 {v4.16b}, [x0], #16 eor v0.16b, v0.16b, v4.16b st1 {v0.16b}, [x1], #16 L_AES_GCM_encrypt_NEON_data_done: rev32 v2.16b, v2.16b st1 {v2.2d}, [x5] ldp d8, d9, [x29, #16] ldp d10, d11, [x29, #32] ldp d12, d13, [x29, #48] ldp d14, d15, [x29, #64] ldp x29, x30, [sp], #0x50 ret #ifndef __APPLE__ .size AES_GCM_encrypt_NEON,.-AES_GCM_encrypt_NEON #endif /* __APPLE__ */ #endif /* HAVE_AESGCM */ #ifdef WOLFSSL_AES_XTS #ifndef __APPLE__ .text .globl AES_XTS_encrypt_NEON .type AES_XTS_encrypt_NEON,@function .align 2 AES_XTS_encrypt_NEON: #else .section __TEXT,__text .globl _AES_XTS_encrypt_NEON .p2align 2 _AES_XTS_encrypt_NEON: #endif /* __APPLE__ */ stp x29, x30, [sp, #-128]! add x29, sp, #0 stp x17, x19, [x29, #24] stp x20, x21, [x29, #40] str x22, [x29, #56] stp d8, d9, [x29, #64] stp d10, d11, [x29, #80] stp d12, d13, [x29, #96] stp d14, d15, [x29, #112] #ifndef __APPLE__ adrp x19, L_AES_ARM64_NEON_te add x19, x19, :lo12:L_AES_ARM64_NEON_te #else adrp x19, L_AES_ARM64_NEON_te@PAGE add x19, x19, L_AES_ARM64_NEON_te@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x20, L_AES_ARM64_NEON_shift_rows_shuffle add x20, x20, :lo12:L_AES_ARM64_NEON_shift_rows_shuffle #else adrp x20, L_AES_ARM64_NEON_shift_rows_shuffle@PAGE add x20, x20, L_AES_ARM64_NEON_shift_rows_shuffle@PAGEOFF #endif /* __APPLE__ */ ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x19], #0x40 ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x19], #0x40 ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x19], #0x40 ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x19] movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 movi v15.16b, #27 ld1 {v3.2d}, [x20] mov x17, #0x87 ld1 {v2.2d}, [x3] ld1 {v4.2d}, [x5] rev32 v2.16b, v2.16b add x22, x5, #16 # Round: 0 - XOR in key schedule eor v2.16b, v2.16b, v4.16b sub w21, w7, #2 L_AES_XTS_encrypt_NEON_loop_nr_tweak: eor v8.16b, v2.16b, v12.16b eor v9.16b, v2.16b, v13.16b eor v10.16b, v2.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b ld1 {v2.2d}, [x22], #16 sshr v10.16b, v4.16b, #7 shl v9.16b, v4.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v4.8h eor v11.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v4.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v2.16b sri v9.4s, v4.4s, #8 sri v8.4s, v11.4s, #24 eor v4.16b, v10.16b, v9.16b eor v4.16b, v4.16b, v8.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v2.16b, v2.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v2.16b, v2.16b, v9.16b tbl v2.16b, {v2.16b}, v3.16b ld1 {v4.2d}, [x22], #16 sshr v10.16b, v2.16b, #7 shl v9.16b, v2.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v2.8h eor v11.16b, v10.16b, v2.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v2.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v4.16b sri v9.4s, v2.4s, #8 sri v8.4s, v11.4s, #24 eor v2.16b, v10.16b, v9.16b eor v2.16b, v2.16b, v8.16b subs w21, w21, #2 bne L_AES_XTS_encrypt_NEON_loop_nr_tweak eor v8.16b, v2.16b, v12.16b eor v9.16b, v2.16b, v13.16b eor v10.16b, v2.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b ld1 {v2.2d}, [x22], #16 sshr v10.16b, v4.16b, #7 shl v9.16b, v4.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v4.8h eor v11.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v4.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v2.16b sri v9.4s, v4.4s, #8 sri v8.4s, v11.4s, #24 eor v4.16b, v10.16b, v9.16b eor v4.16b, v4.16b, v8.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v2.16b, v2.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v2.16b, v2.16b, v9.16b tbl v2.16b, {v2.16b}, v3.16b ld1 {v4.2d}, [x22], #16 # XOR in Key Schedule eor v2.16b, v2.16b, v4.16b rev32 v2.16b, v2.16b mov x8, v2.d[0] mov x9, v2.d[1] cmp w2, #0x40 blt L_AES_XTS_encrypt_NEON_start_2 L_AES_XTS_encrypt_NEON_loop_4: mov x22, x4 ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 ld1 {v4.16b}, [x22], #16 and x16, x17, x9, asr 63 extr x11, x9, x8, #63 eor x10, x16, x8, lsl 1 and x16, x17, x11, asr 63 extr x13, x11, x10, #63 eor x12, x16, x10, lsl 1 and x16, x17, x13, asr 63 extr x15, x13, x12, #63 eor x14, x16, x12, lsl 1 mov v8.d[0], x8 mov v8.d[1], x9 mov v9.d[0], x10 mov v9.d[1], x11 mov v10.d[0], x12 mov v10.d[1], x13 mov v11.d[0], x14 mov v11.d[1], x15 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b rev32 v2.16b, v2.16b rev32 v3.16b, v3.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b sub w21, w7, #2 L_AES_XTS_encrypt_NEON_loop_nr_4: tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b eor v10.16b, v2.16b, v12.16b eor v11.16b, v3.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v13.16b eor v9.16b, v1.16b, v13.16b eor v10.16b, v2.16b, v13.16b eor v11.16b, v3.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b eor v10.16b, v2.16b, v14.16b eor v11.16b, v3.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b ld1 {v0.16b}, [x20] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b tbl v6.16b, {v6.16b}, v0.16b tbl v7.16b, {v7.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 sshr v10.16b, v6.16b, #7 sshr v11.16b, v7.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 shl v14.16b, v6.16b, #1 shl v15.16b, v7.16b, #1 movi v0.16b, #27 and v8.16b, v8.16b, v0.16b and v9.16b, v9.16b, v0.16b and v10.16b, v10.16b, v0.16b and v11.16b, v11.16b, v0.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b eor v2.16b, v10.16b, v6.16b eor v3.16b, v11.16b, v7.16b shl v12.4s, v0.4s, #8 shl v13.4s, v1.4s, #8 shl v14.4s, v2.4s, #8 shl v15.4s, v3.4s, #8 sri v12.4s, v0.4s, #24 sri v13.4s, v1.4s, #24 sri v14.4s, v2.4s, #24 sri v15.4s, v3.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 shl v2.4s, v6.4s, #24 shl v3.4s, v7.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 sri v2.4s, v6.4s, #8 sri v3.4s, v7.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h rev32 v6.8h, v6.8h rev32 v7.8h, v7.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b eor v6.16b, v6.16b, v2.16b eor v7.16b, v7.16b, v3.16b # XOR in Key Schedule ld1 {v0.2d}, [x22], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v6.16b, v6.16b, v0.16b eor v7.16b, v7.16b, v0.16b eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b eor v6.16b, v6.16b, v14.16b eor v7.16b, v7.16b, v15.16b # Round Done tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b eor v10.16b, v6.16b, v12.16b eor v11.16b, v7.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v13.16b eor v9.16b, v5.16b, v13.16b eor v10.16b, v6.16b, v13.16b eor v11.16b, v7.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b eor v10.16b, v6.16b, v14.16b eor v11.16b, v7.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b ld1 {v4.16b}, [x20] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b tbl v2.16b, {v2.16b}, v4.16b tbl v3.16b, {v3.16b}, v4.16b sshr v8.16b, v0.16b, #7 sshr v9.16b, v1.16b, #7 sshr v10.16b, v2.16b, #7 sshr v11.16b, v3.16b, #7 shl v12.16b, v0.16b, #1 shl v13.16b, v1.16b, #1 shl v14.16b, v2.16b, #1 shl v15.16b, v3.16b, #1 movi v4.16b, #27 and v8.16b, v8.16b, v4.16b and v9.16b, v9.16b, v4.16b and v10.16b, v10.16b, v4.16b and v11.16b, v11.16b, v4.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b eor v4.16b, v8.16b, v0.16b eor v5.16b, v9.16b, v1.16b eor v6.16b, v10.16b, v2.16b eor v7.16b, v11.16b, v3.16b shl v12.4s, v4.4s, #8 shl v13.4s, v5.4s, #8 shl v14.4s, v6.4s, #8 shl v15.4s, v7.4s, #8 sri v12.4s, v4.4s, #24 sri v13.4s, v5.4s, #24 sri v14.4s, v6.4s, #24 sri v15.4s, v7.4s, #24 shl v4.4s, v0.4s, #24 shl v5.4s, v1.4s, #24 shl v6.4s, v2.4s, #24 shl v7.4s, v3.4s, #24 sri v4.4s, v0.4s, #8 sri v5.4s, v1.4s, #8 sri v6.4s, v2.4s, #8 sri v7.4s, v3.4s, #8 rev32 v0.8h, v0.8h rev32 v1.8h, v1.8h rev32 v2.8h, v2.8h rev32 v3.8h, v3.8h eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b # XOR in Key Schedule ld1 {v4.2d}, [x22], #16 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b eor v0.16b, v0.16b, v12.16b eor v1.16b, v1.16b, v13.16b eor v2.16b, v2.16b, v14.16b eor v3.16b, v3.16b, v15.16b # Round Done subs w21, w21, #2 bne L_AES_XTS_encrypt_NEON_loop_nr_4 tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b eor v10.16b, v2.16b, v12.16b eor v11.16b, v3.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v13.16b eor v9.16b, v1.16b, v13.16b eor v10.16b, v2.16b, v13.16b eor v11.16b, v3.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b eor v10.16b, v2.16b, v14.16b eor v11.16b, v3.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b ld1 {v0.16b}, [x20] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b tbl v6.16b, {v6.16b}, v0.16b tbl v7.16b, {v7.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 sshr v10.16b, v6.16b, #7 sshr v11.16b, v7.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 shl v14.16b, v6.16b, #1 shl v15.16b, v7.16b, #1 movi v0.16b, #27 and v8.16b, v8.16b, v0.16b and v9.16b, v9.16b, v0.16b and v10.16b, v10.16b, v0.16b and v11.16b, v11.16b, v0.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b eor v2.16b, v10.16b, v6.16b eor v3.16b, v11.16b, v7.16b shl v12.4s, v0.4s, #8 shl v13.4s, v1.4s, #8 shl v14.4s, v2.4s, #8 shl v15.4s, v3.4s, #8 sri v12.4s, v0.4s, #24 sri v13.4s, v1.4s, #24 sri v14.4s, v2.4s, #24 sri v15.4s, v3.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 shl v2.4s, v6.4s, #24 shl v3.4s, v7.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 sri v2.4s, v6.4s, #8 sri v3.4s, v7.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h rev32 v6.8h, v6.8h rev32 v7.8h, v7.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b eor v6.16b, v6.16b, v2.16b eor v7.16b, v7.16b, v3.16b # XOR in Key Schedule ld1 {v0.2d}, [x22], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v6.16b, v6.16b, v0.16b eor v7.16b, v7.16b, v0.16b eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b eor v6.16b, v6.16b, v14.16b eor v7.16b, v7.16b, v15.16b # Round Done tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b eor v10.16b, v6.16b, v12.16b eor v11.16b, v7.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v13.16b eor v9.16b, v5.16b, v13.16b eor v10.16b, v6.16b, v13.16b eor v11.16b, v7.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b eor v10.16b, v6.16b, v14.16b eor v11.16b, v7.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b ld1 {v4.16b}, [x20] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b tbl v2.16b, {v2.16b}, v4.16b tbl v3.16b, {v3.16b}, v4.16b # XOR in Key Schedule ld1 {v4.2d}, [x22], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b # Round Done rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b rev32 v2.16b, v2.16b rev32 v3.16b, v3.16b mov v8.d[0], x8 mov v8.d[1], x9 mov v9.d[0], x10 mov v9.d[1], x11 mov v10.d[0], x12 mov v10.d[1], x13 mov v11.d[0], x14 mov v11.d[1], x15 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 and x16, x17, x15, asr 63 extr x9, x15, x14, #63 eor x8, x16, x14, lsl 1 sub w2, w2, #0x40 cmp w2, #0x40 bge L_AES_XTS_encrypt_NEON_loop_4 movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 movi v15.16b, #27 L_AES_XTS_encrypt_NEON_start_2: cmp w2, #32 blt L_AES_XTS_encrypt_NEON_start_1 mov x22, x4 ld1 {v0.16b, v1.16b}, [x0], #32 ld1 {v4.16b}, [x22], #16 and x16, x17, x9, asr 63 extr x11, x9, x8, #63 eor x10, x16, x8, lsl 1 and x16, x17, x11, asr 63 extr x13, x11, x10, #63 eor x12, x16, x10, lsl 1 mov v2.d[0], x8 mov v2.d[1], x9 mov v3.d[0], x10 mov v3.d[1], x11 eor v0.16b, v0.16b, v2.16b eor v1.16b, v1.16b, v3.16b rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b sub w21, w7, #2 L_AES_XTS_encrypt_NEON_loop_nr_2: eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v0.16b, v13.16b eor v11.16b, v1.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b orr v4.16b, v4.16b, v10.16b orr v5.16b, v5.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b ld1 {v0.16b}, [x20] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 shl v10.16b, v4.16b, #1 shl v11.16b, v5.16b, #1 and v8.16b, v8.16b, v15.16b and v9.16b, v9.16b, v15.16b eor v8.16b, v8.16b, v10.16b eor v9.16b, v9.16b, v11.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b shl v10.4s, v0.4s, #8 shl v11.4s, v1.4s, #8 sri v10.4s, v0.4s, #24 sri v11.4s, v1.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b # XOR in Key Schedule ld1 {v0.2d}, [x22], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v4.16b, v4.16b, v10.16b eor v5.16b, v5.16b, v11.16b # Round Done eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v4.16b, v13.16b eor v11.16b, v5.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b orr v0.16b, v0.16b, v10.16b orr v1.16b, v1.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b ld1 {v4.16b}, [x20] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b sshr v8.16b, v0.16b, #7 sshr v9.16b, v1.16b, #7 shl v10.16b, v0.16b, #1 shl v11.16b, v1.16b, #1 and v8.16b, v8.16b, v15.16b and v9.16b, v9.16b, v15.16b eor v8.16b, v8.16b, v10.16b eor v9.16b, v9.16b, v11.16b eor v4.16b, v8.16b, v0.16b eor v5.16b, v9.16b, v1.16b shl v10.4s, v4.4s, #8 shl v11.4s, v5.4s, #8 sri v10.4s, v4.4s, #24 sri v11.4s, v5.4s, #24 shl v4.4s, v0.4s, #24 shl v5.4s, v1.4s, #24 sri v4.4s, v0.4s, #8 sri v5.4s, v1.4s, #8 rev32 v0.8h, v0.8h rev32 v1.8h, v1.8h eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b # XOR in Key Schedule ld1 {v4.2d}, [x22], #16 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v0.16b, v0.16b, v10.16b eor v1.16b, v1.16b, v11.16b # Round Done subs w21, w21, #2 bne L_AES_XTS_encrypt_NEON_loop_nr_2 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v0.16b, v13.16b eor v11.16b, v1.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b orr v4.16b, v4.16b, v10.16b orr v5.16b, v5.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b ld1 {v0.16b}, [x20] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 shl v10.16b, v4.16b, #1 shl v11.16b, v5.16b, #1 and v8.16b, v8.16b, v15.16b and v9.16b, v9.16b, v15.16b eor v8.16b, v8.16b, v10.16b eor v9.16b, v9.16b, v11.16b eor v0.16b, v8.16b, v4.16b eor v1.16b, v9.16b, v5.16b shl v10.4s, v0.4s, #8 shl v11.4s, v1.4s, #8 sri v10.4s, v0.4s, #24 sri v11.4s, v1.4s, #24 shl v0.4s, v4.4s, #24 shl v1.4s, v5.4s, #24 sri v0.4s, v4.4s, #8 sri v1.4s, v5.4s, #8 rev32 v4.8h, v4.8h rev32 v5.8h, v5.8h eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b # XOR in Key Schedule ld1 {v0.2d}, [x22], #16 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v4.16b, v4.16b, v10.16b eor v5.16b, v5.16b, v11.16b # Round Done eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v4.16b, v13.16b eor v11.16b, v5.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b orr v0.16b, v0.16b, v10.16b orr v1.16b, v1.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b ld1 {v4.16b}, [x20] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b # XOR in Key Schedule ld1 {v4.2d}, [x22], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b # Round Done rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b eor v0.16b, v0.16b, v2.16b eor v1.16b, v1.16b, v3.16b st1 {v0.16b, v1.16b}, [x1], #32 and x16, x17, x11, asr 63 extr x9, x11, x10, #63 eor x8, x16, x10, lsl 1 sub w2, w2, #32 L_AES_XTS_encrypt_NEON_start_1: ld1 {v3.2d}, [x20] mov v2.d[0], x8 mov v2.d[1], x9 cmp w2, #16 blt L_AES_XTS_encrypt_NEON_start_partial mov x22, x4 ld1 {v0.16b}, [x0], #16 ld1 {v4.2d}, [x22], #16 eor v0.16b, v0.16b, v2.16b rev32 v0.16b, v0.16b eor v0.16b, v0.16b, v4.16b sub w21, w7, #2 L_AES_XTS_encrypt_NEON_loop_nr_1: eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b ld1 {v0.2d}, [x22], #16 sshr v10.16b, v4.16b, #7 shl v9.16b, v4.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v4.8h eor v11.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v4.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v0.16b sri v9.4s, v4.4s, #8 sri v8.4s, v11.4s, #24 eor v4.16b, v10.16b, v9.16b eor v4.16b, v4.16b, v8.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b ld1 {v4.2d}, [x22], #16 sshr v10.16b, v0.16b, #7 shl v9.16b, v0.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v0.8h eor v11.16b, v10.16b, v0.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v0.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v4.16b sri v9.4s, v0.4s, #8 sri v8.4s, v11.4s, #24 eor v0.16b, v10.16b, v9.16b eor v0.16b, v0.16b, v8.16b subs w21, w21, #2 bne L_AES_XTS_encrypt_NEON_loop_nr_1 eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b ld1 {v0.2d}, [x22], #16 sshr v10.16b, v4.16b, #7 shl v9.16b, v4.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v4.8h eor v11.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v4.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v0.16b sri v9.4s, v4.4s, #8 sri v8.4s, v11.4s, #24 eor v4.16b, v10.16b, v9.16b eor v4.16b, v4.16b, v8.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b ld1 {v4.2d}, [x22], #16 # XOR in Key Schedule eor v0.16b, v0.16b, v4.16b rev32 v0.16b, v0.16b eor v0.16b, v0.16b, v2.16b st1 {v0.16b}, [x1], #16 subs w2, w2, #16 beq L_AES_XTS_encrypt_NEON_data_done and x16, x17, x9, asr 63 extr x9, x9, x8, #63 eor x8, x16, x8, lsl 1 L_AES_XTS_encrypt_NEON_start_partial: cbz w2, L_AES_XTS_encrypt_NEON_data_done mov v2.d[0], x8 mov v2.d[1], x9 mov x22, x4 sub x1, x1, #16 ld1 {v0.16b}, [x1], #16 st1 {v0.2d}, [x6] mov w16, w2 L_AES_XTS_encrypt_NEON_start_byte: ldrb w10, [x6] ldrb w11, [x0], #1 strb w10, [x1], #1 strb w11, [x6], #1 subs w16, w16, #1 bgt L_AES_XTS_encrypt_NEON_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 ld1 {v0.2d}, [x6] ld1 {v4.2d}, [x22], #16 eor v0.16b, v0.16b, v2.16b rev32 v0.16b, v0.16b eor v0.16b, v0.16b, v4.16b sub w21, w7, #2 L_AES_XTS_encrypt_NEON_loop_nr_partial: eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b ld1 {v0.2d}, [x22], #16 sshr v10.16b, v4.16b, #7 shl v9.16b, v4.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v4.8h eor v11.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v4.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v0.16b sri v9.4s, v4.4s, #8 sri v8.4s, v11.4s, #24 eor v4.16b, v10.16b, v9.16b eor v4.16b, v4.16b, v8.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b ld1 {v4.2d}, [x22], #16 sshr v10.16b, v0.16b, #7 shl v9.16b, v0.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v0.8h eor v11.16b, v10.16b, v0.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v0.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v4.16b sri v9.4s, v0.4s, #8 sri v8.4s, v11.4s, #24 eor v0.16b, v10.16b, v9.16b eor v0.16b, v0.16b, v8.16b subs w21, w21, #2 bne L_AES_XTS_encrypt_NEON_loop_nr_partial eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b ld1 {v0.2d}, [x22], #16 sshr v10.16b, v4.16b, #7 shl v9.16b, v4.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v4.8h eor v11.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v4.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v0.16b sri v9.4s, v4.4s, #8 sri v8.4s, v11.4s, #24 eor v4.16b, v10.16b, v9.16b eor v4.16b, v4.16b, v8.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b ld1 {v4.2d}, [x22], #16 # XOR in Key Schedule eor v0.16b, v0.16b, v4.16b rev32 v0.16b, v0.16b eor v0.16b, v0.16b, v2.16b st1 {v0.16b}, [x1] L_AES_XTS_encrypt_NEON_data_done: ldp x17, x19, [x29, #24] ldp x20, x21, [x29, #40] ldr x22, [x29, #56] ldp d8, d9, [x29, #64] ldp d10, d11, [x29, #80] ldp d12, d13, [x29, #96] ldp d14, d15, [x29, #112] ldp x29, x30, [sp], #0x80 ret #ifndef __APPLE__ .size AES_XTS_encrypt_NEON,.-AES_XTS_encrypt_NEON #endif /* __APPLE__ */ #ifdef HAVE_AES_DECRYPT #ifndef __APPLE__ .text .globl AES_XTS_decrypt_NEON .type AES_XTS_decrypt_NEON,@function .align 2 AES_XTS_decrypt_NEON: #else .section __TEXT,__text .globl _AES_XTS_decrypt_NEON .p2align 2 _AES_XTS_decrypt_NEON: #endif /* __APPLE__ */ stp x29, x30, [sp, #-144]! add x29, sp, #0 stp x17, x19, [x29, #16] stp x20, x21, [x29, #32] stp x22, x23, [x29, #48] stp x24, x25, [x29, #64] stp d8, d9, [x29, #80] stp d10, d11, [x29, #96] stp d12, d13, [x29, #112] stp d14, d15, [x29, #128] #ifndef __APPLE__ adrp x20, L_AES_ARM64_NEON_te add x20, x20, :lo12:L_AES_ARM64_NEON_te #else adrp x20, L_AES_ARM64_NEON_te@PAGE add x20, x20, L_AES_ARM64_NEON_te@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x21, L_AES_ARM64_NEON_td add x21, x21, :lo12:L_AES_ARM64_NEON_td #else adrp x21, L_AES_ARM64_NEON_td@PAGE add x21, x21, L_AES_ARM64_NEON_td@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x22, L_AES_ARM64_NEON_shift_rows_shuffle add x22, x22, :lo12:L_AES_ARM64_NEON_shift_rows_shuffle #else adrp x22, L_AES_ARM64_NEON_shift_rows_shuffle@PAGE add x22, x22, L_AES_ARM64_NEON_shift_rows_shuffle@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x23, L_AES_ARM64_NEON_shift_rows_invshuffle add x23, x23, :lo12:L_AES_ARM64_NEON_shift_rows_invshuffle #else adrp x23, L_AES_ARM64_NEON_shift_rows_invshuffle@PAGE add x23, x23, L_AES_ARM64_NEON_shift_rows_invshuffle@PAGEOFF #endif /* __APPLE__ */ ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x20], #0x40 ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x20], #0x40 ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x20], #0x40 ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x20] movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 movi v15.16b, #27 ld1 {v3.2d}, [x22] mov x17, #0x87 ands w19, w2, #15 cset w16, ne lsl w16, w16, #4 sub w2, w2, w16 ld1 {v2.2d}, [x3] ld1 {v4.2d}, [x5] rev32 v2.16b, v2.16b add x25, x5, #16 # Round: 0 - XOR in key schedule eor v2.16b, v2.16b, v4.16b sub w24, w7, #2 L_AES_XTS_decrypt_NEON_loop_nr_tweak: eor v8.16b, v2.16b, v12.16b eor v9.16b, v2.16b, v13.16b eor v10.16b, v2.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b ld1 {v2.2d}, [x25], #16 sshr v10.16b, v4.16b, #7 shl v9.16b, v4.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v4.8h eor v11.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v4.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v2.16b sri v9.4s, v4.4s, #8 sri v8.4s, v11.4s, #24 eor v4.16b, v10.16b, v9.16b eor v4.16b, v4.16b, v8.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v2.16b, v2.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v2.16b, v2.16b, v9.16b tbl v2.16b, {v2.16b}, v3.16b ld1 {v4.2d}, [x25], #16 sshr v10.16b, v2.16b, #7 shl v9.16b, v2.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v2.8h eor v11.16b, v10.16b, v2.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v2.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v4.16b sri v9.4s, v2.4s, #8 sri v8.4s, v11.4s, #24 eor v2.16b, v10.16b, v9.16b eor v2.16b, v2.16b, v8.16b subs w24, w24, #2 bne L_AES_XTS_decrypt_NEON_loop_nr_tweak eor v8.16b, v2.16b, v12.16b eor v9.16b, v2.16b, v13.16b eor v10.16b, v2.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b ld1 {v2.2d}, [x25], #16 sshr v10.16b, v4.16b, #7 shl v9.16b, v4.16b, #1 and v10.16b, v10.16b, v15.16b eor v10.16b, v10.16b, v9.16b rev32 v8.8h, v4.8h eor v11.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v8.16b shl v9.4s, v4.4s, #24 shl v8.4s, v11.4s, #8 # XOR in Key Schedule eor v10.16b, v10.16b, v2.16b sri v9.4s, v4.4s, #8 sri v8.4s, v11.4s, #24 eor v4.16b, v10.16b, v9.16b eor v4.16b, v4.16b, v8.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v2.16b, v2.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v2.16b, v2.16b, v9.16b tbl v2.16b, {v2.16b}, v3.16b ld1 {v4.2d}, [x25], #16 # XOR in Key Schedule eor v2.16b, v2.16b, v4.16b rev32 v2.16b, v2.16b mov x8, v2.d[0] mov x9, v2.d[1] ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x21], #0x40 ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x21], #0x40 ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x21], #0x40 ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x21] ld1 {v3.2d}, [x23] cmp w2, #0x40 blt L_AES_XTS_decrypt_NEON_start_2 L_AES_XTS_decrypt_NEON_loop_4: mov x25, x4 ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 ld1 {v4.16b}, [x25], #16 and x16, x17, x9, asr 63 extr x11, x9, x8, #63 eor x10, x16, x8, lsl 1 and x16, x17, x11, asr 63 extr x13, x11, x10, #63 eor x12, x16, x10, lsl 1 and x16, x17, x13, asr 63 extr x15, x13, x12, #63 eor x14, x16, x12, lsl 1 mov v8.d[0], x8 mov v8.d[1], x9 mov v9.d[0], x10 mov v9.d[1], x11 mov v10.d[0], x12 mov v10.d[1], x13 mov v11.d[0], x14 mov v11.d[1], x15 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b rev32 v2.16b, v2.16b rev32 v3.16b, v3.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b sub w24, w7, #2 L_AES_XTS_decrypt_NEON_loop_nr_4: tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b eor v10.16b, v2.16b, v12.16b eor v11.16b, v3.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v13.16b eor v9.16b, v1.16b, v13.16b eor v10.16b, v2.16b, v13.16b eor v11.16b, v3.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b eor v10.16b, v2.16b, v14.16b eor v11.16b, v3.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b ld1 {v0.16b}, [x23] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b tbl v6.16b, {v6.16b}, v0.16b tbl v7.16b, {v7.16b}, v0.16b movi v28.16b, #27 sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 sshr v10.16b, v6.16b, #7 sshr v11.16b, v7.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 shl v14.16b, v6.16b, #1 shl v15.16b, v7.16b, #1 and v8.16b, v8.16b, v28.16b and v9.16b, v9.16b, v28.16b and v10.16b, v10.16b, v28.16b and v11.16b, v11.16b, v28.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b ushr v12.16b, v4.16b, #6 ushr v13.16b, v5.16b, #6 ushr v14.16b, v6.16b, #6 ushr v15.16b, v7.16b, #6 shl v0.16b, v4.16b, #2 shl v1.16b, v5.16b, #2 shl v2.16b, v6.16b, #2 shl v3.16b, v7.16b, #2 pmul v12.16b, v12.16b, v28.16b pmul v13.16b, v13.16b, v28.16b pmul v14.16b, v14.16b, v28.16b pmul v15.16b, v15.16b, v28.16b eor v12.16b, v12.16b, v0.16b eor v13.16b, v13.16b, v1.16b eor v14.16b, v14.16b, v2.16b eor v15.16b, v15.16b, v3.16b ushr v0.16b, v4.16b, #5 ushr v1.16b, v5.16b, #5 ushr v2.16b, v6.16b, #5 ushr v3.16b, v7.16b, #5 pmul v0.16b, v0.16b, v28.16b pmul v1.16b, v1.16b, v28.16b pmul v2.16b, v2.16b, v28.16b pmul v3.16b, v3.16b, v28.16b shl v28.16b, v4.16b, #3 shl v29.16b, v5.16b, #3 shl v30.16b, v6.16b, #3 shl v31.16b, v7.16b, #3 eor v0.16b, v0.16b, v28.16b eor v1.16b, v1.16b, v29.16b eor v2.16b, v2.16b, v30.16b eor v3.16b, v3.16b, v31.16b eor v28.16b, v8.16b, v0.16b eor v29.16b, v9.16b, v1.16b eor v30.16b, v10.16b, v2.16b eor v31.16b, v11.16b, v3.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b eor v8.16b, v12.16b, v0.16b eor v9.16b, v13.16b, v1.16b eor v10.16b, v14.16b, v2.16b eor v11.16b, v15.16b, v3.16b eor v12.16b, v12.16b, v28.16b eor v13.16b, v13.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v15.16b, v15.16b, v31.16b eor v28.16b, v28.16b, v4.16b eor v29.16b, v29.16b, v5.16b eor v30.16b, v30.16b, v6.16b eor v31.16b, v31.16b, v7.16b shl v4.4s, v28.4s, #8 shl v5.4s, v29.4s, #8 shl v6.4s, v30.4s, #8 shl v7.4s, v31.4s, #8 rev32 v8.8h, v8.8h rev32 v9.8h, v9.8h rev32 v10.8h, v10.8h rev32 v11.8h, v11.8h sri v4.4s, v28.4s, #24 sri v5.4s, v29.4s, #24 sri v6.4s, v30.4s, #24 sri v7.4s, v31.4s, #24 eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b eor v6.16b, v6.16b, v14.16b eor v7.16b, v7.16b, v15.16b shl v28.4s, v0.4s, #24 shl v29.4s, v1.4s, #24 shl v30.4s, v2.4s, #24 shl v31.4s, v3.4s, #24 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b sri v28.4s, v0.4s, #8 sri v29.4s, v1.4s, #8 sri v30.4s, v2.4s, #8 sri v31.4s, v3.4s, #8 eor v4.16b, v4.16b, v28.16b eor v5.16b, v5.16b, v29.16b eor v6.16b, v6.16b, v30.16b eor v7.16b, v7.16b, v31.16b ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x21] # XOR in Key Schedule ld1 {v0.2d}, [x25], #16 eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v6.16b, v6.16b, v0.16b eor v7.16b, v7.16b, v0.16b # Round Done tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b eor v10.16b, v6.16b, v12.16b eor v11.16b, v7.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v13.16b eor v9.16b, v5.16b, v13.16b eor v10.16b, v6.16b, v13.16b eor v11.16b, v7.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b eor v10.16b, v6.16b, v14.16b eor v11.16b, v7.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b ld1 {v4.16b}, [x23] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b tbl v2.16b, {v2.16b}, v4.16b tbl v3.16b, {v3.16b}, v4.16b movi v28.16b, #27 sshr v8.16b, v0.16b, #7 sshr v9.16b, v1.16b, #7 sshr v10.16b, v2.16b, #7 sshr v11.16b, v3.16b, #7 shl v12.16b, v0.16b, #1 shl v13.16b, v1.16b, #1 shl v14.16b, v2.16b, #1 shl v15.16b, v3.16b, #1 and v8.16b, v8.16b, v28.16b and v9.16b, v9.16b, v28.16b and v10.16b, v10.16b, v28.16b and v11.16b, v11.16b, v28.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b ushr v12.16b, v0.16b, #6 ushr v13.16b, v1.16b, #6 ushr v14.16b, v2.16b, #6 ushr v15.16b, v3.16b, #6 shl v4.16b, v0.16b, #2 shl v5.16b, v1.16b, #2 shl v6.16b, v2.16b, #2 shl v7.16b, v3.16b, #2 pmul v12.16b, v12.16b, v28.16b pmul v13.16b, v13.16b, v28.16b pmul v14.16b, v14.16b, v28.16b pmul v15.16b, v15.16b, v28.16b eor v12.16b, v12.16b, v4.16b eor v13.16b, v13.16b, v5.16b eor v14.16b, v14.16b, v6.16b eor v15.16b, v15.16b, v7.16b ushr v4.16b, v0.16b, #5 ushr v5.16b, v1.16b, #5 ushr v6.16b, v2.16b, #5 ushr v7.16b, v3.16b, #5 pmul v4.16b, v4.16b, v28.16b pmul v5.16b, v5.16b, v28.16b pmul v6.16b, v6.16b, v28.16b pmul v7.16b, v7.16b, v28.16b shl v28.16b, v0.16b, #3 shl v29.16b, v1.16b, #3 shl v30.16b, v2.16b, #3 shl v31.16b, v3.16b, #3 eor v4.16b, v4.16b, v28.16b eor v5.16b, v5.16b, v29.16b eor v6.16b, v6.16b, v30.16b eor v7.16b, v7.16b, v31.16b eor v28.16b, v8.16b, v4.16b eor v29.16b, v9.16b, v5.16b eor v30.16b, v10.16b, v6.16b eor v31.16b, v11.16b, v7.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b eor v6.16b, v6.16b, v2.16b eor v7.16b, v7.16b, v3.16b eor v8.16b, v12.16b, v4.16b eor v9.16b, v13.16b, v5.16b eor v10.16b, v14.16b, v6.16b eor v11.16b, v15.16b, v7.16b eor v12.16b, v12.16b, v28.16b eor v13.16b, v13.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v15.16b, v15.16b, v31.16b eor v28.16b, v28.16b, v0.16b eor v29.16b, v29.16b, v1.16b eor v30.16b, v30.16b, v2.16b eor v31.16b, v31.16b, v3.16b shl v0.4s, v28.4s, #8 shl v1.4s, v29.4s, #8 shl v2.4s, v30.4s, #8 shl v3.4s, v31.4s, #8 rev32 v8.8h, v8.8h rev32 v9.8h, v9.8h rev32 v10.8h, v10.8h rev32 v11.8h, v11.8h sri v0.4s, v28.4s, #24 sri v1.4s, v29.4s, #24 sri v2.4s, v30.4s, #24 sri v3.4s, v31.4s, #24 eor v0.16b, v0.16b, v12.16b eor v1.16b, v1.16b, v13.16b eor v2.16b, v2.16b, v14.16b eor v3.16b, v3.16b, v15.16b shl v28.4s, v4.4s, #24 shl v29.4s, v5.4s, #24 shl v30.4s, v6.4s, #24 shl v31.4s, v7.4s, #24 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b sri v28.4s, v4.4s, #8 sri v29.4s, v5.4s, #8 sri v30.4s, v6.4s, #8 sri v31.4s, v7.4s, #8 eor v0.16b, v0.16b, v28.16b eor v1.16b, v1.16b, v29.16b eor v2.16b, v2.16b, v30.16b eor v3.16b, v3.16b, v31.16b ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x21] # XOR in Key Schedule ld1 {v4.2d}, [x25], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b # Round Done subs w24, w24, #2 bne L_AES_XTS_decrypt_NEON_loop_nr_4 tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b eor v10.16b, v2.16b, v12.16b eor v11.16b, v3.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v13.16b eor v9.16b, v1.16b, v13.16b eor v10.16b, v2.16b, v13.16b eor v11.16b, v3.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b eor v10.16b, v2.16b, v14.16b eor v11.16b, v3.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b orr v6.16b, v6.16b, v10.16b orr v7.16b, v7.16b, v11.16b ld1 {v0.16b}, [x23] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b tbl v6.16b, {v6.16b}, v0.16b tbl v7.16b, {v7.16b}, v0.16b movi v28.16b, #27 sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 sshr v10.16b, v6.16b, #7 sshr v11.16b, v7.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 shl v14.16b, v6.16b, #1 shl v15.16b, v7.16b, #1 and v8.16b, v8.16b, v28.16b and v9.16b, v9.16b, v28.16b and v10.16b, v10.16b, v28.16b and v11.16b, v11.16b, v28.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b eor v10.16b, v10.16b, v14.16b eor v11.16b, v11.16b, v15.16b ushr v12.16b, v4.16b, #6 ushr v13.16b, v5.16b, #6 ushr v14.16b, v6.16b, #6 ushr v15.16b, v7.16b, #6 shl v0.16b, v4.16b, #2 shl v1.16b, v5.16b, #2 shl v2.16b, v6.16b, #2 shl v3.16b, v7.16b, #2 pmul v12.16b, v12.16b, v28.16b pmul v13.16b, v13.16b, v28.16b pmul v14.16b, v14.16b, v28.16b pmul v15.16b, v15.16b, v28.16b eor v12.16b, v12.16b, v0.16b eor v13.16b, v13.16b, v1.16b eor v14.16b, v14.16b, v2.16b eor v15.16b, v15.16b, v3.16b ushr v0.16b, v4.16b, #5 ushr v1.16b, v5.16b, #5 ushr v2.16b, v6.16b, #5 ushr v3.16b, v7.16b, #5 pmul v0.16b, v0.16b, v28.16b pmul v1.16b, v1.16b, v28.16b pmul v2.16b, v2.16b, v28.16b pmul v3.16b, v3.16b, v28.16b shl v28.16b, v4.16b, #3 shl v29.16b, v5.16b, #3 shl v30.16b, v6.16b, #3 shl v31.16b, v7.16b, #3 eor v0.16b, v0.16b, v28.16b eor v1.16b, v1.16b, v29.16b eor v2.16b, v2.16b, v30.16b eor v3.16b, v3.16b, v31.16b eor v28.16b, v8.16b, v0.16b eor v29.16b, v9.16b, v1.16b eor v30.16b, v10.16b, v2.16b eor v31.16b, v11.16b, v3.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v2.16b, v2.16b, v6.16b eor v3.16b, v3.16b, v7.16b eor v8.16b, v12.16b, v0.16b eor v9.16b, v13.16b, v1.16b eor v10.16b, v14.16b, v2.16b eor v11.16b, v15.16b, v3.16b eor v12.16b, v12.16b, v28.16b eor v13.16b, v13.16b, v29.16b eor v14.16b, v14.16b, v30.16b eor v15.16b, v15.16b, v31.16b eor v28.16b, v28.16b, v4.16b eor v29.16b, v29.16b, v5.16b eor v30.16b, v30.16b, v6.16b eor v31.16b, v31.16b, v7.16b shl v4.4s, v28.4s, #8 shl v5.4s, v29.4s, #8 shl v6.4s, v30.4s, #8 shl v7.4s, v31.4s, #8 rev32 v8.8h, v8.8h rev32 v9.8h, v9.8h rev32 v10.8h, v10.8h rev32 v11.8h, v11.8h sri v4.4s, v28.4s, #24 sri v5.4s, v29.4s, #24 sri v6.4s, v30.4s, #24 sri v7.4s, v31.4s, #24 eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b eor v6.16b, v6.16b, v14.16b eor v7.16b, v7.16b, v15.16b shl v28.4s, v0.4s, #24 shl v29.4s, v1.4s, #24 shl v30.4s, v2.4s, #24 shl v31.4s, v3.4s, #24 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b eor v6.16b, v6.16b, v10.16b eor v7.16b, v7.16b, v11.16b sri v28.4s, v0.4s, #8 sri v29.4s, v1.4s, #8 sri v30.4s, v2.4s, #8 sri v31.4s, v3.4s, #8 eor v4.16b, v4.16b, v28.16b eor v5.16b, v5.16b, v29.16b eor v6.16b, v6.16b, v30.16b eor v7.16b, v7.16b, v31.16b ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x21] # XOR in Key Schedule ld1 {v0.2d}, [x25], #16 eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b eor v6.16b, v6.16b, v0.16b eor v7.16b, v7.16b, v0.16b # Round Done tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b eor v10.16b, v6.16b, v12.16b eor v11.16b, v7.16b, v12.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v13.16b eor v9.16b, v5.16b, v13.16b eor v10.16b, v6.16b, v13.16b eor v11.16b, v7.16b, v13.16b tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b eor v10.16b, v6.16b, v14.16b eor v11.16b, v7.16b, v14.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b orr v2.16b, v2.16b, v10.16b orr v3.16b, v3.16b, v11.16b ld1 {v4.16b}, [x23] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b tbl v2.16b, {v2.16b}, v4.16b tbl v3.16b, {v3.16b}, v4.16b # XOR in Key Schedule ld1 {v4.2d}, [x25], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b eor v2.16b, v2.16b, v4.16b eor v3.16b, v3.16b, v4.16b # Round Done rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b rev32 v2.16b, v2.16b rev32 v3.16b, v3.16b mov v8.d[0], x8 mov v8.d[1], x9 mov v9.d[0], x10 mov v9.d[1], x11 mov v10.d[0], x12 mov v10.d[1], x13 mov v11.d[0], x14 mov v11.d[1], x15 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b eor v2.16b, v2.16b, v10.16b eor v3.16b, v3.16b, v11.16b st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 and x16, x17, x15, asr 63 extr x9, x15, x14, #63 eor x8, x16, x14, lsl 1 sub w2, w2, #0x40 cmp w2, #0x40 bge L_AES_XTS_decrypt_NEON_loop_4 movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 movi v15.16b, #27 L_AES_XTS_decrypt_NEON_start_2: cmp w2, #32 blt L_AES_XTS_decrypt_NEON_start_1 mov x25, x4 ld1 {v0.16b, v1.16b}, [x0], #32 ld1 {v4.16b}, [x25], #16 and x16, x17, x9, asr 63 extr x11, x9, x8, #63 eor x10, x16, x8, lsl 1 and x16, x17, x11, asr 63 extr x13, x11, x10, #63 eor x12, x16, x10, lsl 1 mov v2.d[0], x8 mov v2.d[1], x9 mov v3.d[0], x10 mov v3.d[1], x11 eor v0.16b, v0.16b, v2.16b eor v1.16b, v1.16b, v3.16b rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b sub w24, w7, #2 L_AES_XTS_decrypt_NEON_loop_nr_2: movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v0.16b, v13.16b eor v11.16b, v1.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b orr v4.16b, v4.16b, v10.16b orr v5.16b, v5.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b ld1 {v0.16b}, [x23] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b movi v10.16b, #27 sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 and v8.16b, v8.16b, v10.16b and v9.16b, v9.16b, v10.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b ushr v12.16b, v4.16b, #6 ushr v13.16b, v5.16b, #6 shl v0.16b, v4.16b, #2 shl v1.16b, v5.16b, #2 pmul v12.16b, v12.16b, v10.16b pmul v13.16b, v13.16b, v10.16b eor v12.16b, v12.16b, v0.16b eor v13.16b, v13.16b, v1.16b ushr v0.16b, v4.16b, #5 ushr v1.16b, v5.16b, #5 pmul v0.16b, v0.16b, v10.16b pmul v1.16b, v1.16b, v10.16b shl v10.16b, v4.16b, #3 shl v11.16b, v5.16b, #3 eor v0.16b, v0.16b, v10.16b eor v1.16b, v1.16b, v11.16b eor v10.16b, v8.16b, v0.16b eor v11.16b, v9.16b, v1.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v8.16b, v12.16b, v0.16b eor v9.16b, v13.16b, v1.16b eor v12.16b, v12.16b, v10.16b eor v13.16b, v13.16b, v11.16b eor v10.16b, v10.16b, v4.16b eor v11.16b, v11.16b, v5.16b shl v4.4s, v10.4s, #8 shl v5.4s, v11.4s, #8 rev32 v8.8h, v8.8h rev32 v9.8h, v9.8h sri v4.4s, v10.4s, #24 sri v5.4s, v11.4s, #24 eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b shl v10.4s, v0.4s, #24 shl v11.4s, v1.4s, #24 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b sri v10.4s, v0.4s, #8 sri v11.4s, v1.4s, #8 eor v4.16b, v4.16b, v10.16b eor v5.16b, v5.16b, v11.16b # XOR in Key Schedule ld1 {v0.2d}, [x25], #16 eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b # Round Done movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v4.16b, v13.16b eor v11.16b, v5.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b orr v0.16b, v0.16b, v10.16b orr v1.16b, v1.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b ld1 {v4.16b}, [x23] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b movi v10.16b, #27 sshr v8.16b, v0.16b, #7 sshr v9.16b, v1.16b, #7 shl v12.16b, v0.16b, #1 shl v13.16b, v1.16b, #1 and v8.16b, v8.16b, v10.16b and v9.16b, v9.16b, v10.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b ushr v12.16b, v0.16b, #6 ushr v13.16b, v1.16b, #6 shl v4.16b, v0.16b, #2 shl v5.16b, v1.16b, #2 pmul v12.16b, v12.16b, v10.16b pmul v13.16b, v13.16b, v10.16b eor v12.16b, v12.16b, v4.16b eor v13.16b, v13.16b, v5.16b ushr v4.16b, v0.16b, #5 ushr v5.16b, v1.16b, #5 pmul v4.16b, v4.16b, v10.16b pmul v5.16b, v5.16b, v10.16b shl v10.16b, v0.16b, #3 shl v11.16b, v1.16b, #3 eor v4.16b, v4.16b, v10.16b eor v5.16b, v5.16b, v11.16b eor v10.16b, v8.16b, v4.16b eor v11.16b, v9.16b, v5.16b eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v1.16b eor v8.16b, v12.16b, v4.16b eor v9.16b, v13.16b, v5.16b eor v12.16b, v12.16b, v10.16b eor v13.16b, v13.16b, v11.16b eor v10.16b, v10.16b, v0.16b eor v11.16b, v11.16b, v1.16b shl v0.4s, v10.4s, #8 shl v1.4s, v11.4s, #8 rev32 v8.8h, v8.8h rev32 v9.8h, v9.8h sri v0.4s, v10.4s, #24 sri v1.4s, v11.4s, #24 eor v0.16b, v0.16b, v12.16b eor v1.16b, v1.16b, v13.16b shl v10.4s, v4.4s, #24 shl v11.4s, v5.4s, #24 eor v0.16b, v0.16b, v8.16b eor v1.16b, v1.16b, v9.16b sri v10.4s, v4.4s, #8 sri v11.4s, v5.4s, #8 eor v0.16b, v0.16b, v10.16b eor v1.16b, v1.16b, v11.16b # XOR in Key Schedule ld1 {v4.2d}, [x25], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b # Round Done subs w24, w24, #2 bne L_AES_XTS_decrypt_NEON_loop_nr_2 movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v0.16b, v12.16b eor v9.16b, v1.16b, v12.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v0.16b, v13.16b eor v11.16b, v1.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b eor v8.16b, v0.16b, v14.16b eor v9.16b, v1.16b, v14.16b orr v4.16b, v4.16b, v10.16b orr v5.16b, v5.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v4.16b, v4.16b, v8.16b orr v5.16b, v5.16b, v9.16b ld1 {v0.16b}, [x23] tbl v4.16b, {v4.16b}, v0.16b tbl v5.16b, {v5.16b}, v0.16b movi v10.16b, #27 sshr v8.16b, v4.16b, #7 sshr v9.16b, v5.16b, #7 shl v12.16b, v4.16b, #1 shl v13.16b, v5.16b, #1 and v8.16b, v8.16b, v10.16b and v9.16b, v9.16b, v10.16b eor v8.16b, v8.16b, v12.16b eor v9.16b, v9.16b, v13.16b ushr v12.16b, v4.16b, #6 ushr v13.16b, v5.16b, #6 shl v0.16b, v4.16b, #2 shl v1.16b, v5.16b, #2 pmul v12.16b, v12.16b, v10.16b pmul v13.16b, v13.16b, v10.16b eor v12.16b, v12.16b, v0.16b eor v13.16b, v13.16b, v1.16b ushr v0.16b, v4.16b, #5 ushr v1.16b, v5.16b, #5 pmul v0.16b, v0.16b, v10.16b pmul v1.16b, v1.16b, v10.16b shl v10.16b, v4.16b, #3 shl v11.16b, v5.16b, #3 eor v0.16b, v0.16b, v10.16b eor v1.16b, v1.16b, v11.16b eor v10.16b, v8.16b, v0.16b eor v11.16b, v9.16b, v1.16b eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v5.16b eor v8.16b, v12.16b, v0.16b eor v9.16b, v13.16b, v1.16b eor v12.16b, v12.16b, v10.16b eor v13.16b, v13.16b, v11.16b eor v10.16b, v10.16b, v4.16b eor v11.16b, v11.16b, v5.16b shl v4.4s, v10.4s, #8 shl v5.4s, v11.4s, #8 rev32 v8.8h, v8.8h rev32 v9.8h, v9.8h sri v4.4s, v10.4s, #24 sri v5.4s, v11.4s, #24 eor v4.16b, v4.16b, v12.16b eor v5.16b, v5.16b, v13.16b shl v10.4s, v0.4s, #24 shl v11.4s, v1.4s, #24 eor v4.16b, v4.16b, v8.16b eor v5.16b, v5.16b, v9.16b sri v10.4s, v0.4s, #8 sri v11.4s, v1.4s, #8 eor v4.16b, v4.16b, v10.16b eor v5.16b, v5.16b, v11.16b # XOR in Key Schedule ld1 {v0.2d}, [x25], #16 eor v4.16b, v4.16b, v0.16b eor v5.16b, v5.16b, v0.16b # Round Done movi v12.16b, #0x40 movi v13.16b, #0x80 movi v14.16b, #0xc0 eor v8.16b, v4.16b, v12.16b eor v9.16b, v5.16b, v12.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b eor v10.16b, v4.16b, v13.16b eor v11.16b, v5.16b, v13.16b tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b eor v8.16b, v4.16b, v14.16b eor v9.16b, v5.16b, v14.16b orr v0.16b, v0.16b, v10.16b orr v1.16b, v1.16b, v11.16b tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b orr v0.16b, v0.16b, v8.16b orr v1.16b, v1.16b, v9.16b ld1 {v4.16b}, [x23] tbl v0.16b, {v0.16b}, v4.16b tbl v1.16b, {v1.16b}, v4.16b # XOR in Key Schedule ld1 {v4.2d}, [x25], #16 eor v0.16b, v0.16b, v4.16b eor v1.16b, v1.16b, v4.16b # Round Done rev32 v0.16b, v0.16b rev32 v1.16b, v1.16b eor v0.16b, v0.16b, v2.16b eor v1.16b, v1.16b, v3.16b st1 {v0.16b, v1.16b}, [x1], #32 and x16, x17, x11, asr 63 extr x9, x11, x10, #63 eor x8, x16, x10, lsl 1 sub w2, w2, #32 L_AES_XTS_decrypt_NEON_start_1: ld1 {v3.2d}, [x23] mov v2.d[0], x8 mov v2.d[1], x9 cmp w2, #16 blt L_AES_XTS_decrypt_NEON_start_partial mov x25, x4 ld1 {v0.16b}, [x0], #16 ld1 {v4.2d}, [x25], #16 eor v0.16b, v0.16b, v2.16b rev32 v0.16b, v0.16b eor v0.16b, v0.16b, v4.16b sub w24, w7, #2 L_AES_XTS_decrypt_NEON_loop_nr_1: eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b sshr v10.16b, v4.16b, #7 ushr v11.16b, v4.16b, #6 ushr v8.16b, v4.16b, #5 and v10.16b, v10.16b, v15.16b pmul v11.16b, v11.16b, v15.16b pmul v8.16b, v8.16b, v15.16b shl v9.16b, v4.16b, #1 eor v10.16b, v10.16b, v9.16b shl v9.16b, v4.16b, #3 eor v8.16b, v8.16b, v9.16b shl v9.16b, v4.16b, #2 eor v11.16b, v11.16b, v9.16b eor v9.16b, v10.16b, v8.16b eor v8.16b, v8.16b, v4.16b eor v10.16b, v11.16b, v8.16b eor v11.16b, v11.16b, v9.16b eor v9.16b, v9.16b, v4.16b shl v4.4s, v9.4s, #8 rev32 v10.8h, v10.8h sri v4.4s, v9.4s, #24 eor v4.16b, v4.16b, v11.16b shl v9.4s, v8.4s, #24 eor v4.16b, v4.16b, v10.16b sri v9.4s, v8.4s, #8 eor v4.16b, v4.16b, v9.16b ld1 {v0.2d}, [x25], #16 # XOR in Key Schedule eor v4.16b, v4.16b, v0.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b sshr v10.16b, v0.16b, #7 ushr v11.16b, v0.16b, #6 ushr v8.16b, v0.16b, #5 and v10.16b, v10.16b, v15.16b pmul v11.16b, v11.16b, v15.16b pmul v8.16b, v8.16b, v15.16b shl v9.16b, v0.16b, #1 eor v10.16b, v10.16b, v9.16b shl v9.16b, v0.16b, #3 eor v8.16b, v8.16b, v9.16b shl v9.16b, v0.16b, #2 eor v11.16b, v11.16b, v9.16b eor v9.16b, v10.16b, v8.16b eor v8.16b, v8.16b, v0.16b eor v10.16b, v11.16b, v8.16b eor v11.16b, v11.16b, v9.16b eor v9.16b, v9.16b, v0.16b shl v0.4s, v9.4s, #8 rev32 v10.8h, v10.8h sri v0.4s, v9.4s, #24 eor v0.16b, v0.16b, v11.16b shl v9.4s, v8.4s, #24 eor v0.16b, v0.16b, v10.16b sri v9.4s, v8.4s, #8 eor v0.16b, v0.16b, v9.16b ld1 {v4.2d}, [x25], #16 # XOR in Key Schedule eor v0.16b, v0.16b, v4.16b subs w24, w24, #2 bne L_AES_XTS_decrypt_NEON_loop_nr_1 eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b sshr v10.16b, v4.16b, #7 ushr v11.16b, v4.16b, #6 ushr v8.16b, v4.16b, #5 and v10.16b, v10.16b, v15.16b pmul v11.16b, v11.16b, v15.16b pmul v8.16b, v8.16b, v15.16b shl v9.16b, v4.16b, #1 eor v10.16b, v10.16b, v9.16b shl v9.16b, v4.16b, #3 eor v8.16b, v8.16b, v9.16b shl v9.16b, v4.16b, #2 eor v11.16b, v11.16b, v9.16b eor v9.16b, v10.16b, v8.16b eor v8.16b, v8.16b, v4.16b eor v10.16b, v11.16b, v8.16b eor v11.16b, v11.16b, v9.16b eor v9.16b, v9.16b, v4.16b shl v4.4s, v9.4s, #8 rev32 v10.8h, v10.8h sri v4.4s, v9.4s, #24 eor v4.16b, v4.16b, v11.16b shl v9.4s, v8.4s, #24 eor v4.16b, v4.16b, v10.16b sri v9.4s, v8.4s, #8 eor v4.16b, v4.16b, v9.16b ld1 {v0.2d}, [x25], #16 # XOR in Key Schedule eor v4.16b, v4.16b, v0.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b ld1 {v4.2d}, [x25], #16 # XOR in Key Schedule eor v0.16b, v0.16b, v4.16b rev32 v0.16b, v0.16b eor v0.16b, v0.16b, v2.16b st1 {v0.16b}, [x1], #16 sub w2, w2, #16 cbz w19, L_AES_XTS_decrypt_NEON_data_done and x16, x17, x9, asr 63 extr x9, x9, x8, #63 eor x8, x16, x8, lsl 1 L_AES_XTS_decrypt_NEON_start_partial: mov w2, w19 cbz w2, L_AES_XTS_decrypt_NEON_data_done mov v2.d[0], x8 mov v2.d[1], x9 and x16, x17, x9, asr 63 extr x11, x9, x8, #63 eor x10, x16, x8, lsl 1 mov v1.d[0], x10 mov v1.d[1], x11 mov x25, x4 ld1 {v0.16b}, [x0], #16 ld1 {v4.2d}, [x25], #16 eor v0.16b, v0.16b, v1.16b rev32 v0.16b, v0.16b eor v0.16b, v0.16b, v4.16b sub w24, w7, #2 L_AES_XTS_decrypt_NEON_loop_nr_partial_1: eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b sshr v10.16b, v4.16b, #7 ushr v11.16b, v4.16b, #6 ushr v8.16b, v4.16b, #5 and v10.16b, v10.16b, v15.16b pmul v11.16b, v11.16b, v15.16b pmul v8.16b, v8.16b, v15.16b shl v9.16b, v4.16b, #1 eor v10.16b, v10.16b, v9.16b shl v9.16b, v4.16b, #3 eor v8.16b, v8.16b, v9.16b shl v9.16b, v4.16b, #2 eor v11.16b, v11.16b, v9.16b eor v9.16b, v10.16b, v8.16b eor v8.16b, v8.16b, v4.16b eor v10.16b, v11.16b, v8.16b eor v11.16b, v11.16b, v9.16b eor v9.16b, v9.16b, v4.16b shl v4.4s, v9.4s, #8 rev32 v10.8h, v10.8h sri v4.4s, v9.4s, #24 eor v4.16b, v4.16b, v11.16b shl v9.4s, v8.4s, #24 eor v4.16b, v4.16b, v10.16b sri v9.4s, v8.4s, #8 eor v4.16b, v4.16b, v9.16b ld1 {v0.2d}, [x25], #16 # XOR in Key Schedule eor v4.16b, v4.16b, v0.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b sshr v10.16b, v0.16b, #7 ushr v11.16b, v0.16b, #6 ushr v8.16b, v0.16b, #5 and v10.16b, v10.16b, v15.16b pmul v11.16b, v11.16b, v15.16b pmul v8.16b, v8.16b, v15.16b shl v9.16b, v0.16b, #1 eor v10.16b, v10.16b, v9.16b shl v9.16b, v0.16b, #3 eor v8.16b, v8.16b, v9.16b shl v9.16b, v0.16b, #2 eor v11.16b, v11.16b, v9.16b eor v9.16b, v10.16b, v8.16b eor v8.16b, v8.16b, v0.16b eor v10.16b, v11.16b, v8.16b eor v11.16b, v11.16b, v9.16b eor v9.16b, v9.16b, v0.16b shl v0.4s, v9.4s, #8 rev32 v10.8h, v10.8h sri v0.4s, v9.4s, #24 eor v0.16b, v0.16b, v11.16b shl v9.4s, v8.4s, #24 eor v0.16b, v0.16b, v10.16b sri v9.4s, v8.4s, #8 eor v0.16b, v0.16b, v9.16b ld1 {v4.2d}, [x25], #16 # XOR in Key Schedule eor v0.16b, v0.16b, v4.16b subs w24, w24, #2 bne L_AES_XTS_decrypt_NEON_loop_nr_partial_1 eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b sshr v10.16b, v4.16b, #7 ushr v11.16b, v4.16b, #6 ushr v8.16b, v4.16b, #5 and v10.16b, v10.16b, v15.16b pmul v11.16b, v11.16b, v15.16b pmul v8.16b, v8.16b, v15.16b shl v9.16b, v4.16b, #1 eor v10.16b, v10.16b, v9.16b shl v9.16b, v4.16b, #3 eor v8.16b, v8.16b, v9.16b shl v9.16b, v4.16b, #2 eor v11.16b, v11.16b, v9.16b eor v9.16b, v10.16b, v8.16b eor v8.16b, v8.16b, v4.16b eor v10.16b, v11.16b, v8.16b eor v11.16b, v11.16b, v9.16b eor v9.16b, v9.16b, v4.16b shl v4.4s, v9.4s, #8 rev32 v10.8h, v10.8h sri v4.4s, v9.4s, #24 eor v4.16b, v4.16b, v11.16b shl v9.4s, v8.4s, #24 eor v4.16b, v4.16b, v10.16b sri v9.4s, v8.4s, #8 eor v4.16b, v4.16b, v9.16b ld1 {v0.2d}, [x25], #16 # XOR in Key Schedule eor v4.16b, v4.16b, v0.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b ld1 {v4.2d}, [x25], #16 # XOR in Key Schedule eor v0.16b, v0.16b, v4.16b rev32 v0.16b, v0.16b eor v0.16b, v0.16b, v1.16b st1 {v0.2d}, [x6] add x1, x1, #16 mov w16, w2 L_AES_XTS_decrypt_NEON_start_byte: ldrb w10, [x6] ldrb w11, [x0], #1 strb w10, [x1], #1 strb w11, [x6], #1 subs w16, w16, #1 bgt L_AES_XTS_decrypt_NEON_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 mov x25, x4 ld1 {v0.2d}, [x6] ld1 {v4.2d}, [x25], #16 eor v0.16b, v0.16b, v2.16b rev32 v0.16b, v0.16b eor v0.16b, v0.16b, v4.16b sub w24, w7, #2 L_AES_XTS_decrypt_NEON_loop_nr_partial_2: eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b sshr v10.16b, v4.16b, #7 ushr v11.16b, v4.16b, #6 ushr v8.16b, v4.16b, #5 and v10.16b, v10.16b, v15.16b pmul v11.16b, v11.16b, v15.16b pmul v8.16b, v8.16b, v15.16b shl v9.16b, v4.16b, #1 eor v10.16b, v10.16b, v9.16b shl v9.16b, v4.16b, #3 eor v8.16b, v8.16b, v9.16b shl v9.16b, v4.16b, #2 eor v11.16b, v11.16b, v9.16b eor v9.16b, v10.16b, v8.16b eor v8.16b, v8.16b, v4.16b eor v10.16b, v11.16b, v8.16b eor v11.16b, v11.16b, v9.16b eor v9.16b, v9.16b, v4.16b shl v4.4s, v9.4s, #8 rev32 v10.8h, v10.8h sri v4.4s, v9.4s, #24 eor v4.16b, v4.16b, v11.16b shl v9.4s, v8.4s, #24 eor v4.16b, v4.16b, v10.16b sri v9.4s, v8.4s, #8 eor v4.16b, v4.16b, v9.16b ld1 {v0.2d}, [x25], #16 # XOR in Key Schedule eor v4.16b, v4.16b, v0.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b sshr v10.16b, v0.16b, #7 ushr v11.16b, v0.16b, #6 ushr v8.16b, v0.16b, #5 and v10.16b, v10.16b, v15.16b pmul v11.16b, v11.16b, v15.16b pmul v8.16b, v8.16b, v15.16b shl v9.16b, v0.16b, #1 eor v10.16b, v10.16b, v9.16b shl v9.16b, v0.16b, #3 eor v8.16b, v8.16b, v9.16b shl v9.16b, v0.16b, #2 eor v11.16b, v11.16b, v9.16b eor v9.16b, v10.16b, v8.16b eor v8.16b, v8.16b, v0.16b eor v10.16b, v11.16b, v8.16b eor v11.16b, v11.16b, v9.16b eor v9.16b, v9.16b, v0.16b shl v0.4s, v9.4s, #8 rev32 v10.8h, v10.8h sri v0.4s, v9.4s, #24 eor v0.16b, v0.16b, v11.16b shl v9.4s, v8.4s, #24 eor v0.16b, v0.16b, v10.16b sri v9.4s, v8.4s, #8 eor v0.16b, v0.16b, v9.16b ld1 {v4.2d}, [x25], #16 # XOR in Key Schedule eor v0.16b, v0.16b, v4.16b subs w24, w24, #2 bne L_AES_XTS_decrypt_NEON_loop_nr_partial_2 eor v8.16b, v0.16b, v12.16b eor v9.16b, v0.16b, v13.16b eor v10.16b, v0.16b, v14.16b tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v4.16b, v4.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v4.16b, v4.16b, v9.16b tbl v4.16b, {v4.16b}, v3.16b sshr v10.16b, v4.16b, #7 ushr v11.16b, v4.16b, #6 ushr v8.16b, v4.16b, #5 and v10.16b, v10.16b, v15.16b pmul v11.16b, v11.16b, v15.16b pmul v8.16b, v8.16b, v15.16b shl v9.16b, v4.16b, #1 eor v10.16b, v10.16b, v9.16b shl v9.16b, v4.16b, #3 eor v8.16b, v8.16b, v9.16b shl v9.16b, v4.16b, #2 eor v11.16b, v11.16b, v9.16b eor v9.16b, v10.16b, v8.16b eor v8.16b, v8.16b, v4.16b eor v10.16b, v11.16b, v8.16b eor v11.16b, v11.16b, v9.16b eor v9.16b, v9.16b, v4.16b shl v4.4s, v9.4s, #8 rev32 v10.8h, v10.8h sri v4.4s, v9.4s, #24 eor v4.16b, v4.16b, v11.16b shl v9.4s, v8.4s, #24 eor v4.16b, v4.16b, v10.16b sri v9.4s, v8.4s, #8 eor v4.16b, v4.16b, v9.16b ld1 {v0.2d}, [x25], #16 # XOR in Key Schedule eor v4.16b, v4.16b, v0.16b eor v8.16b, v4.16b, v12.16b eor v9.16b, v4.16b, v13.16b eor v10.16b, v4.16b, v14.16b tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b orr v0.16b, v0.16b, v8.16b orr v9.16b, v9.16b, v10.16b orr v0.16b, v0.16b, v9.16b tbl v0.16b, {v0.16b}, v3.16b ld1 {v4.2d}, [x25], #16 # XOR in Key Schedule eor v0.16b, v0.16b, v4.16b rev32 v0.16b, v0.16b eor v0.16b, v0.16b, v2.16b st1 {v0.16b}, [x1] L_AES_XTS_decrypt_NEON_data_done: ldp x17, x19, [x29, #16] ldp x20, x21, [x29, #32] ldp x22, x23, [x29, #48] ldp x24, x25, [x29, #64] ldp d8, d9, [x29, #80] ldp d10, d11, [x29, #96] ldp d12, d13, [x29, #112] ldp d14, d15, [x29, #128] ldp x29, x30, [sp], #0x90 ret #ifndef __APPLE__ .size AES_XTS_decrypt_NEON,.-AES_XTS_decrypt_NEON #endif /* __APPLE__ */ #endif /* HAVE_AES_DECRYPT */ #endif /* WOLFSSL_AES_XTS */ #endif /* !WOLFSSL_ARMASM_NO_NEON */ #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP #ifdef HAVE_AES_DECRYPT #ifndef __APPLE__ .text .section .rodata .type L_AES_ARM64_td, %object .size L_AES_ARM64_td, 1024 #else .section __DATA,__data #endif /* __APPLE__ */ # 8-byte aligned, 64-bit aligned #ifndef __APPLE__ .align 3 #else .p2align 3 #endif /* __APPLE__ */ L_AES_ARM64_td: .long 0x5051f4a7,0x537e4165,0xc31a17a4,0x963a275e .long 0xcb3bab6b,0xf11f9d45,0xabacfa58,0x934be303 .long 0x552030fa,0xf6ad766d,0x9188cc76,0x25f5024c .long 0xfc4fe5d7,0xd7c52acb,0x80263544,0x8fb562a3 .long 0x49deb15a,0x6725ba1b,0x9845ea0e,0xe15dfec0 .long 0x02c32f75,0x12814cf0,0xa38d4697,0xc66bd3f9 .long 0xe7038f5f,0x9515929c,0xebbf6d7a,0xda955259 .long 0x2dd4be83,0xd3587421,0x2949e069,0x448ec9c8 .long 0x6a75c289,0x78f48e79,0x6b99583e,0xdd27b971 .long 0xb6bee14f,0x17f088ad,0x66c920ac,0xb47dce3a .long 0x1863df4a,0x82e51a31,0x60975133,0x4562537f .long 0xe0b16477,0x84bb6bae,0x1cfe81a0,0x94f9082b .long 0x58704868,0x198f45fd,0x8794de6c,0xb7527bf8 .long 0x23ab73d3,0xe2724b02,0x57e31f8f,0x2a6655ab .long 0x07b2eb28,0x032fb5c2,0x9a86c57b,0xa5d33708 .long 0xf2302887,0xb223bfa5,0xba02036a,0x5ced1682 .long 0x2b8acf1c,0x92a779b4,0xf0f307f2,0xa14e69e2 .long 0xcd65daf4,0xd50605be,0x1fd13462,0x8ac4a6fe .long 0x9d342e53,0xa0a2f355,0x32058ae1,0x75a4f6eb .long 0x390b83ec,0xaa4060ef,0x065e719f,0x51bd6e10 .long 0xf93e218a,0x3d96dd06,0xaedd3e05,0x464de6bd .long 0xb591548d,0x0571c45d,0x6f0406d4,0xff605015 .long 0x241998fb,0x97d6bde9,0xcc894043,0x7767d99e .long 0xbdb0e842,0x8807898b,0x38e7195b,0xdb79c8ee .long 0x47a17c0a,0xe97c420f,0xc9f8841e,0x00000000 .long 0x83098086,0x48322bed,0xac1e1170,0x4e6c5a72 .long 0xfbfd0eff,0x560f8538,0x1e3daed5,0x27362d39 .long 0x640a0fd9,0x21685ca6,0xd19b5b54,0x3a24362e .long 0xb10c0a67,0x0f9357e7,0xd2b4ee96,0x9e1b9b91 .long 0x4f80c0c5,0xa261dc20,0x695a774b,0x161c121a .long 0x0ae293ba,0xe5c0a02a,0x433c22e0,0x1d121b17 .long 0x0b0e090d,0xadf28bc7,0xb92db6a8,0xc8141ea9 .long 0x8557f119,0x4caf7507,0xbbee99dd,0xfda37f60 .long 0x9ff70126,0xbc5c72f5,0xc544663b,0x345bfb7e .long 0x768b4329,0xdccb23c6,0x68b6edfc,0x63b8e4f1 .long 0xcad731dc,0x10426385,0x40139722,0x2084c611 .long 0x7d854a24,0xf8d2bb3d,0x11aef932,0x6dc729a1 .long 0x4b1d9e2f,0xf3dcb230,0xec0d8652,0xd077c1e3 .long 0x6c2bb316,0x99a970b9,0xfa119448,0x2247e964 .long 0xc4a8fc8c,0x1aa0f03f,0xd8567d2c,0xef223390 .long 0xc787494e,0xc1d938d1,0xfe8ccaa2,0x3698d40b .long 0xcfa6f581,0x28a57ade,0x26dab78e,0xa43fadbf .long 0xe42c3a9d,0x0d507892,0x9b6a5fcc,0x62547e46 .long 0xc2f68d13,0xe890d8b8,0x5e2e39f7,0xf582c3af .long 0xbe9f5d80,0x7c69d093,0xa96fd52d,0xb3cf2512 .long 0x3bc8ac99,0xa710187d,0x6ee89c63,0x7bdb3bbb .long 0x09cd2678,0xf46e5918,0x01ec9ab7,0xa8834f9a .long 0x65e6956e,0x7eaaffe6,0x0821bccf,0xe6ef15e8 .long 0xd9bae79b,0xce4a6f36,0xd4ea9f09,0xd629b07c .long 0xaf31a4b2,0x312a3f23,0x30c6a594,0xc035a266 .long 0x37744ebc,0xa6fc82ca,0xb0e090d0,0x1533a7d8 .long 0x4af10498,0xf741ecda,0x0e7fcd50,0x2f1791f6 .long 0x8d764dd6,0x4d43efb0,0x54ccaa4d,0xdfe49604 .long 0xe39ed1b5,0x1b4c6a88,0xb8c12c1f,0x7f466551 .long 0x049d5eea,0x5d018c35,0x73fa8774,0x2efb0b41 .long 0x5ab3671d,0x5292dbd2,0x33e91056,0x136dd647 .long 0x8c9ad761,0x7a37a10c,0x8e59f814,0x89eb133c .long 0xeecea927,0x35b761c9,0xede11ce5,0x3c7a47b1 .long 0x599cd2df,0x3f55f273,0x791814ce,0xbf73c737 .long 0xea53f7cd,0x5b5ffdaa,0x14df3d6f,0x867844db .long 0x81caaff3,0x3eb968c4,0x2c382434,0x5fc2a340 .long 0x72161dc3,0x0cbce225,0x8b283c49,0x41ff0d95 .long 0x7139a801,0xde080cb3,0x9cd8b4e4,0x906456c1 .long 0x617bcb84,0x70d532b6,0x74486c5c,0x42d0b857 #endif /* HAVE_AES_DECRYPT */ #if defined(HAVE_AES_DECRYPT) || defined(HAVE_AES_CBC) || \ defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) #ifndef __APPLE__ .text .section .rodata .type L_AES_ARM64_te, %object .size L_AES_ARM64_te, 1024 #else .section __DATA,__data #endif /* __APPLE__ */ # 8-byte aligned, 64-bit aligned #ifndef __APPLE__ .align 3 #else .p2align 3 #endif /* __APPLE__ */ L_AES_ARM64_te: .long 0xa5c66363,0x84f87c7c,0x99ee7777,0x8df67b7b .long 0x0dfff2f2,0xbdd66b6b,0xb1de6f6f,0x5491c5c5 .long 0x50603030,0x03020101,0xa9ce6767,0x7d562b2b .long 0x19e7fefe,0x62b5d7d7,0xe64dabab,0x9aec7676 .long 0x458fcaca,0x9d1f8282,0x4089c9c9,0x87fa7d7d .long 0x15effafa,0xebb25959,0xc98e4747,0x0bfbf0f0 .long 0xec41adad,0x67b3d4d4,0xfd5fa2a2,0xea45afaf .long 0xbf239c9c,0xf753a4a4,0x96e47272,0x5b9bc0c0 .long 0xc275b7b7,0x1ce1fdfd,0xae3d9393,0x6a4c2626 .long 0x5a6c3636,0x417e3f3f,0x02f5f7f7,0x4f83cccc .long 0x5c683434,0xf451a5a5,0x34d1e5e5,0x08f9f1f1 .long 0x93e27171,0x73abd8d8,0x53623131,0x3f2a1515 .long 0x0c080404,0x5295c7c7,0x65462323,0x5e9dc3c3 .long 0x28301818,0xa1379696,0x0f0a0505,0xb52f9a9a .long 0x090e0707,0x36241212,0x9b1b8080,0x3ddfe2e2 .long 0x26cdebeb,0x694e2727,0xcd7fb2b2,0x9fea7575 .long 0x1b120909,0x9e1d8383,0x74582c2c,0x2e341a1a .long 0x2d361b1b,0xb2dc6e6e,0xeeb45a5a,0xfb5ba0a0 .long 0xf6a45252,0x4d763b3b,0x61b7d6d6,0xce7db3b3 .long 0x7b522929,0x3edde3e3,0x715e2f2f,0x97138484 .long 0xf5a65353,0x68b9d1d1,0x00000000,0x2cc1eded .long 0x60402020,0x1fe3fcfc,0xc879b1b1,0xedb65b5b .long 0xbed46a6a,0x468dcbcb,0xd967bebe,0x4b723939 .long 0xde944a4a,0xd4984c4c,0xe8b05858,0x4a85cfcf .long 0x6bbbd0d0,0x2ac5efef,0xe54faaaa,0x16edfbfb .long 0xc5864343,0xd79a4d4d,0x55663333,0x94118585 .long 0xcf8a4545,0x10e9f9f9,0x06040202,0x81fe7f7f .long 0xf0a05050,0x44783c3c,0xba259f9f,0xe34ba8a8 .long 0xf3a25151,0xfe5da3a3,0xc0804040,0x8a058f8f .long 0xad3f9292,0xbc219d9d,0x48703838,0x04f1f5f5 .long 0xdf63bcbc,0xc177b6b6,0x75afdada,0x63422121 .long 0x30201010,0x1ae5ffff,0x0efdf3f3,0x6dbfd2d2 .long 0x4c81cdcd,0x14180c0c,0x35261313,0x2fc3ecec .long 0xe1be5f5f,0xa2359797,0xcc884444,0x392e1717 .long 0x5793c4c4,0xf255a7a7,0x82fc7e7e,0x477a3d3d .long 0xacc86464,0xe7ba5d5d,0x2b321919,0x95e67373 .long 0xa0c06060,0x98198181,0xd19e4f4f,0x7fa3dcdc .long 0x66442222,0x7e542a2a,0xab3b9090,0x830b8888 .long 0xca8c4646,0x29c7eeee,0xd36bb8b8,0x3c281414 .long 0x79a7dede,0xe2bc5e5e,0x1d160b0b,0x76addbdb .long 0x3bdbe0e0,0x56643232,0x4e743a3a,0x1e140a0a .long 0xdb924949,0x0a0c0606,0x6c482424,0xe4b85c5c .long 0x5d9fc2c2,0x6ebdd3d3,0xef43acac,0xa6c46262 .long 0xa8399191,0xa4319595,0x37d3e4e4,0x8bf27979 .long 0x32d5e7e7,0x438bc8c8,0x596e3737,0xb7da6d6d .long 0x8c018d8d,0x64b1d5d5,0xd29c4e4e,0xe049a9a9 .long 0xb4d86c6c,0xfaac5656,0x07f3f4f4,0x25cfeaea .long 0xafca6565,0x8ef47a7a,0xe947aeae,0x18100808 .long 0xd56fbaba,0x88f07878,0x6f4a2525,0x725c2e2e .long 0x24381c1c,0xf157a6a6,0xc773b4b4,0x5197c6c6 .long 0x23cbe8e8,0x7ca1dddd,0x9ce87474,0x213e1f1f .long 0xdd964b4b,0xdc61bdbd,0x860d8b8b,0x850f8a8a .long 0x90e07070,0x427c3e3e,0xc471b5b5,0xaacc6666 .long 0xd8904848,0x05060303,0x01f7f6f6,0x121c0e0e .long 0xa3c26161,0x5f6a3535,0xf9ae5757,0xd069b9b9 .long 0x91178686,0x5899c1c1,0x273a1d1d,0xb9279e9e .long 0x38d9e1e1,0x13ebf8f8,0xb32b9898,0x33221111 .long 0xbbd26969,0x70a9d9d9,0x89078e8e,0xa7339494 .long 0xb62d9b9b,0x223c1e1e,0x92158787,0x20c9e9e9 .long 0x4987cece,0xffaa5555,0x78502828,0x7aa5dfdf .long 0x8f038c8c,0xf859a1a1,0x80098989,0x171a0d0d .long 0xda65bfbf,0x31d7e6e6,0xc6844242,0xb8d06868 .long 0xc3824141,0xb0299999,0x775a2d2d,0x111e0f0f .long 0xcb7bb0b0,0xfca85454,0xd66dbbbb,0x3a2c1616 #endif /* HAVE_AES_DECRYPT || HAVE_AES_CBC || HAVE_AESCCM || HAVE_AESGCM || * WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER */ #ifdef HAVE_AES_DECRYPT #ifndef __APPLE__ .text .globl AES_invert_key .type AES_invert_key,@function .align 2 AES_invert_key: #else .section __TEXT,__text .globl _AES_invert_key .p2align 2 _AES_invert_key: #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x2, L_AES_ARM64_te add x2, x2, :lo12:L_AES_ARM64_te #else adrp x2, L_AES_ARM64_te@PAGE add x2, x2, L_AES_ARM64_te@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x3, L_AES_ARM64_td add x3, x3, :lo12:L_AES_ARM64_td #else adrp x3, L_AES_ARM64_td@PAGE add x3, x3, L_AES_ARM64_td@PAGEOFF #endif /* __APPLE__ */ add x12, x0, x1, lsl 4 mov w13, w1 L_AES_invert_key_loop: ldp w4, w5, [x0] ldnp w6, w7, [x0, #8] ldp w8, w9, [x12] ldnp w10, w11, [x12, #8] stp w4, w5, [x12] stnp w6, w7, [x12, #8] stp w8, w9, [x0], #8 stp w10, w11, [x0], #8 subs w13, w13, #2 sub x12, x12, #16 bne L_AES_invert_key_loop sub x0, x0, x1, lsl 3 add x0, x0, #16 sub w13, w1, #1 L_AES_invert_key_mix_loop: ldp w4, w5, [x0] ldnp w6, w7, [x0, #8] ubfx w8, w4, #0, #8 ubfx w9, w4, #8, #8 ubfx w10, w4, #16, #8 ubfx w11, w4, #24, #8 lsl w8, w8, #2 lsl w9, w9, #2 lsl w10, w10, #2 lsl w11, w11, #2 ldrb w8, [x2, x8, LSL 0] ldrb w9, [x2, x9, LSL 0] ldrb w10, [x2, x10, LSL 0] ldrb w11, [x2, x11, LSL 0] ldr w8, [x3, x8, LSL 2] ldr w9, [x3, x9, LSL 2] ldr w10, [x3, x10, LSL 2] ldr w11, [x3, x11, LSL 2] eor w10, w10, w8, ror 16 eor w10, w10, w9, ror 8 eor w10, w10, w11, ror 24 str w10, [x0], #4 ubfx w8, w5, #0, #8 ubfx w9, w5, #8, #8 ubfx w10, w5, #16, #8 ubfx w11, w5, #24, #8 lsl w8, w8, #2 lsl w9, w9, #2 lsl w10, w10, #2 lsl w11, w11, #2 ldrb w8, [x2, x8, LSL 0] ldrb w9, [x2, x9, LSL 0] ldrb w10, [x2, x10, LSL 0] ldrb w11, [x2, x11, LSL 0] ldr w8, [x3, x8, LSL 2] ldr w9, [x3, x9, LSL 2] ldr w10, [x3, x10, LSL 2] ldr w11, [x3, x11, LSL 2] eor w10, w10, w8, ror 16 eor w10, w10, w9, ror 8 eor w10, w10, w11, ror 24 str w10, [x0], #4 ubfx w8, w6, #0, #8 ubfx w9, w6, #8, #8 ubfx w10, w6, #16, #8 ubfx w11, w6, #24, #8 lsl w8, w8, #2 lsl w9, w9, #2 lsl w10, w10, #2 lsl w11, w11, #2 ldrb w8, [x2, x8, LSL 0] ldrb w9, [x2, x9, LSL 0] ldrb w10, [x2, x10, LSL 0] ldrb w11, [x2, x11, LSL 0] ldr w8, [x3, x8, LSL 2] ldr w9, [x3, x9, LSL 2] ldr w10, [x3, x10, LSL 2] ldr w11, [x3, x11, LSL 2] eor w10, w10, w8, ror 16 eor w10, w10, w9, ror 8 eor w10, w10, w11, ror 24 str w10, [x0], #4 ubfx w8, w7, #0, #8 ubfx w9, w7, #8, #8 ubfx w10, w7, #16, #8 ubfx w11, w7, #24, #8 lsl w8, w8, #2 lsl w9, w9, #2 lsl w10, w10, #2 lsl w11, w11, #2 ldrb w8, [x2, x8, LSL 0] ldrb w9, [x2, x9, LSL 0] ldrb w10, [x2, x10, LSL 0] ldrb w11, [x2, x11, LSL 0] ldr w8, [x3, x8, LSL 2] ldr w9, [x3, x9, LSL 2] ldr w10, [x3, x10, LSL 2] ldr w11, [x3, x11, LSL 2] eor w10, w10, w8, ror 16 eor w10, w10, w9, ror 8 eor w10, w10, w11, ror 24 str w10, [x0], #4 subs w13, w13, #1 bne L_AES_invert_key_mix_loop ret #ifndef __APPLE__ .size AES_invert_key,.-AES_invert_key #endif /* __APPLE__ */ #endif /* HAVE_AES_DECRYPT */ #ifndef __APPLE__ .text .section .rodata .type L_AES_ARM64_rcon, %object .size L_AES_ARM64_rcon, 40 #else .section __DATA,__data #endif /* __APPLE__ */ # 8-byte aligned, 64-bit aligned #ifndef __APPLE__ .align 3 #else .p2align 3 #endif /* __APPLE__ */ L_AES_ARM64_rcon: .long 0x01000000,0x02000000,0x04000000,0x08000000 .long 0x10000000,0x20000000,0x40000000,0x80000000 .long 0x1b000000,0x36000000 #ifndef __APPLE__ .text .globl AES_set_encrypt_key .type AES_set_encrypt_key,@function .align 2 AES_set_encrypt_key: #else .section __TEXT,__text .globl _AES_set_encrypt_key .p2align 2 _AES_set_encrypt_key: #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x5, L_AES_ARM64_rcon add x5, x5, :lo12:L_AES_ARM64_rcon #else adrp x5, L_AES_ARM64_rcon@PAGE add x5, x5, L_AES_ARM64_rcon@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x12, L_AES_ARM64_te add x12, x12, :lo12:L_AES_ARM64_te #else adrp x12, L_AES_ARM64_te@PAGE add x12, x12, L_AES_ARM64_te@PAGEOFF #endif /* __APPLE__ */ cmp x1, #0x80 beq L_AES_set_encrypt_key_start_128 cmp x1, #0xc0 beq L_AES_set_encrypt_key_start_192 ldr w6, [x0] ldr w7, [x0, #4] ldr w8, [x0, #8] ldr w9, [x0, #12] rev w6, w6 rev w7, w7 rev w8, w8 rev w9, w9 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 ldr w6, [x0, #16] ldr w7, [x0, #20] ldr w8, [x0, #24] ldr w9, [x0, #28] rev w6, w6 rev w7, w7 rev w8, w8 rev w9, w9 stp w6, w7, [x2] stnp w8, w9, [x2, #8] sub x2, x2, #16 mov x4, #6 L_AES_set_encrypt_key_loop_256: ubfx w6, w9, #0, #8 ubfx w7, w9, #8, #8 ubfx w8, w9, #16, #8 ubfx w9, w9, #24, #8 lsl w6, w6, #2 lsl w7, w7, #2 lsl w8, w8, #2 lsl w9, w9, #2 ldrb w6, [x12, x6, LSL 0] ldrb w7, [x12, x7, LSL 0] ldrb w8, [x12, x8, LSL 0] ldrb w9, [x12, x9, LSL 0] eor w3, w9, w6, lsl 8 eor w3, w3, w7, lsl 16 eor w3, w3, w8, lsl 24 ldp w6, w7, [x2], #8 ldp w8, w9, [x2], #8 eor w6, w6, w3 ldr w3, [x5], #4 eor w6, w6, w3 eor w7, w7, w6 eor w8, w8, w7 eor w9, w9, w8 add x2, x2, #16 stp w6, w7, [x2] stnp w8, w9, [x2, #8] sub x2, x2, #16 mov w3, w9 ubfx w6, w3, #8, #8 ubfx w7, w3, #16, #8 ubfx w8, w3, #24, #8 ubfx w3, w3, #0, #8 lsl w6, w6, #2 lsl w7, w7, #2 lsl w8, w8, #2 lsl w3, w3, #2 ldrb w6, [x12, x6, LSL 0] ldrb w8, [x12, x8, LSL 0] ldrb w7, [x12, x7, LSL 0] ldrb w3, [x12, x3, LSL 0] eor w3, w3, w6, lsl 8 eor w3, w3, w7, lsl 16 eor w3, w3, w8, lsl 24 ldp w6, w7, [x2], #8 ldp w8, w9, [x2], #8 eor w6, w6, w3 eor w7, w7, w6 eor w8, w8, w7 eor w9, w9, w8 add x2, x2, #16 stp w6, w7, [x2] stnp w8, w9, [x2, #8] sub x2, x2, #16 subs x4, x4, #1 bne L_AES_set_encrypt_key_loop_256 ubfx w6, w9, #0, #8 ubfx w7, w9, #8, #8 ubfx w8, w9, #16, #8 ubfx w9, w9, #24, #8 lsl w6, w6, #2 lsl w7, w7, #2 lsl w8, w8, #2 lsl w9, w9, #2 ldrb w6, [x12, x6, LSL 0] ldrb w7, [x12, x7, LSL 0] ldrb w8, [x12, x8, LSL 0] ldrb w9, [x12, x9, LSL 0] eor w3, w9, w6, lsl 8 eor w3, w3, w7, lsl 16 eor w3, w3, w8, lsl 24 ldp w6, w7, [x2], #8 ldp w8, w9, [x2], #8 eor w6, w6, w3 ldr w3, [x5], #4 eor w6, w6, w3 eor w7, w7, w6 eor w8, w8, w7 eor w9, w9, w8 add x2, x2, #16 stp w6, w7, [x2] stnp w8, w9, [x2, #8] sub x2, x2, #16 b L_AES_set_encrypt_key_end L_AES_set_encrypt_key_start_192: ldr w6, [x0] ldr w7, [x0, #4] ldr w8, [x0, #8] ldr w9, [x0, #12] ldr w10, [x0, #16] ldr w11, [x0, #20] rev w6, w6 rev w7, w7 rev w8, w8 rev w9, w9 rev w10, w10 rev w11, w11 stp w6, w7, [x2] stnp w8, w9, [x2, #8] stnp w10, w11, [x2, #16] mov x4, #7 L_AES_set_encrypt_key_loop_192: ubfx w6, w11, #0, #8 ubfx w7, w11, #8, #8 ubfx w8, w11, #16, #8 ubfx w11, w11, #24, #8 lsl w6, w6, #2 lsl w7, w7, #2 lsl w8, w8, #2 lsl w11, w11, #2 ldrb w6, [x12, x6, LSL 0] ldrb w7, [x12, x7, LSL 0] ldrb w8, [x12, x8, LSL 0] ldrb w11, [x12, x11, LSL 0] eor w3, w11, w6, lsl 8 eor w3, w3, w7, lsl 16 eor w3, w3, w8, lsl 24 ldp w6, w7, [x2], #8 ldp w8, w9, [x2], #8 ldp w10, w11, [x2], #8 eor w6, w6, w3 ldr w3, [x5], #4 eor w6, w6, w3 eor w7, w7, w6 eor w8, w8, w7 eor w9, w9, w8 eor w10, w10, w9 eor w11, w11, w10 stp w6, w7, [x2] stnp w8, w9, [x2, #8] stnp w10, w11, [x2, #16] subs x4, x4, #1 bne L_AES_set_encrypt_key_loop_192 ubfx w6, w11, #0, #8 ubfx w7, w11, #8, #8 ubfx w8, w11, #16, #8 ubfx w11, w11, #24, #8 lsl w6, w6, #2 lsl w7, w7, #2 lsl w8, w8, #2 lsl w11, w11, #2 ldrb w6, [x12, x6, LSL 0] ldrb w7, [x12, x7, LSL 0] ldrb w8, [x12, x8, LSL 0] ldrb w11, [x12, x11, LSL 0] eor w3, w11, w6, lsl 8 eor w3, w3, w7, lsl 16 eor w3, w3, w8, lsl 24 ldp w6, w7, [x2], #8 ldp w8, w9, [x2], #8 ldp w10, w11, [x2], #8 eor w6, w6, w3 ldr w3, [x5], #4 eor w6, w6, w3 eor w7, w7, w6 eor w8, w8, w7 eor w9, w9, w8 stp w6, w7, [x2] stnp w8, w9, [x2, #8] b L_AES_set_encrypt_key_end L_AES_set_encrypt_key_start_128: ldr w6, [x0] ldr w7, [x0, #4] ldr w8, [x0, #8] ldr w9, [x0, #12] rev w6, w6 rev w7, w7 rev w8, w8 rev w9, w9 stp w6, w7, [x2] stnp w8, w9, [x2, #8] mov x4, #10 L_AES_set_encrypt_key_loop_128: ubfx w6, w9, #0, #8 ubfx w7, w9, #8, #8 ubfx w8, w9, #16, #8 ubfx w9, w9, #24, #8 lsl w6, w6, #2 lsl w7, w7, #2 lsl w8, w8, #2 lsl w9, w9, #2 ldrb w6, [x12, x6, LSL 0] ldrb w7, [x12, x7, LSL 0] ldrb w8, [x12, x8, LSL 0] ldrb w9, [x12, x9, LSL 0] eor w3, w9, w6, lsl 8 eor w3, w3, w7, lsl 16 eor w3, w3, w8, lsl 24 ldp w6, w7, [x2], #8 ldp w8, w9, [x2], #8 eor w6, w6, w3 ldr w3, [x5], #4 eor w6, w6, w3 eor w7, w7, w6 eor w8, w8, w7 eor w9, w9, w8 stp w6, w7, [x2] stnp w8, w9, [x2, #8] subs x4, x4, #1 bne L_AES_set_encrypt_key_loop_128 L_AES_set_encrypt_key_end: ret #ifndef __APPLE__ .size AES_set_encrypt_key,.-AES_set_encrypt_key #endif /* __APPLE__ */ #if defined(HAVE_AESCCM) || defined(HAVE_AESGCM) || \ defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || \ defined(HAVE_AES_ECB) #ifndef __APPLE__ .text .globl AES_ECB_encrypt .type AES_ECB_encrypt,@function .align 2 AES_ECB_encrypt: #else .section __TEXT,__text .globl _AES_ECB_encrypt .p2align 2 _AES_ECB_encrypt: #endif /* __APPLE__ */ stp x29, x30, [sp, #-32]! add x29, sp, #0 str x17, [x29, #24] #ifndef __APPLE__ adrp x5, L_AES_ARM64_te add x5, x5, :lo12:L_AES_ARM64_te #else adrp x5, L_AES_ARM64_te@PAGE add x5, x5, L_AES_ARM64_te@PAGEOFF #endif /* __APPLE__ */ L_AES_ECB_encrypt_loop_block_128: mov x17, x3 ldr x6, [x0] ldr x7, [x0, #8] rev32 x6, x6 rev32 x7, x7 ldp x10, x11, [x17], #16 # Round: 0 - XOR in key schedule eor x6, x6, x10 eor x7, x7, x11 sub w16, w4, #2 L_AES_ECB_encrypt_loop_nr: ubfx x10, x6, #48, #8 ubfx x13, x6, #24, #8 ubfx x14, x7, #8, #8 ubfx x15, x7, #32, #8 ldr x8, [x5] ldr x8, [x5, #64] ldr x8, [x5, #128] ldr x8, [x5, #192] ldr x8, [x5, #256] ldr x8, [x5, #320] ldr x8, [x5, #384] ldr x8, [x5, #448] ldr x8, [x5, #512] ldr x8, [x5, #576] ldr x8, [x5, #640] ldr x8, [x5, #704] ldr x8, [x5, #768] ldr x8, [x5, #832] ldr x8, [x5, #896] ldr x8, [x5, #960] ldr w10, [x5, x10, LSL 2] ldr w13, [x5, x13, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ubfx x11, x7, #16, #8 eor w10, w10, w13, ror 24 ubfx x13, x6, #56, #8 eor w10, w10, w14, ror 8 ubfx x14, x7, #40, #8 eor w10, w10, w15, ror 16 ubfx x15, x6, #0, #8 ldr w11, [x5, x11, LSL 2] ldr w13, [x5, x13, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ubfx x12, x7, #48, #8 eor w11, w11, w13, ror 24 ubfx x13, x7, #24, #8 eor w11, w11, w14, ror 8 ubfx x14, x6, #8, #8 eor w11, w11, w15, ror 16 ubfx x15, x6, #32, #8 bfi x10, x11, #32, #32 ldr w12, [x5, x12, LSL 2] ldr w13, [x5, x13, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ubfx x8, x7, #0, #8 eor w12, w12, w13, ror 24 ubfx x13, x6, #16, #8 eor w12, w12, w14, ror 8 ubfx x14, x7, #56, #8 eor w11, w12, w15, ror 16 ubfx x15, x6, #40, #8 ldr w8, [x5, x8, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w13, [x5, x13, LSL 2] ldr w15, [x5, x15, LSL 2] eor w14, w14, w8, ror 24 ldp x6, x7, [x17], #16 eor w13, w13, w14, ror 24 eor w13, w13, w15, ror 8 bfi x11, x13, #32, #32 # XOR in Key Schedule eor x10, x10, x6 eor x11, x11, x7 ubfx x6, x10, #48, #8 ubfx x9, x10, #24, #8 ubfx x14, x11, #8, #8 ubfx x15, x11, #32, #8 ldr x12, [x5] ldr x12, [x5, #64] ldr x12, [x5, #128] ldr x12, [x5, #192] ldr x12, [x5, #256] ldr x12, [x5, #320] ldr x12, [x5, #384] ldr x12, [x5, #448] ldr x12, [x5, #512] ldr x12, [x5, #576] ldr x12, [x5, #640] ldr x12, [x5, #704] ldr x12, [x5, #768] ldr x12, [x5, #832] ldr x12, [x5, #896] ldr x12, [x5, #960] ldr w6, [x5, x6, LSL 2] ldr w9, [x5, x9, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ubfx x7, x11, #16, #8 eor w6, w6, w9, ror 24 ubfx x9, x10, #56, #8 eor w6, w6, w14, ror 8 ubfx x14, x11, #40, #8 eor w6, w6, w15, ror 16 ubfx x15, x10, #0, #8 ldr w7, [x5, x7, LSL 2] ldr w9, [x5, x9, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ubfx x8, x11, #48, #8 eor w7, w7, w9, ror 24 ubfx x9, x11, #24, #8 eor w7, w7, w14, ror 8 ubfx x14, x10, #8, #8 eor w7, w7, w15, ror 16 ubfx x15, x10, #32, #8 bfi x6, x7, #32, #32 ldr w8, [x5, x8, LSL 2] ldr w9, [x5, x9, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ubfx x12, x11, #0, #8 eor w8, w8, w9, ror 24 ubfx x9, x10, #16, #8 eor w8, w8, w14, ror 8 ubfx x14, x11, #56, #8 eor w7, w8, w15, ror 16 ubfx x15, x10, #40, #8 ldr w12, [x5, x12, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w9, [x5, x9, LSL 2] ldr w15, [x5, x15, LSL 2] eor w14, w14, w12, ror 24 ldp x10, x11, [x17], #16 eor w9, w9, w14, ror 24 eor w9, w9, w15, ror 8 bfi x7, x9, #32, #32 # XOR in Key Schedule eor x6, x6, x10 eor x7, x7, x11 subs w16, w16, #2 bne L_AES_ECB_encrypt_loop_nr ubfx x10, x6, #48, #8 ubfx x13, x6, #24, #8 ubfx x14, x7, #8, #8 ubfx x15, x7, #32, #8 ldr x8, [x5] ldr x8, [x5, #64] ldr x8, [x5, #128] ldr x8, [x5, #192] ldr x8, [x5, #256] ldr x8, [x5, #320] ldr x8, [x5, #384] ldr x8, [x5, #448] ldr x8, [x5, #512] ldr x8, [x5, #576] ldr x8, [x5, #640] ldr x8, [x5, #704] ldr x8, [x5, #768] ldr x8, [x5, #832] ldr x8, [x5, #896] ldr x8, [x5, #960] ldr w10, [x5, x10, LSL 2] ldr w13, [x5, x13, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ubfx x11, x7, #16, #8 eor w10, w10, w13, ror 24 ubfx x13, x6, #56, #8 eor w10, w10, w14, ror 8 ubfx x14, x7, #40, #8 eor w10, w10, w15, ror 16 ubfx x15, x6, #0, #8 ldr w11, [x5, x11, LSL 2] ldr w13, [x5, x13, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ubfx x12, x7, #48, #8 eor w11, w11, w13, ror 24 ubfx x13, x7, #24, #8 eor w11, w11, w14, ror 8 ubfx x14, x6, #8, #8 eor w11, w11, w15, ror 16 ubfx x15, x6, #32, #8 bfi x10, x11, #32, #32 ldr w12, [x5, x12, LSL 2] ldr w13, [x5, x13, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ubfx x8, x7, #0, #8 eor w12, w12, w13, ror 24 ubfx x13, x6, #16, #8 eor w12, w12, w14, ror 8 ubfx x14, x7, #56, #8 eor w11, w12, w15, ror 16 ubfx x15, x6, #40, #8 ldr w8, [x5, x8, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w13, [x5, x13, LSL 2] ldr w15, [x5, x15, LSL 2] eor w14, w14, w8, ror 24 ldp x6, x7, [x17], #16 eor w13, w13, w14, ror 24 eor w13, w13, w15, ror 8 bfi x11, x13, #32, #32 # XOR in Key Schedule eor x10, x10, x6 eor x11, x11, x7 ubfx x6, x11, #32, #8 ubfx x9, x11, #8, #8 ubfx x14, x10, #48, #8 ubfx x15, x10, #24, #8 lsl w6, w6, #2 lsl w9, w9, #2 lsl w14, w14, #2 lsl w15, w15, #2 ldr x13, [x5] ldr x13, [x5, #64] ldr x13, [x5, #128] ldr x13, [x5, #192] ldr x13, [x5, #256] ldr x13, [x5, #320] ldr x13, [x5, #384] ldr x13, [x5, #448] ldr x13, [x5, #512] ldr x13, [x5, #576] ldr x13, [x5, #640] ldr x13, [x5, #704] ldr x13, [x5, #768] ldr x13, [x5, #832] ldr x13, [x5, #896] ldr x13, [x5, #960] ldrb w6, [x5, x6, LSL 0] ldrb w9, [x5, x9, LSL 0] ldrb w14, [x5, x14, LSL 0] ldrb w15, [x5, x15, LSL 0] ubfx x7, x10, #0, #8 eor w6, w6, w9, lsl 8 ubfx x9, x11, #40, #8 eor w6, w6, w14, lsl 16 ubfx x14, x11, #16, #8 eor w6, w6, w15, lsl 24 ubfx x15, x10, #56, #8 lsl w7, w7, #2 lsl w9, w9, #2 lsl w14, w14, #2 lsl w15, w15, #2 ldrb w7, [x5, x7, LSL 0] ldrb w9, [x5, x9, LSL 0] ldrb w14, [x5, x14, LSL 0] ldrb w15, [x5, x15, LSL 0] ubfx x8, x10, #32, #8 eor w7, w7, w9, lsl 8 ubfx x9, x10, #8, #8 eor w7, w7, w14, lsl 16 ubfx x14, x11, #48, #8 eor w7, w7, w15, lsl 24 ubfx x15, x11, #24, #8 bfi x6, x7, #32, #32 lsl w8, w8, #2 lsl w9, w9, #2 lsl w14, w14, #2 lsl w15, w15, #2 ldrb w8, [x5, x8, LSL 0] ldrb w9, [x5, x9, LSL 0] ldrb w14, [x5, x14, LSL 0] ldrb w15, [x5, x15, LSL 0] ubfx x13, x11, #56, #8 eor w8, w8, w9, lsl 8 ubfx x9, x11, #0, #8 eor w8, w8, w14, lsl 16 ubfx x14, x10, #40, #8 eor w7, w8, w15, lsl 24 ubfx x15, x10, #16, #8 lsl w13, w13, #2 lsl w9, w9, #2 lsl w14, w14, #2 lsl w15, w15, #2 ldrb w13, [x5, x13, LSL 0] ldrb w9, [x5, x9, LSL 0] ldrb w14, [x5, x14, LSL 0] ldrb w15, [x5, x15, LSL 0] eor w14, w14, w13, lsl 16 ldp x10, x11, [x17] eor w9, w9, w14, lsl 8 eor w9, w9, w15, lsl 16 bfi x7, x9, #32, #32 # XOR in Key Schedule eor x6, x6, x10 eor x7, x7, x11 rev32 x6, x6 rev32 x7, x7 str x6, [x1] str x7, [x1, #8] subs x2, x2, #16 add x0, x0, #16 add x1, x1, #16 bne L_AES_ECB_encrypt_loop_block_128 ldr x17, [x29, #24] ldp x29, x30, [sp], #32 ret #ifndef __APPLE__ .size AES_ECB_encrypt,.-AES_ECB_encrypt #endif /* __APPLE__ */ #endif /* HAVE_AESCCM || HAVE_AESGCM || WOLFSSL_AES_DIRECT || * WOLFSSL_AES_COUNTER || HAVE_AES_ECB */ #ifdef HAVE_AES_CBC #ifndef __APPLE__ .text .globl AES_CBC_encrypt .type AES_CBC_encrypt,@function .align 2 AES_CBC_encrypt: #else .section __TEXT,__text .globl _AES_CBC_encrypt .p2align 2 _AES_CBC_encrypt: #endif /* __APPLE__ */ stp x29, x30, [sp, #-32]! add x29, sp, #0 stp x17, x19, [x29, #16] #ifndef __APPLE__ adrp x6, L_AES_ARM64_te add x6, x6, :lo12:L_AES_ARM64_te #else adrp x6, L_AES_ARM64_te@PAGE add x6, x6, L_AES_ARM64_te@PAGEOFF #endif /* __APPLE__ */ ldp x7, x8, [x5] L_AES_CBC_encrypt_loop_block: mov x19, x3 ldr x11, [x0] ldr x12, [x0, #8] eor x7, x7, x11 eor x8, x8, x12 rev32 x7, x7 rev32 x8, x8 ldp x11, x12, [x19], #16 # Round: 0 - XOR in key schedule eor x7, x7, x11 eor x8, x8, x12 sub w17, w4, #2 L_AES_CBC_encrypt_loop_nr: ubfx x11, x7, #48, #8 ubfx x14, x7, #24, #8 ubfx x15, x8, #8, #8 ubfx x16, x8, #32, #8 ldr x9, [x6] ldr x9, [x6, #64] ldr x9, [x6, #128] ldr x9, [x6, #192] ldr x9, [x6, #256] ldr x9, [x6, #320] ldr x9, [x6, #384] ldr x9, [x6, #448] ldr x9, [x6, #512] ldr x9, [x6, #576] ldr x9, [x6, #640] ldr x9, [x6, #704] ldr x9, [x6, #768] ldr x9, [x6, #832] ldr x9, [x6, #896] ldr x9, [x6, #960] ldr w11, [x6, x11, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w15, [x6, x15, LSL 2] ldr w16, [x6, x16, LSL 2] ubfx x12, x8, #16, #8 eor w11, w11, w14, ror 24 ubfx x14, x7, #56, #8 eor w11, w11, w15, ror 8 ubfx x15, x8, #40, #8 eor w11, w11, w16, ror 16 ubfx x16, x7, #0, #8 ldr w12, [x6, x12, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w15, [x6, x15, LSL 2] ldr w16, [x6, x16, LSL 2] ubfx x13, x8, #48, #8 eor w12, w12, w14, ror 24 ubfx x14, x8, #24, #8 eor w12, w12, w15, ror 8 ubfx x15, x7, #8, #8 eor w12, w12, w16, ror 16 ubfx x16, x7, #32, #8 bfi x11, x12, #32, #32 ldr w13, [x6, x13, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w15, [x6, x15, LSL 2] ldr w16, [x6, x16, LSL 2] ubfx x9, x8, #0, #8 eor w13, w13, w14, ror 24 ubfx x14, x7, #16, #8 eor w13, w13, w15, ror 8 ubfx x15, x8, #56, #8 eor w12, w13, w16, ror 16 ubfx x16, x7, #40, #8 ldr w9, [x6, x9, LSL 2] ldr w15, [x6, x15, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w16, [x6, x16, LSL 2] eor w15, w15, w9, ror 24 ldp x7, x8, [x19], #16 eor w14, w14, w15, ror 24 eor w14, w14, w16, ror 8 bfi x12, x14, #32, #32 # XOR in Key Schedule eor x11, x11, x7 eor x12, x12, x8 ubfx x7, x11, #48, #8 ubfx x10, x11, #24, #8 ubfx x15, x12, #8, #8 ubfx x16, x12, #32, #8 ldr x13, [x6] ldr x13, [x6, #64] ldr x13, [x6, #128] ldr x13, [x6, #192] ldr x13, [x6, #256] ldr x13, [x6, #320] ldr x13, [x6, #384] ldr x13, [x6, #448] ldr x13, [x6, #512] ldr x13, [x6, #576] ldr x13, [x6, #640] ldr x13, [x6, #704] ldr x13, [x6, #768] ldr x13, [x6, #832] ldr x13, [x6, #896] ldr x13, [x6, #960] ldr w7, [x6, x7, LSL 2] ldr w10, [x6, x10, LSL 2] ldr w15, [x6, x15, LSL 2] ldr w16, [x6, x16, LSL 2] ubfx x8, x12, #16, #8 eor w7, w7, w10, ror 24 ubfx x10, x11, #56, #8 eor w7, w7, w15, ror 8 ubfx x15, x12, #40, #8 eor w7, w7, w16, ror 16 ubfx x16, x11, #0, #8 ldr w8, [x6, x8, LSL 2] ldr w10, [x6, x10, LSL 2] ldr w15, [x6, x15, LSL 2] ldr w16, [x6, x16, LSL 2] ubfx x9, x12, #48, #8 eor w8, w8, w10, ror 24 ubfx x10, x12, #24, #8 eor w8, w8, w15, ror 8 ubfx x15, x11, #8, #8 eor w8, w8, w16, ror 16 ubfx x16, x11, #32, #8 bfi x7, x8, #32, #32 ldr w9, [x6, x9, LSL 2] ldr w10, [x6, x10, LSL 2] ldr w15, [x6, x15, LSL 2] ldr w16, [x6, x16, LSL 2] ubfx x13, x12, #0, #8 eor w9, w9, w10, ror 24 ubfx x10, x11, #16, #8 eor w9, w9, w15, ror 8 ubfx x15, x12, #56, #8 eor w8, w9, w16, ror 16 ubfx x16, x11, #40, #8 ldr w13, [x6, x13, LSL 2] ldr w15, [x6, x15, LSL 2] ldr w10, [x6, x10, LSL 2] ldr w16, [x6, x16, LSL 2] eor w15, w15, w13, ror 24 ldp x11, x12, [x19], #16 eor w10, w10, w15, ror 24 eor w10, w10, w16, ror 8 bfi x8, x10, #32, #32 # XOR in Key Schedule eor x7, x7, x11 eor x8, x8, x12 subs w17, w17, #2 bne L_AES_CBC_encrypt_loop_nr ubfx x11, x7, #48, #8 ubfx x14, x7, #24, #8 ubfx x15, x8, #8, #8 ubfx x16, x8, #32, #8 ldr x9, [x6] ldr x9, [x6, #64] ldr x9, [x6, #128] ldr x9, [x6, #192] ldr x9, [x6, #256] ldr x9, [x6, #320] ldr x9, [x6, #384] ldr x9, [x6, #448] ldr x9, [x6, #512] ldr x9, [x6, #576] ldr x9, [x6, #640] ldr x9, [x6, #704] ldr x9, [x6, #768] ldr x9, [x6, #832] ldr x9, [x6, #896] ldr x9, [x6, #960] ldr w11, [x6, x11, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w15, [x6, x15, LSL 2] ldr w16, [x6, x16, LSL 2] ubfx x12, x8, #16, #8 eor w11, w11, w14, ror 24 ubfx x14, x7, #56, #8 eor w11, w11, w15, ror 8 ubfx x15, x8, #40, #8 eor w11, w11, w16, ror 16 ubfx x16, x7, #0, #8 ldr w12, [x6, x12, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w15, [x6, x15, LSL 2] ldr w16, [x6, x16, LSL 2] ubfx x13, x8, #48, #8 eor w12, w12, w14, ror 24 ubfx x14, x8, #24, #8 eor w12, w12, w15, ror 8 ubfx x15, x7, #8, #8 eor w12, w12, w16, ror 16 ubfx x16, x7, #32, #8 bfi x11, x12, #32, #32 ldr w13, [x6, x13, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w15, [x6, x15, LSL 2] ldr w16, [x6, x16, LSL 2] ubfx x9, x8, #0, #8 eor w13, w13, w14, ror 24 ubfx x14, x7, #16, #8 eor w13, w13, w15, ror 8 ubfx x15, x8, #56, #8 eor w12, w13, w16, ror 16 ubfx x16, x7, #40, #8 ldr w9, [x6, x9, LSL 2] ldr w15, [x6, x15, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w16, [x6, x16, LSL 2] eor w15, w15, w9, ror 24 ldp x7, x8, [x19], #16 eor w14, w14, w15, ror 24 eor w14, w14, w16, ror 8 bfi x12, x14, #32, #32 # XOR in Key Schedule eor x11, x11, x7 eor x12, x12, x8 ubfx x7, x12, #32, #8 ubfx x10, x12, #8, #8 ubfx x15, x11, #48, #8 ubfx x16, x11, #24, #8 lsl w7, w7, #2 lsl w10, w10, #2 lsl w15, w15, #2 lsl w16, w16, #2 ldr x14, [x6] ldr x14, [x6, #64] ldr x14, [x6, #128] ldr x14, [x6, #192] ldr x14, [x6, #256] ldr x14, [x6, #320] ldr x14, [x6, #384] ldr x14, [x6, #448] ldr x14, [x6, #512] ldr x14, [x6, #576] ldr x14, [x6, #640] ldr x14, [x6, #704] ldr x14, [x6, #768] ldr x14, [x6, #832] ldr x14, [x6, #896] ldr x14, [x6, #960] ldrb w7, [x6, x7, LSL 0] ldrb w10, [x6, x10, LSL 0] ldrb w15, [x6, x15, LSL 0] ldrb w16, [x6, x16, LSL 0] ubfx x8, x11, #0, #8 eor w7, w7, w10, lsl 8 ubfx x10, x12, #40, #8 eor w7, w7, w15, lsl 16 ubfx x15, x12, #16, #8 eor w7, w7, w16, lsl 24 ubfx x16, x11, #56, #8 lsl w8, w8, #2 lsl w10, w10, #2 lsl w15, w15, #2 lsl w16, w16, #2 ldrb w8, [x6, x8, LSL 0] ldrb w10, [x6, x10, LSL 0] ldrb w15, [x6, x15, LSL 0] ldrb w16, [x6, x16, LSL 0] ubfx x9, x11, #32, #8 eor w8, w8, w10, lsl 8 ubfx x10, x11, #8, #8 eor w8, w8, w15, lsl 16 ubfx x15, x12, #48, #8 eor w8, w8, w16, lsl 24 ubfx x16, x12, #24, #8 bfi x7, x8, #32, #32 lsl w9, w9, #2 lsl w10, w10, #2 lsl w15, w15, #2 lsl w16, w16, #2 ldrb w9, [x6, x9, LSL 0] ldrb w10, [x6, x10, LSL 0] ldrb w15, [x6, x15, LSL 0] ldrb w16, [x6, x16, LSL 0] ubfx x14, x12, #56, #8 eor w9, w9, w10, lsl 8 ubfx x10, x12, #0, #8 eor w9, w9, w15, lsl 16 ubfx x15, x11, #40, #8 eor w8, w9, w16, lsl 24 ubfx x16, x11, #16, #8 lsl w14, w14, #2 lsl w10, w10, #2 lsl w15, w15, #2 lsl w16, w16, #2 ldrb w14, [x6, x14, LSL 0] ldrb w10, [x6, x10, LSL 0] ldrb w15, [x6, x15, LSL 0] ldrb w16, [x6, x16, LSL 0] eor w15, w15, w14, lsl 16 ldp x11, x12, [x19] eor w10, w10, w15, lsl 8 eor w10, w10, w16, lsl 16 bfi x8, x10, #32, #32 # XOR in Key Schedule eor x7, x7, x11 eor x8, x8, x12 rev32 x7, x7 rev32 x8, x8 str x7, [x1] str x8, [x1, #8] subs x2, x2, #16 add x0, x0, #16 add x1, x1, #16 bne L_AES_CBC_encrypt_loop_block stp x7, x8, [x5] ldp x17, x19, [x29, #16] ldp x29, x30, [sp], #32 ret #ifndef __APPLE__ .size AES_CBC_encrypt,.-AES_CBC_encrypt #endif /* __APPLE__ */ #endif /* HAVE_AES_CBC */ #ifdef WOLFSSL_AES_COUNTER #ifndef __APPLE__ .text .globl AES_CTR_encrypt .type AES_CTR_encrypt,@function .align 2 AES_CTR_encrypt: #else .section __TEXT,__text .globl _AES_CTR_encrypt .p2align 2 _AES_CTR_encrypt: #endif /* __APPLE__ */ stp x29, x30, [sp, #-48]! add x29, sp, #0 stp x17, x19, [x29, #16] stp x20, x21, [x29, #32] #ifndef __APPLE__ adrp x6, L_AES_ARM64_te add x6, x6, :lo12:L_AES_ARM64_te #else adrp x6, L_AES_ARM64_te@PAGE add x6, x6, L_AES_ARM64_te@PAGEOFF #endif /* __APPLE__ */ ldp x15, x16, [x5] rev32 x15, x15 rev32 x16, x16 L_AES_CTR_encrypt_loop_block_128: mov x21, x3 ldp x11, x12, [x21], #16 # Round: 0 - XOR in key schedule eor x7, x15, x11 eor x8, x16, x12 sub w20, w4, #2 L_AES_CTR_encrypt_loop_nr: ubfx x11, x7, #48, #8 ubfx x14, x7, #24, #8 ubfx x17, x8, #8, #8 ubfx x19, x8, #32, #8 ldr x9, [x6] ldr x9, [x6, #64] ldr x9, [x6, #128] ldr x9, [x6, #192] ldr x9, [x6, #256] ldr x9, [x6, #320] ldr x9, [x6, #384] ldr x9, [x6, #448] ldr x9, [x6, #512] ldr x9, [x6, #576] ldr x9, [x6, #640] ldr x9, [x6, #704] ldr x9, [x6, #768] ldr x9, [x6, #832] ldr x9, [x6, #896] ldr x9, [x6, #960] ldr w11, [x6, x11, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w17, [x6, x17, LSL 2] ldr w19, [x6, x19, LSL 2] ubfx x12, x8, #16, #8 eor w11, w11, w14, ror 24 ubfx x14, x7, #56, #8 eor w11, w11, w17, ror 8 ubfx x17, x8, #40, #8 eor w11, w11, w19, ror 16 ubfx x19, x7, #0, #8 ldr w12, [x6, x12, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w17, [x6, x17, LSL 2] ldr w19, [x6, x19, LSL 2] ubfx x13, x8, #48, #8 eor w12, w12, w14, ror 24 ubfx x14, x8, #24, #8 eor w12, w12, w17, ror 8 ubfx x17, x7, #8, #8 eor w12, w12, w19, ror 16 ubfx x19, x7, #32, #8 bfi x11, x12, #32, #32 ldr w13, [x6, x13, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w17, [x6, x17, LSL 2] ldr w19, [x6, x19, LSL 2] ubfx x9, x8, #0, #8 eor w13, w13, w14, ror 24 ubfx x14, x7, #16, #8 eor w13, w13, w17, ror 8 ubfx x17, x8, #56, #8 eor w12, w13, w19, ror 16 ubfx x19, x7, #40, #8 ldr w9, [x6, x9, LSL 2] ldr w17, [x6, x17, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w19, [x6, x19, LSL 2] eor w17, w17, w9, ror 24 ldp x7, x8, [x21], #16 eor w14, w14, w17, ror 24 eor w14, w14, w19, ror 8 bfi x12, x14, #32, #32 # XOR in Key Schedule eor x11, x11, x7 eor x12, x12, x8 ubfx x7, x11, #48, #8 ubfx x10, x11, #24, #8 ubfx x17, x12, #8, #8 ubfx x19, x12, #32, #8 ldr x13, [x6] ldr x13, [x6, #64] ldr x13, [x6, #128] ldr x13, [x6, #192] ldr x13, [x6, #256] ldr x13, [x6, #320] ldr x13, [x6, #384] ldr x13, [x6, #448] ldr x13, [x6, #512] ldr x13, [x6, #576] ldr x13, [x6, #640] ldr x13, [x6, #704] ldr x13, [x6, #768] ldr x13, [x6, #832] ldr x13, [x6, #896] ldr x13, [x6, #960] ldr w7, [x6, x7, LSL 2] ldr w10, [x6, x10, LSL 2] ldr w17, [x6, x17, LSL 2] ldr w19, [x6, x19, LSL 2] ubfx x8, x12, #16, #8 eor w7, w7, w10, ror 24 ubfx x10, x11, #56, #8 eor w7, w7, w17, ror 8 ubfx x17, x12, #40, #8 eor w7, w7, w19, ror 16 ubfx x19, x11, #0, #8 ldr w8, [x6, x8, LSL 2] ldr w10, [x6, x10, LSL 2] ldr w17, [x6, x17, LSL 2] ldr w19, [x6, x19, LSL 2] ubfx x9, x12, #48, #8 eor w8, w8, w10, ror 24 ubfx x10, x12, #24, #8 eor w8, w8, w17, ror 8 ubfx x17, x11, #8, #8 eor w8, w8, w19, ror 16 ubfx x19, x11, #32, #8 bfi x7, x8, #32, #32 ldr w9, [x6, x9, LSL 2] ldr w10, [x6, x10, LSL 2] ldr w17, [x6, x17, LSL 2] ldr w19, [x6, x19, LSL 2] ubfx x13, x12, #0, #8 eor w9, w9, w10, ror 24 ubfx x10, x11, #16, #8 eor w9, w9, w17, ror 8 ubfx x17, x12, #56, #8 eor w8, w9, w19, ror 16 ubfx x19, x11, #40, #8 ldr w13, [x6, x13, LSL 2] ldr w17, [x6, x17, LSL 2] ldr w10, [x6, x10, LSL 2] ldr w19, [x6, x19, LSL 2] eor w17, w17, w13, ror 24 ldp x11, x12, [x21], #16 eor w10, w10, w17, ror 24 eor w10, w10, w19, ror 8 bfi x8, x10, #32, #32 # XOR in Key Schedule eor x7, x7, x11 eor x8, x8, x12 subs w20, w20, #2 bne L_AES_CTR_encrypt_loop_nr ubfx x11, x7, #48, #8 ubfx x14, x7, #24, #8 ubfx x17, x8, #8, #8 ubfx x19, x8, #32, #8 ldr x9, [x6] ldr x9, [x6, #64] ldr x9, [x6, #128] ldr x9, [x6, #192] ldr x9, [x6, #256] ldr x9, [x6, #320] ldr x9, [x6, #384] ldr x9, [x6, #448] ldr x9, [x6, #512] ldr x9, [x6, #576] ldr x9, [x6, #640] ldr x9, [x6, #704] ldr x9, [x6, #768] ldr x9, [x6, #832] ldr x9, [x6, #896] ldr x9, [x6, #960] ldr w11, [x6, x11, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w17, [x6, x17, LSL 2] ldr w19, [x6, x19, LSL 2] ubfx x12, x8, #16, #8 eor w11, w11, w14, ror 24 ubfx x14, x7, #56, #8 eor w11, w11, w17, ror 8 ubfx x17, x8, #40, #8 eor w11, w11, w19, ror 16 ubfx x19, x7, #0, #8 ldr w12, [x6, x12, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w17, [x6, x17, LSL 2] ldr w19, [x6, x19, LSL 2] ubfx x13, x8, #48, #8 eor w12, w12, w14, ror 24 ubfx x14, x8, #24, #8 eor w12, w12, w17, ror 8 ubfx x17, x7, #8, #8 eor w12, w12, w19, ror 16 ubfx x19, x7, #32, #8 bfi x11, x12, #32, #32 ldr w13, [x6, x13, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w17, [x6, x17, LSL 2] ldr w19, [x6, x19, LSL 2] ubfx x9, x8, #0, #8 eor w13, w13, w14, ror 24 ubfx x14, x7, #16, #8 eor w13, w13, w17, ror 8 ubfx x17, x8, #56, #8 eor w12, w13, w19, ror 16 ubfx x19, x7, #40, #8 ldr w9, [x6, x9, LSL 2] ldr w17, [x6, x17, LSL 2] ldr w14, [x6, x14, LSL 2] ldr w19, [x6, x19, LSL 2] eor w17, w17, w9, ror 24 ldp x7, x8, [x21], #16 eor w14, w14, w17, ror 24 eor w14, w14, w19, ror 8 bfi x12, x14, #32, #32 # XOR in Key Schedule eor x11, x11, x7 eor x12, x12, x8 ubfx x7, x12, #32, #8 ubfx x10, x12, #8, #8 ubfx x17, x11, #48, #8 ubfx x19, x11, #24, #8 lsl w7, w7, #2 lsl w10, w10, #2 lsl w17, w17, #2 lsl w19, w19, #2 ldr x14, [x6] ldr x14, [x6, #64] ldr x14, [x6, #128] ldr x14, [x6, #192] ldr x14, [x6, #256] ldr x14, [x6, #320] ldr x14, [x6, #384] ldr x14, [x6, #448] ldr x14, [x6, #512] ldr x14, [x6, #576] ldr x14, [x6, #640] ldr x14, [x6, #704] ldr x14, [x6, #768] ldr x14, [x6, #832] ldr x14, [x6, #896] ldr x14, [x6, #960] ldrb w7, [x6, x7, LSL 0] ldrb w10, [x6, x10, LSL 0] ldrb w17, [x6, x17, LSL 0] ldrb w19, [x6, x19, LSL 0] ubfx x8, x11, #0, #8 eor w7, w7, w10, lsl 8 ubfx x10, x12, #40, #8 eor w7, w7, w17, lsl 16 ubfx x17, x12, #16, #8 eor w7, w7, w19, lsl 24 ubfx x19, x11, #56, #8 lsl w8, w8, #2 lsl w10, w10, #2 lsl w17, w17, #2 lsl w19, w19, #2 ldrb w8, [x6, x8, LSL 0] ldrb w10, [x6, x10, LSL 0] ldrb w17, [x6, x17, LSL 0] ldrb w19, [x6, x19, LSL 0] ubfx x9, x11, #32, #8 eor w8, w8, w10, lsl 8 ubfx x10, x11, #8, #8 eor w8, w8, w17, lsl 16 ubfx x17, x12, #48, #8 eor w8, w8, w19, lsl 24 ubfx x19, x12, #24, #8 bfi x7, x8, #32, #32 lsl w9, w9, #2 lsl w10, w10, #2 lsl w17, w17, #2 lsl w19, w19, #2 ldrb w9, [x6, x9, LSL 0] ldrb w10, [x6, x10, LSL 0] ldrb w17, [x6, x17, LSL 0] ldrb w19, [x6, x19, LSL 0] ubfx x14, x12, #56, #8 eor w9, w9, w10, lsl 8 ubfx x10, x12, #0, #8 eor w9, w9, w17, lsl 16 ubfx x17, x11, #40, #8 eor w8, w9, w19, lsl 24 ubfx x19, x11, #16, #8 lsl w14, w14, #2 lsl w10, w10, #2 lsl w17, w17, #2 lsl w19, w19, #2 ldrb w14, [x6, x14, LSL 0] ldrb w10, [x6, x10, LSL 0] ldrb w17, [x6, x17, LSL 0] ldrb w19, [x6, x19, LSL 0] eor w17, w17, w14, lsl 16 ldp x11, x12, [x21] eor w10, w10, w17, lsl 8 eor w10, w10, w19, lsl 16 bfi x8, x10, #32, #32 # XOR in Key Schedule eor x7, x7, x11 eor x8, x8, x12 rev32 x7, x7 rev32 x8, x8 ldr x11, [x0] ldr x12, [x0, #8] eor x7, x7, x11 eor x8, x8, x12 str x7, [x1] str x8, [x1, #8] ror x16, x16, #32 ror x15, x15, #32 adds x16, x16, #1 adc x15, x15, xzr ror x16, x16, #32 ror x15, x15, #32 subs x2, x2, #16 add x0, x0, #16 add x1, x1, #16 bne L_AES_CTR_encrypt_loop_block_128 rev32 x15, x15 rev32 x16, x16 stp x15, x16, [x5] ldp x17, x19, [x29, #16] ldp x20, x21, [x29, #32] ldp x29, x30, [sp], #48 ret #ifndef __APPLE__ .size AES_CTR_encrypt,.-AES_CTR_encrypt #endif /* __APPLE__ */ #endif /* WOLFSSL_AES_COUNTER */ #ifdef HAVE_AES_DECRYPT #if defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || \ defined(HAVE_AES_CBC) || defined(HAVE_AES_ECB) #ifndef __APPLE__ .text .section .rodata .type L_AES_ARM64_td4, %object .size L_AES_ARM64_td4, 256 #else .section __DATA,__data #endif /* __APPLE__ */ # 8-byte aligned, 64-bit aligned #ifndef __APPLE__ .align 3 #else .p2align 3 #endif /* __APPLE__ */ L_AES_ARM64_td4: .byte 0x52,0x09,0x6a,0xd5,0x30,0x36,0xa5,0x38 .byte 0xbf,0x40,0xa3,0x9e,0x81,0xf3,0xd7,0xfb .byte 0x7c,0xe3,0x39,0x82,0x9b,0x2f,0xff,0x87 .byte 0x34,0x8e,0x43,0x44,0xc4,0xde,0xe9,0xcb .byte 0x54,0x7b,0x94,0x32,0xa6,0xc2,0x23,0x3d .byte 0xee,0x4c,0x95,0x0b,0x42,0xfa,0xc3,0x4e .byte 0x08,0x2e,0xa1,0x66,0x28,0xd9,0x24,0xb2 .byte 0x76,0x5b,0xa2,0x49,0x6d,0x8b,0xd1,0x25 .byte 0x72,0xf8,0xf6,0x64,0x86,0x68,0x98,0x16 .byte 0xd4,0xa4,0x5c,0xcc,0x5d,0x65,0xb6,0x92 .byte 0x6c,0x70,0x48,0x50,0xfd,0xed,0xb9,0xda .byte 0x5e,0x15,0x46,0x57,0xa7,0x8d,0x9d,0x84 .byte 0x90,0xd8,0xab,0x00,0x8c,0xbc,0xd3,0x0a .byte 0xf7,0xe4,0x58,0x05,0xb8,0xb3,0x45,0x06 .byte 0xd0,0x2c,0x1e,0x8f,0xca,0x3f,0x0f,0x02 .byte 0xc1,0xaf,0xbd,0x03,0x01,0x13,0x8a,0x6b .byte 0x3a,0x91,0x11,0x41,0x4f,0x67,0xdc,0xea .byte 0x97,0xf2,0xcf,0xce,0xf0,0xb4,0xe6,0x73 .byte 0x96,0xac,0x74,0x22,0xe7,0xad,0x35,0x85 .byte 0xe2,0xf9,0x37,0xe8,0x1c,0x75,0xdf,0x6e .byte 0x47,0xf1,0x1a,0x71,0x1d,0x29,0xc5,0x89 .byte 0x6f,0xb7,0x62,0x0e,0xaa,0x18,0xbe,0x1b .byte 0xfc,0x56,0x3e,0x4b,0xc6,0xd2,0x79,0x20 .byte 0x9a,0xdb,0xc0,0xfe,0x78,0xcd,0x5a,0xf4 .byte 0x1f,0xdd,0xa8,0x33,0x88,0x07,0xc7,0x31 .byte 0xb1,0x12,0x10,0x59,0x27,0x80,0xec,0x5f .byte 0x60,0x51,0x7f,0xa9,0x19,0xb5,0x4a,0x0d .byte 0x2d,0xe5,0x7a,0x9f,0x93,0xc9,0x9c,0xef .byte 0xa0,0xe0,0x3b,0x4d,0xae,0x2a,0xf5,0xb0 .byte 0xc8,0xeb,0xbb,0x3c,0x83,0x53,0x99,0x61 .byte 0x17,0x2b,0x04,0x7e,0xba,0x77,0xd6,0x26 .byte 0xe1,0x69,0x14,0x63,0x55,0x21,0x0c,0x7d #if defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER) || defined(HAVE_AES_ECB) #ifndef __APPLE__ .text .globl AES_ECB_decrypt .type AES_ECB_decrypt,@function .align 2 AES_ECB_decrypt: #else .section __TEXT,__text .globl _AES_ECB_decrypt .p2align 2 _AES_ECB_decrypt: #endif /* __APPLE__ */ stp x29, x30, [sp, #-32]! add x29, sp, #0 stp x17, x19, [x29, #16] #ifndef __APPLE__ adrp x5, L_AES_ARM64_td add x5, x5, :lo12:L_AES_ARM64_td #else adrp x5, L_AES_ARM64_td@PAGE add x5, x5, L_AES_ARM64_td@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x6, L_AES_ARM64_td4 add x6, x6, :lo12:L_AES_ARM64_td4 #else adrp x6, L_AES_ARM64_td4@PAGE add x6, x6, L_AES_ARM64_td4@PAGEOFF #endif /* __APPLE__ */ L_AES_ECB_decrypt_loop_block: mov x19, x3 ldr x7, [x0] ldr x8, [x0, #8] rev32 x7, x7 rev32 x8, x8 ldp x11, x12, [x19], #16 # Round: 0 - XOR in key schedule eor x7, x7, x11 eor x8, x8, x12 sub w17, w4, #2 L_AES_ECB_decrypt_loop_nr: ubfx x11, x8, #48, #8 ubfx x14, x7, #24, #8 ubfx x15, x8, #8, #8 ubfx x16, x7, #32, #8 ldr x9, [x5] ldr x9, [x5, #64] ldr x9, [x5, #128] ldr x9, [x5, #192] ldr x9, [x5, #256] ldr x9, [x5, #320] ldr x9, [x5, #384] ldr x9, [x5, #448] ldr x9, [x5, #512] ldr x9, [x5, #576] ldr x9, [x5, #640] ldr x9, [x5, #704] ldr x9, [x5, #768] ldr x9, [x5, #832] ldr x9, [x5, #896] ldr x9, [x5, #960] ldr w11, [x5, x11, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ldr w16, [x5, x16, LSL 2] ubfx x12, x7, #16, #8 eor w11, w11, w14, ror 24 ubfx x14, x7, #56, #8 eor w11, w11, w15, ror 8 ubfx x15, x8, #40, #8 eor w11, w11, w16, ror 16 ubfx x16, x8, #0, #8 ldr w12, [x5, x12, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ldr w16, [x5, x16, LSL 2] ubfx x13, x7, #48, #8 eor w12, w12, w14, ror 24 ubfx x14, x8, #24, #8 eor w12, w12, w15, ror 8 ubfx x15, x7, #8, #8 eor w12, w12, w16, ror 16 ubfx x16, x8, #32, #8 bfi x11, x12, #32, #32 ldr w13, [x5, x13, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ldr w16, [x5, x16, LSL 2] ubfx x9, x7, #0, #8 eor w13, w13, w14, ror 24 ubfx x14, x8, #16, #8 eor w13, w13, w15, ror 8 ubfx x15, x8, #56, #8 eor w12, w13, w16, ror 16 ubfx x16, x7, #40, #8 ldr w9, [x5, x9, LSL 2] ldr w15, [x5, x15, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w16, [x5, x16, LSL 2] eor w15, w15, w9, ror 24 ldp x7, x8, [x19], #16 eor w14, w14, w16, ror 8 eor w14, w14, w15, ror 24 bfi x12, x14, #32, #32 # XOR in Key Schedule eor x11, x11, x7 eor x12, x12, x8 ubfx x7, x12, #48, #8 ubfx x10, x11, #24, #8 ubfx x15, x12, #8, #8 ubfx x16, x11, #32, #8 ldr x13, [x5] ldr x13, [x5, #64] ldr x13, [x5, #128] ldr x13, [x5, #192] ldr x13, [x5, #256] ldr x13, [x5, #320] ldr x13, [x5, #384] ldr x13, [x5, #448] ldr x13, [x5, #512] ldr x13, [x5, #576] ldr x13, [x5, #640] ldr x13, [x5, #704] ldr x13, [x5, #768] ldr x13, [x5, #832] ldr x13, [x5, #896] ldr x13, [x5, #960] ldr w7, [x5, x7, LSL 2] ldr w10, [x5, x10, LSL 2] ldr w15, [x5, x15, LSL 2] ldr w16, [x5, x16, LSL 2] ubfx x8, x11, #16, #8 eor w7, w7, w10, ror 24 ubfx x10, x11, #56, #8 eor w7, w7, w15, ror 8 ubfx x15, x12, #40, #8 eor w7, w7, w16, ror 16 ubfx x16, x12, #0, #8 ldr w8, [x5, x8, LSL 2] ldr w10, [x5, x10, LSL 2] ldr w15, [x5, x15, LSL 2] ldr w16, [x5, x16, LSL 2] ubfx x9, x11, #48, #8 eor w8, w8, w10, ror 24 ubfx x10, x12, #24, #8 eor w8, w8, w15, ror 8 ubfx x15, x11, #8, #8 eor w8, w8, w16, ror 16 ubfx x16, x12, #32, #8 bfi x7, x8, #32, #32 ldr w9, [x5, x9, LSL 2] ldr w10, [x5, x10, LSL 2] ldr w15, [x5, x15, LSL 2] ldr w16, [x5, x16, LSL 2] ubfx x13, x11, #0, #8 eor w9, w9, w10, ror 24 ubfx x10, x12, #16, #8 eor w9, w9, w15, ror 8 ubfx x15, x12, #56, #8 eor w8, w9, w16, ror 16 ubfx x16, x11, #40, #8 ldr w13, [x5, x13, LSL 2] ldr w15, [x5, x15, LSL 2] ldr w10, [x5, x10, LSL 2] ldr w16, [x5, x16, LSL 2] eor w15, w15, w13, ror 24 ldp x11, x12, [x19], #16 eor w10, w10, w16, ror 8 eor w10, w10, w15, ror 24 bfi x8, x10, #32, #32 # XOR in Key Schedule eor x7, x7, x11 eor x8, x8, x12 subs w17, w17, #2 bne L_AES_ECB_decrypt_loop_nr ubfx x11, x8, #48, #8 ubfx x14, x7, #24, #8 ubfx x15, x8, #8, #8 ubfx x16, x7, #32, #8 ldr x9, [x5] ldr x9, [x5, #64] ldr x9, [x5, #128] ldr x9, [x5, #192] ldr x9, [x5, #256] ldr x9, [x5, #320] ldr x9, [x5, #384] ldr x9, [x5, #448] ldr x9, [x5, #512] ldr x9, [x5, #576] ldr x9, [x5, #640] ldr x9, [x5, #704] ldr x9, [x5, #768] ldr x9, [x5, #832] ldr x9, [x5, #896] ldr x9, [x5, #960] ldr w11, [x5, x11, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ldr w16, [x5, x16, LSL 2] ubfx x12, x7, #16, #8 eor w11, w11, w14, ror 24 ubfx x14, x7, #56, #8 eor w11, w11, w15, ror 8 ubfx x15, x8, #40, #8 eor w11, w11, w16, ror 16 ubfx x16, x8, #0, #8 ldr w12, [x5, x12, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ldr w16, [x5, x16, LSL 2] ubfx x13, x7, #48, #8 eor w12, w12, w14, ror 24 ubfx x14, x8, #24, #8 eor w12, w12, w15, ror 8 ubfx x15, x7, #8, #8 eor w12, w12, w16, ror 16 ubfx x16, x8, #32, #8 bfi x11, x12, #32, #32 ldr w13, [x5, x13, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w15, [x5, x15, LSL 2] ldr w16, [x5, x16, LSL 2] ubfx x9, x7, #0, #8 eor w13, w13, w14, ror 24 ubfx x14, x8, #16, #8 eor w13, w13, w15, ror 8 ubfx x15, x8, #56, #8 eor w12, w13, w16, ror 16 ubfx x16, x7, #40, #8 ldr w9, [x5, x9, LSL 2] ldr w15, [x5, x15, LSL 2] ldr w14, [x5, x14, LSL 2] ldr w16, [x5, x16, LSL 2] eor w15, w15, w9, ror 24 ldp x7, x8, [x19], #16 eor w14, w14, w16, ror 8 eor w14, w14, w15, ror 24 bfi x12, x14, #32, #32 # XOR in Key Schedule eor x11, x11, x7 eor x12, x12, x8 ubfx x7, x11, #32, #8 ubfx x10, x12, #8, #8 ubfx x15, x12, #48, #8 ubfx x16, x11, #24, #8 ldr x14, [x6] ldr x14, [x6, #64] ldr x14, [x6, #128] ldr x14, [x6, #192] ldrb w7, [x6, x7, LSL 0] ldrb w10, [x6, x10, LSL 0] ldrb w15, [x6, x15, LSL 0] ldrb w16, [x6, x16, LSL 0] ubfx x8, x12, #0, #8 eor w7, w7, w10, lsl 8 ubfx x10, x12, #40, #8 eor w7, w7, w15, lsl 16 ubfx x15, x11, #16, #8 eor w7, w7, w16, lsl 24 ubfx x16, x11, #56, #8 ldrb w10, [x6, x10, LSL 0] ldrb w16, [x6, x16, LSL 0] ldrb w8, [x6, x8, LSL 0] ldrb w15, [x6, x15, LSL 0] ubfx x9, x12, #32, #8 eor w8, w8, w10, lsl 8 ubfx x10, x11, #8, #8 eor w8, w8, w15, lsl 16 ubfx x15, x11, #48, #8 eor w8, w8, w16, lsl 24 ubfx x16, x12, #24, #8 bfi x7, x8, #32, #32 ldrb w10, [x6, x10, LSL 0] ldrb w16, [x6, x16, LSL 0] ldrb w9, [x6, x9, LSL 0] ldrb w15, [x6, x15, LSL 0] ubfx x14, x12, #56, #8 eor w9, w9, w10, lsl 8 ubfx x10, x11, #0, #8 eor w9, w9, w15, lsl 16 ubfx x15, x11, #40, #8 eor w8, w9, w16, lsl 24 ubfx x16, x12, #16, #8 ldrb w14, [x6, x14, LSL 0] ldrb w15, [x6, x15, LSL 0] ldrb w10, [x6, x10, LSL 0] ldrb w16, [x6, x16, LSL 0] eor w15, w15, w14, lsl 16 ldp x11, x12, [x19] eor w10, w10, w15, lsl 8 eor w10, w10, w16, lsl 16 bfi x8, x10, #32, #32 # XOR in Key Schedule eor x7, x7, x11 eor x8, x8, x12 rev32 x7, x7 rev32 x8, x8 str x7, [x1] str x8, [x1, #8] subs x2, x2, #16 add x0, x0, #16 add x1, x1, #16 bne L_AES_ECB_decrypt_loop_block ldp x17, x19, [x29, #16] ldp x29, x30, [sp], #32 ret #ifndef __APPLE__ .size AES_ECB_decrypt,.-AES_ECB_decrypt #endif /* __APPLE__ */ #endif /* WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER || defined(HAVE_AES_ECB) */ #ifdef HAVE_AES_CBC #ifndef __APPLE__ .text .globl AES_CBC_decrypt .type AES_CBC_decrypt,@function .align 2 AES_CBC_decrypt: #else .section __TEXT,__text .globl _AES_CBC_decrypt .p2align 2 _AES_CBC_decrypt: #endif /* __APPLE__ */ stp x29, x30, [sp, #-48]! add x29, sp, #0 stp x17, x19, [x29, #24] str x20, [x29, #40] #ifndef __APPLE__ adrp x6, L_AES_ARM64_td4 add x6, x6, :lo12:L_AES_ARM64_td4 #else adrp x6, L_AES_ARM64_td4@PAGE add x6, x6, L_AES_ARM64_td4@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x7, L_AES_ARM64_td add x7, x7, :lo12:L_AES_ARM64_td #else adrp x7, L_AES_ARM64_td@PAGE add x7, x7, L_AES_ARM64_td@PAGEOFF #endif /* __APPLE__ */ L_AES_CBC_decrypt_loop_block: mov x20, x3 ldr x8, [x0] ldr x9, [x0, #8] stnp x8, x9, [x5, #16] rev32 x8, x8 rev32 x9, x9 ldp x12, x13, [x20], #16 # Round: 0 - XOR in key schedule eor x8, x8, x12 eor x9, x9, x13 sub w19, w4, #2 L_AES_CBC_decrypt_loop_nr_even: ubfx x12, x9, #48, #8 ubfx x15, x8, #24, #8 ubfx x16, x9, #8, #8 ubfx x17, x8, #32, #8 ldr x10, [x7] ldr x10, [x7, #64] ldr x10, [x7, #128] ldr x10, [x7, #192] ldr x10, [x7, #256] ldr x10, [x7, #320] ldr x10, [x7, #384] ldr x10, [x7, #448] ldr x10, [x7, #512] ldr x10, [x7, #576] ldr x10, [x7, #640] ldr x10, [x7, #704] ldr x10, [x7, #768] ldr x10, [x7, #832] ldr x10, [x7, #896] ldr x10, [x7, #960] ldr w12, [x7, x12, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x13, x8, #16, #8 eor w12, w12, w15, ror 24 ubfx x15, x8, #56, #8 eor w12, w12, w16, ror 8 ubfx x16, x9, #40, #8 eor w12, w12, w17, ror 16 ubfx x17, x9, #0, #8 ldr w13, [x7, x13, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x14, x8, #48, #8 eor w13, w13, w15, ror 24 ubfx x15, x9, #24, #8 eor w13, w13, w16, ror 8 ubfx x16, x8, #8, #8 eor w13, w13, w17, ror 16 ubfx x17, x9, #32, #8 bfi x12, x13, #32, #32 ldr w14, [x7, x14, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x10, x8, #0, #8 eor w14, w14, w15, ror 24 ubfx x15, x9, #16, #8 eor w14, w14, w16, ror 8 ubfx x16, x9, #56, #8 eor w13, w14, w17, ror 16 ubfx x17, x8, #40, #8 ldr w10, [x7, x10, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w17, [x7, x17, LSL 2] eor w16, w16, w10, ror 24 ldp x8, x9, [x20], #16 eor w15, w15, w17, ror 8 eor w15, w15, w16, ror 24 bfi x13, x15, #32, #32 # XOR in Key Schedule eor x12, x12, x8 eor x13, x13, x9 ubfx x8, x13, #48, #8 ubfx x11, x12, #24, #8 ubfx x16, x13, #8, #8 ubfx x17, x12, #32, #8 ldr x14, [x7] ldr x14, [x7, #64] ldr x14, [x7, #128] ldr x14, [x7, #192] ldr x14, [x7, #256] ldr x14, [x7, #320] ldr x14, [x7, #384] ldr x14, [x7, #448] ldr x14, [x7, #512] ldr x14, [x7, #576] ldr x14, [x7, #640] ldr x14, [x7, #704] ldr x14, [x7, #768] ldr x14, [x7, #832] ldr x14, [x7, #896] ldr x14, [x7, #960] ldr w8, [x7, x8, LSL 2] ldr w11, [x7, x11, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x9, x12, #16, #8 eor w8, w8, w11, ror 24 ubfx x11, x12, #56, #8 eor w8, w8, w16, ror 8 ubfx x16, x13, #40, #8 eor w8, w8, w17, ror 16 ubfx x17, x13, #0, #8 ldr w9, [x7, x9, LSL 2] ldr w11, [x7, x11, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x10, x12, #48, #8 eor w9, w9, w11, ror 24 ubfx x11, x13, #24, #8 eor w9, w9, w16, ror 8 ubfx x16, x12, #8, #8 eor w9, w9, w17, ror 16 ubfx x17, x13, #32, #8 bfi x8, x9, #32, #32 ldr w10, [x7, x10, LSL 2] ldr w11, [x7, x11, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x14, x12, #0, #8 eor w10, w10, w11, ror 24 ubfx x11, x13, #16, #8 eor w10, w10, w16, ror 8 ubfx x16, x13, #56, #8 eor w9, w10, w17, ror 16 ubfx x17, x12, #40, #8 ldr w14, [x7, x14, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w11, [x7, x11, LSL 2] ldr w17, [x7, x17, LSL 2] eor w16, w16, w14, ror 24 ldp x12, x13, [x20], #16 eor w11, w11, w17, ror 8 eor w11, w11, w16, ror 24 bfi x9, x11, #32, #32 # XOR in Key Schedule eor x8, x8, x12 eor x9, x9, x13 subs w19, w19, #2 bne L_AES_CBC_decrypt_loop_nr_even ubfx x12, x9, #48, #8 ubfx x15, x8, #24, #8 ubfx x16, x9, #8, #8 ubfx x17, x8, #32, #8 ldr x10, [x7] ldr x10, [x7, #64] ldr x10, [x7, #128] ldr x10, [x7, #192] ldr x10, [x7, #256] ldr x10, [x7, #320] ldr x10, [x7, #384] ldr x10, [x7, #448] ldr x10, [x7, #512] ldr x10, [x7, #576] ldr x10, [x7, #640] ldr x10, [x7, #704] ldr x10, [x7, #768] ldr x10, [x7, #832] ldr x10, [x7, #896] ldr x10, [x7, #960] ldr w12, [x7, x12, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x13, x8, #16, #8 eor w12, w12, w15, ror 24 ubfx x15, x8, #56, #8 eor w12, w12, w16, ror 8 ubfx x16, x9, #40, #8 eor w12, w12, w17, ror 16 ubfx x17, x9, #0, #8 ldr w13, [x7, x13, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x14, x8, #48, #8 eor w13, w13, w15, ror 24 ubfx x15, x9, #24, #8 eor w13, w13, w16, ror 8 ubfx x16, x8, #8, #8 eor w13, w13, w17, ror 16 ubfx x17, x9, #32, #8 bfi x12, x13, #32, #32 ldr w14, [x7, x14, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x10, x8, #0, #8 eor w14, w14, w15, ror 24 ubfx x15, x9, #16, #8 eor w14, w14, w16, ror 8 ubfx x16, x9, #56, #8 eor w13, w14, w17, ror 16 ubfx x17, x8, #40, #8 ldr w10, [x7, x10, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w17, [x7, x17, LSL 2] eor w16, w16, w10, ror 24 ldp x8, x9, [x20], #16 eor w15, w15, w17, ror 8 eor w15, w15, w16, ror 24 bfi x13, x15, #32, #32 # XOR in Key Schedule eor x12, x12, x8 eor x13, x13, x9 ubfx x8, x12, #32, #8 ubfx x11, x13, #8, #8 ubfx x16, x13, #48, #8 ubfx x17, x12, #24, #8 ldr x15, [x6] ldr x15, [x6, #64] ldr x15, [x6, #128] ldr x15, [x6, #192] ldrb w8, [x6, x8, LSL 0] ldrb w11, [x6, x11, LSL 0] ldrb w16, [x6, x16, LSL 0] ldrb w17, [x6, x17, LSL 0] ubfx x9, x13, #0, #8 eor w8, w8, w11, lsl 8 ubfx x11, x13, #40, #8 eor w8, w8, w16, lsl 16 ubfx x16, x12, #16, #8 eor w8, w8, w17, lsl 24 ubfx x17, x12, #56, #8 ldrb w11, [x6, x11, LSL 0] ldrb w17, [x6, x17, LSL 0] ldrb w9, [x6, x9, LSL 0] ldrb w16, [x6, x16, LSL 0] ubfx x10, x13, #32, #8 eor w9, w9, w11, lsl 8 ubfx x11, x12, #8, #8 eor w9, w9, w16, lsl 16 ubfx x16, x12, #48, #8 eor w9, w9, w17, lsl 24 ubfx x17, x13, #24, #8 bfi x8, x9, #32, #32 ldrb w11, [x6, x11, LSL 0] ldrb w17, [x6, x17, LSL 0] ldrb w10, [x6, x10, LSL 0] ldrb w16, [x6, x16, LSL 0] ubfx x15, x13, #56, #8 eor w10, w10, w11, lsl 8 ubfx x11, x12, #0, #8 eor w10, w10, w16, lsl 16 ubfx x16, x12, #40, #8 eor w9, w10, w17, lsl 24 ubfx x17, x13, #16, #8 ldrb w15, [x6, x15, LSL 0] ldrb w16, [x6, x16, LSL 0] ldrb w11, [x6, x11, LSL 0] ldrb w17, [x6, x17, LSL 0] eor w16, w16, w15, lsl 16 ldp x12, x13, [x20] eor w11, w11, w16, lsl 8 eor w11, w11, w17, lsl 16 bfi x9, x11, #32, #32 # XOR in Key Schedule eor x8, x8, x12 eor x9, x9, x13 rev32 x8, x8 rev32 x9, x9 ldp x12, x13, [x5] eor x8, x8, x12 eor x9, x9, x13 str x8, [x1] str x9, [x1, #8] subs x2, x2, #16 add x0, x0, #16 add x1, x1, #16 beq L_AES_CBC_decrypt_end_dec_odd mov x20, x3 ldr x8, [x0] ldr x9, [x0, #8] stp x8, x9, [x5] rev32 x8, x8 rev32 x9, x9 ldp x12, x13, [x20], #16 # Round: 0 - XOR in key schedule eor x8, x8, x12 eor x9, x9, x13 sub w19, w4, #2 L_AES_CBC_decrypt_loop_nr_odd: ubfx x12, x9, #48, #8 ubfx x15, x8, #24, #8 ubfx x16, x9, #8, #8 ubfx x17, x8, #32, #8 ldr x10, [x7] ldr x10, [x7, #64] ldr x10, [x7, #128] ldr x10, [x7, #192] ldr x10, [x7, #256] ldr x10, [x7, #320] ldr x10, [x7, #384] ldr x10, [x7, #448] ldr x10, [x7, #512] ldr x10, [x7, #576] ldr x10, [x7, #640] ldr x10, [x7, #704] ldr x10, [x7, #768] ldr x10, [x7, #832] ldr x10, [x7, #896] ldr x10, [x7, #960] ldr w12, [x7, x12, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x13, x8, #16, #8 eor w12, w12, w15, ror 24 ubfx x15, x8, #56, #8 eor w12, w12, w16, ror 8 ubfx x16, x9, #40, #8 eor w12, w12, w17, ror 16 ubfx x17, x9, #0, #8 ldr w13, [x7, x13, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x14, x8, #48, #8 eor w13, w13, w15, ror 24 ubfx x15, x9, #24, #8 eor w13, w13, w16, ror 8 ubfx x16, x8, #8, #8 eor w13, w13, w17, ror 16 ubfx x17, x9, #32, #8 bfi x12, x13, #32, #32 ldr w14, [x7, x14, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x10, x8, #0, #8 eor w14, w14, w15, ror 24 ubfx x15, x9, #16, #8 eor w14, w14, w16, ror 8 ubfx x16, x9, #56, #8 eor w13, w14, w17, ror 16 ubfx x17, x8, #40, #8 ldr w10, [x7, x10, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w17, [x7, x17, LSL 2] eor w16, w16, w10, ror 24 ldp x8, x9, [x20], #16 eor w15, w15, w17, ror 8 eor w15, w15, w16, ror 24 bfi x13, x15, #32, #32 # XOR in Key Schedule eor x12, x12, x8 eor x13, x13, x9 ubfx x8, x13, #48, #8 ubfx x11, x12, #24, #8 ubfx x16, x13, #8, #8 ubfx x17, x12, #32, #8 ldr x14, [x7] ldr x14, [x7, #64] ldr x14, [x7, #128] ldr x14, [x7, #192] ldr x14, [x7, #256] ldr x14, [x7, #320] ldr x14, [x7, #384] ldr x14, [x7, #448] ldr x14, [x7, #512] ldr x14, [x7, #576] ldr x14, [x7, #640] ldr x14, [x7, #704] ldr x14, [x7, #768] ldr x14, [x7, #832] ldr x14, [x7, #896] ldr x14, [x7, #960] ldr w8, [x7, x8, LSL 2] ldr w11, [x7, x11, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x9, x12, #16, #8 eor w8, w8, w11, ror 24 ubfx x11, x12, #56, #8 eor w8, w8, w16, ror 8 ubfx x16, x13, #40, #8 eor w8, w8, w17, ror 16 ubfx x17, x13, #0, #8 ldr w9, [x7, x9, LSL 2] ldr w11, [x7, x11, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x10, x12, #48, #8 eor w9, w9, w11, ror 24 ubfx x11, x13, #24, #8 eor w9, w9, w16, ror 8 ubfx x16, x12, #8, #8 eor w9, w9, w17, ror 16 ubfx x17, x13, #32, #8 bfi x8, x9, #32, #32 ldr w10, [x7, x10, LSL 2] ldr w11, [x7, x11, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x14, x12, #0, #8 eor w10, w10, w11, ror 24 ubfx x11, x13, #16, #8 eor w10, w10, w16, ror 8 ubfx x16, x13, #56, #8 eor w9, w10, w17, ror 16 ubfx x17, x12, #40, #8 ldr w14, [x7, x14, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w11, [x7, x11, LSL 2] ldr w17, [x7, x17, LSL 2] eor w16, w16, w14, ror 24 ldp x12, x13, [x20], #16 eor w11, w11, w17, ror 8 eor w11, w11, w16, ror 24 bfi x9, x11, #32, #32 # XOR in Key Schedule eor x8, x8, x12 eor x9, x9, x13 subs w19, w19, #2 bne L_AES_CBC_decrypt_loop_nr_odd ubfx x12, x9, #48, #8 ubfx x15, x8, #24, #8 ubfx x16, x9, #8, #8 ubfx x17, x8, #32, #8 ldr x10, [x7] ldr x10, [x7, #64] ldr x10, [x7, #128] ldr x10, [x7, #192] ldr x10, [x7, #256] ldr x10, [x7, #320] ldr x10, [x7, #384] ldr x10, [x7, #448] ldr x10, [x7, #512] ldr x10, [x7, #576] ldr x10, [x7, #640] ldr x10, [x7, #704] ldr x10, [x7, #768] ldr x10, [x7, #832] ldr x10, [x7, #896] ldr x10, [x7, #960] ldr w12, [x7, x12, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x13, x8, #16, #8 eor w12, w12, w15, ror 24 ubfx x15, x8, #56, #8 eor w12, w12, w16, ror 8 ubfx x16, x9, #40, #8 eor w12, w12, w17, ror 16 ubfx x17, x9, #0, #8 ldr w13, [x7, x13, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x14, x8, #48, #8 eor w13, w13, w15, ror 24 ubfx x15, x9, #24, #8 eor w13, w13, w16, ror 8 ubfx x16, x8, #8, #8 eor w13, w13, w17, ror 16 ubfx x17, x9, #32, #8 bfi x12, x13, #32, #32 ldr w14, [x7, x14, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w17, [x7, x17, LSL 2] ubfx x10, x8, #0, #8 eor w14, w14, w15, ror 24 ubfx x15, x9, #16, #8 eor w14, w14, w16, ror 8 ubfx x16, x9, #56, #8 eor w13, w14, w17, ror 16 ubfx x17, x8, #40, #8 ldr w10, [x7, x10, LSL 2] ldr w16, [x7, x16, LSL 2] ldr w15, [x7, x15, LSL 2] ldr w17, [x7, x17, LSL 2] eor w16, w16, w10, ror 24 ldp x8, x9, [x20], #16 eor w15, w15, w17, ror 8 eor w15, w15, w16, ror 24 bfi x13, x15, #32, #32 # XOR in Key Schedule eor x12, x12, x8 eor x13, x13, x9 ubfx x8, x12, #32, #8 ubfx x11, x13, #8, #8 ubfx x16, x13, #48, #8 ubfx x17, x12, #24, #8 ldr x15, [x6] ldr x15, [x6, #64] ldr x15, [x6, #128] ldr x15, [x6, #192] ldrb w8, [x6, x8, LSL 0] ldrb w11, [x6, x11, LSL 0] ldrb w16, [x6, x16, LSL 0] ldrb w17, [x6, x17, LSL 0] ubfx x9, x13, #0, #8 eor w8, w8, w11, lsl 8 ubfx x11, x13, #40, #8 eor w8, w8, w16, lsl 16 ubfx x16, x12, #16, #8 eor w8, w8, w17, lsl 24 ubfx x17, x12, #56, #8 ldrb w11, [x6, x11, LSL 0] ldrb w17, [x6, x17, LSL 0] ldrb w9, [x6, x9, LSL 0] ldrb w16, [x6, x16, LSL 0] ubfx x10, x13, #32, #8 eor w9, w9, w11, lsl 8 ubfx x11, x12, #8, #8 eor w9, w9, w16, lsl 16 ubfx x16, x12, #48, #8 eor w9, w9, w17, lsl 24 ubfx x17, x13, #24, #8 bfi x8, x9, #32, #32 ldrb w11, [x6, x11, LSL 0] ldrb w17, [x6, x17, LSL 0] ldrb w10, [x6, x10, LSL 0] ldrb w16, [x6, x16, LSL 0] ubfx x15, x13, #56, #8 eor w10, w10, w11, lsl 8 ubfx x11, x12, #0, #8 eor w10, w10, w16, lsl 16 ubfx x16, x12, #40, #8 eor w9, w10, w17, lsl 24 ubfx x17, x13, #16, #8 ldrb w15, [x6, x15, LSL 0] ldrb w16, [x6, x16, LSL 0] ldrb w11, [x6, x11, LSL 0] ldrb w17, [x6, x17, LSL 0] eor w16, w16, w15, lsl 16 ldp x12, x13, [x20] eor w11, w11, w16, lsl 8 eor w11, w11, w17, lsl 16 bfi x9, x11, #32, #32 # XOR in Key Schedule eor x8, x8, x12 eor x9, x9, x13 rev32 x8, x8 rev32 x9, x9 ldnp x12, x13, [x5, #16] eor x8, x8, x12 eor x9, x9, x13 str x8, [x1] str x9, [x1, #8] subs x2, x2, #16 add x0, x0, #16 add x1, x1, #16 bne L_AES_CBC_decrypt_loop_block b L_AES_CBC_decrypt_end_dec L_AES_CBC_decrypt_end_dec_odd: ldnp x12, x13, [x5, #16] stp x12, x13, [x5] L_AES_CBC_decrypt_end_dec: ldp x17, x19, [x29, #24] ldr x20, [x29, #40] ldp x29, x30, [sp], #48 ret #ifndef __APPLE__ .size AES_CBC_decrypt,.-AES_CBC_decrypt #endif /* __APPLE__ */ #endif /* HAVE_AES_CBC */ #endif /* WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER || HAVE_AES_CBC * HAVE_AES_ECB */ #endif /* HAVE_AES_DECRYPT */ #ifdef HAVE_AESGCM #ifndef __APPLE__ .text .section .rodata .type L_GCM_gmult_len_r, %object .size L_GCM_gmult_len_r, 128 #else .section __DATA,__data #endif /* __APPLE__ */ # 8-byte aligned, 64-bit aligned #ifndef __APPLE__ .align 3 #else .p2align 3 #endif /* __APPLE__ */ L_GCM_gmult_len_r: .long 0x00000000,0x1c200000,0x38400000,0x24600000 .long 0x70800000,0x6ca00000,0x48c00000,0x54e00000 .long 0xe1000000,0xfd200000,0xd9400000,0xc5600000 .long 0x91800000,0x8da00000,0xa9c00000,0xb5e00000 .long 0x00000000,0x01c20000,0x03840000,0x02460000 .long 0x07080000,0x06ca0000,0x048c0000,0x054e0000 .long 0x0e100000,0x0fd20000,0x0d940000,0x0c560000 .long 0x09180000,0x08da0000,0x0a9c0000,0x0b5e0000 #ifndef __APPLE__ .text .globl GCM_gmult_len .type GCM_gmult_len,@function .align 2 GCM_gmult_len: #else .section __TEXT,__text .globl _GCM_gmult_len .p2align 2 _GCM_gmult_len: #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x10, L_GCM_gmult_len_r add x10, x10, :lo12:L_GCM_gmult_len_r #else adrp x10, L_GCM_gmult_len_r@PAGE add x10, x10, L_GCM_gmult_len_r@PAGEOFF #endif /* __APPLE__ */ L_GCM_gmult_len_start_block: ldp x4, x5, [x0] ldp x6, x7, [x2] eor x4, x4, x6 eor x5, x5, x7 ubfx x12, x5, #56, #4 add x12, x1, x12, lsl 4 ldp x8, x9, [x12] ubfx x12, x5, #60, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfx x12, x5, #48, #4 add x12, x1, x12, lsl 4 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x12, x5, #52, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfx x12, x5, #40, #4 add x12, x1, x12, lsl 4 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x12, x5, #44, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfx x12, x5, #32, #4 add x12, x1, x12, lsl 4 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x12, x5, #36, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfx x12, x5, #24, #4 add x12, x1, x12, lsl 4 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x12, x5, #28, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfx x12, x5, #16, #4 add x12, x1, x12, lsl 4 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x12, x5, #20, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfx x12, x5, #8, #4 add x12, x1, x12, lsl 4 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x12, x5, #12, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfx x12, x5, #0, #4 add x12, x1, x12, lsl 4 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x12, x5, #4, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfx x12, x4, #56, #4 add x12, x1, x12, lsl 4 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x12, x4, #60, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfx x12, x4, #48, #4 add x12, x1, x12, lsl 4 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x12, x4, #52, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfx x12, x4, #40, #4 add x12, x1, x12, lsl 4 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x12, x4, #44, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfx x12, x4, #32, #4 add x12, x1, x12, lsl 4 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x12, x4, #36, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfx x12, x4, #24, #4 add x12, x1, x12, lsl 4 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x12, x4, #28, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfx x12, x4, #16, #4 add x12, x1, x12, lsl 4 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x12, x4, #20, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfx x12, x4, #8, #4 add x12, x1, x12, lsl 4 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x12, x4, #12, #4 mov x11, x9 add x12, x12, #16 lsr x9, x9, #8 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 56 ldp x6, x7, [x12] lsr x8, x8, #8 eor x8, x8, x6 sub x12, x12, #0x100 eor x9, x9, x7 ldr x7, [x12, #8] ubfx w6, w11, #0, #4 eor x11, x11, x7, lsl 4 add w6, w6, #16 ubfx w11, w11, #4, #4 ldr w6, [x10, x6, LSL 2] ldr w7, [x10, x11, LSL 2] eor x8, x8, x6, lsl 32 eor x8, x8, x7, lsl 32 ubfiz x12, x4, #4, #4 add x12, x12, x1 ldp x6, x7, [x12] eor x8, x8, x6 eor x9, x9, x7 ubfx x11, x9, #0, #4 ubfx x12, x4, #4, #4 lsr x9, x9, #4 add x12, x1, x12, lsl 4 orr x9, x9, x8, lsl 60 ldp x6, x7, [x12] lsr x8, x8, #4 eor x8, x8, x6 ldr w6, [x10, x11, LSL 2] eor x9, x9, x7 eor x8, x8, x6, lsl 32 rev x8, x8 rev x9, x9 stp x8, x9, [x0] subs x3, x3, #16 add x2, x2, #16 bne L_GCM_gmult_len_start_block ret #ifndef __APPLE__ .size GCM_gmult_len,.-GCM_gmult_len #endif /* __APPLE__ */ #ifndef __APPLE__ .text .globl AES_GCM_encrypt .type AES_GCM_encrypt,@function .align 2 AES_GCM_encrypt: #else .section __TEXT,__text .globl _AES_GCM_encrypt .p2align 2 _AES_GCM_encrypt: #endif /* __APPLE__ */ stp x29, x30, [sp, #-48]! add x29, sp, #0 stp x17, x19, [x29, #16] stp x20, x21, [x29, #32] #ifndef __APPLE__ adrp x19, L_AES_ARM64_te add x19, x19, :lo12:L_AES_ARM64_te #else adrp x19, L_AES_ARM64_te@PAGE add x19, x19, L_AES_ARM64_te@PAGEOFF #endif /* __APPLE__ */ ldp x16, x17, [x5] rev32 x16, x16 rev32 x17, x17 L_AES_GCM_encrypt_loop_block: mov x21, x3 lsr x9, x17, #32 ldp x10, x11, [x21], #16 add w9, w9, #1 bfi x17, x9, #32, #32 # Round: 0 - XOR in key schedule eor x6, x16, x10 eor x7, x17, x11 sub w20, w4, #2 L_AES_GCM_encrypt_loop_nr: ubfx x10, x6, #48, #8 ubfx x13, x6, #24, #8 ubfx x14, x7, #8, #8 ubfx x15, x7, #32, #8 ldr x8, [x19] ldr x8, [x19, #64] ldr x8, [x19, #128] ldr x8, [x19, #192] ldr x8, [x19, #256] ldr x8, [x19, #320] ldr x8, [x19, #384] ldr x8, [x19, #448] ldr x8, [x19, #512] ldr x8, [x19, #576] ldr x8, [x19, #640] ldr x8, [x19, #704] ldr x8, [x19, #768] ldr x8, [x19, #832] ldr x8, [x19, #896] ldr x8, [x19, #960] ldr w10, [x19, x10, LSL 2] ldr w13, [x19, x13, LSL 2] ldr w14, [x19, x14, LSL 2] ldr w15, [x19, x15, LSL 2] ubfx x11, x7, #16, #8 eor w10, w10, w13, ror 24 ubfx x13, x6, #56, #8 eor w10, w10, w14, ror 8 ubfx x14, x7, #40, #8 eor w10, w10, w15, ror 16 ubfx x15, x6, #0, #8 ldr w11, [x19, x11, LSL 2] ldr w13, [x19, x13, LSL 2] ldr w14, [x19, x14, LSL 2] ldr w15, [x19, x15, LSL 2] ubfx x12, x7, #48, #8 eor w11, w11, w13, ror 24 ubfx x13, x7, #24, #8 eor w11, w11, w14, ror 8 ubfx x14, x6, #8, #8 eor w11, w11, w15, ror 16 ubfx x15, x6, #32, #8 bfi x10, x11, #32, #32 ldr w12, [x19, x12, LSL 2] ldr w13, [x19, x13, LSL 2] ldr w14, [x19, x14, LSL 2] ldr w15, [x19, x15, LSL 2] ubfx x8, x7, #0, #8 eor w12, w12, w13, ror 24 ubfx x13, x6, #16, #8 eor w12, w12, w14, ror 8 ubfx x14, x7, #56, #8 eor w11, w12, w15, ror 16 ubfx x15, x6, #40, #8 ldr w8, [x19, x8, LSL 2] ldr w14, [x19, x14, LSL 2] ldr w13, [x19, x13, LSL 2] ldr w15, [x19, x15, LSL 2] eor w14, w14, w8, ror 24 ldp x6, x7, [x21], #16 eor w13, w13, w14, ror 24 eor w13, w13, w15, ror 8 bfi x11, x13, #32, #32 # XOR in Key Schedule eor x10, x10, x6 eor x11, x11, x7 ubfx x6, x10, #48, #8 ubfx x9, x10, #24, #8 ubfx x14, x11, #8, #8 ubfx x15, x11, #32, #8 ldr x12, [x19] ldr x12, [x19, #64] ldr x12, [x19, #128] ldr x12, [x19, #192] ldr x12, [x19, #256] ldr x12, [x19, #320] ldr x12, [x19, #384] ldr x12, [x19, #448] ldr x12, [x19, #512] ldr x12, [x19, #576] ldr x12, [x19, #640] ldr x12, [x19, #704] ldr x12, [x19, #768] ldr x12, [x19, #832] ldr x12, [x19, #896] ldr x12, [x19, #960] ldr w6, [x19, x6, LSL 2] ldr w9, [x19, x9, LSL 2] ldr w14, [x19, x14, LSL 2] ldr w15, [x19, x15, LSL 2] ubfx x7, x11, #16, #8 eor w6, w6, w9, ror 24 ubfx x9, x10, #56, #8 eor w6, w6, w14, ror 8 ubfx x14, x11, #40, #8 eor w6, w6, w15, ror 16 ubfx x15, x10, #0, #8 ldr w7, [x19, x7, LSL 2] ldr w9, [x19, x9, LSL 2] ldr w14, [x19, x14, LSL 2] ldr w15, [x19, x15, LSL 2] ubfx x8, x11, #48, #8 eor w7, w7, w9, ror 24 ubfx x9, x11, #24, #8 eor w7, w7, w14, ror 8 ubfx x14, x10, #8, #8 eor w7, w7, w15, ror 16 ubfx x15, x10, #32, #8 bfi x6, x7, #32, #32 ldr w8, [x19, x8, LSL 2] ldr w9, [x19, x9, LSL 2] ldr w14, [x19, x14, LSL 2] ldr w15, [x19, x15, LSL 2] ubfx x12, x11, #0, #8 eor w8, w8, w9, ror 24 ubfx x9, x10, #16, #8 eor w8, w8, w14, ror 8 ubfx x14, x11, #56, #8 eor w7, w8, w15, ror 16 ubfx x15, x10, #40, #8 ldr w12, [x19, x12, LSL 2] ldr w14, [x19, x14, LSL 2] ldr w9, [x19, x9, LSL 2] ldr w15, [x19, x15, LSL 2] eor w14, w14, w12, ror 24 ldp x10, x11, [x21], #16 eor w9, w9, w14, ror 24 eor w9, w9, w15, ror 8 bfi x7, x9, #32, #32 # XOR in Key Schedule eor x6, x6, x10 eor x7, x7, x11 subs w20, w20, #2 bne L_AES_GCM_encrypt_loop_nr ubfx x10, x6, #48, #8 ubfx x13, x6, #24, #8 ubfx x14, x7, #8, #8 ubfx x15, x7, #32, #8 ldr x8, [x19] ldr x8, [x19, #64] ldr x8, [x19, #128] ldr x8, [x19, #192] ldr x8, [x19, #256] ldr x8, [x19, #320] ldr x8, [x19, #384] ldr x8, [x19, #448] ldr x8, [x19, #512] ldr x8, [x19, #576] ldr x8, [x19, #640] ldr x8, [x19, #704] ldr x8, [x19, #768] ldr x8, [x19, #832] ldr x8, [x19, #896] ldr x8, [x19, #960] ldr w10, [x19, x10, LSL 2] ldr w13, [x19, x13, LSL 2] ldr w14, [x19, x14, LSL 2] ldr w15, [x19, x15, LSL 2] ubfx x11, x7, #16, #8 eor w10, w10, w13, ror 24 ubfx x13, x6, #56, #8 eor w10, w10, w14, ror 8 ubfx x14, x7, #40, #8 eor w10, w10, w15, ror 16 ubfx x15, x6, #0, #8 ldr w11, [x19, x11, LSL 2] ldr w13, [x19, x13, LSL 2] ldr w14, [x19, x14, LSL 2] ldr w15, [x19, x15, LSL 2] ubfx x12, x7, #48, #8 eor w11, w11, w13, ror 24 ubfx x13, x7, #24, #8 eor w11, w11, w14, ror 8 ubfx x14, x6, #8, #8 eor w11, w11, w15, ror 16 ubfx x15, x6, #32, #8 bfi x10, x11, #32, #32 ldr w12, [x19, x12, LSL 2] ldr w13, [x19, x13, LSL 2] ldr w14, [x19, x14, LSL 2] ldr w15, [x19, x15, LSL 2] ubfx x8, x7, #0, #8 eor w12, w12, w13, ror 24 ubfx x13, x6, #16, #8 eor w12, w12, w14, ror 8 ubfx x14, x7, #56, #8 eor w11, w12, w15, ror 16 ubfx x15, x6, #40, #8 ldr w8, [x19, x8, LSL 2] ldr w14, [x19, x14, LSL 2] ldr w13, [x19, x13, LSL 2] ldr w15, [x19, x15, LSL 2] eor w14, w14, w8, ror 24 ldp x6, x7, [x21], #16 eor w13, w13, w14, ror 24 eor w13, w13, w15, ror 8 bfi x11, x13, #32, #32 # XOR in Key Schedule eor x10, x10, x6 eor x11, x11, x7 ubfx x6, x11, #32, #8 ubfx x9, x11, #8, #8 ubfx x14, x10, #48, #8 ubfx x15, x10, #24, #8 lsl w6, w6, #2 lsl w9, w9, #2 lsl w14, w14, #2 lsl w15, w15, #2 ldr x13, [x19] ldr x13, [x19, #64] ldr x13, [x19, #128] ldr x13, [x19, #192] ldr x13, [x19, #256] ldr x13, [x19, #320] ldr x13, [x19, #384] ldr x13, [x19, #448] ldr x13, [x19, #512] ldr x13, [x19, #576] ldr x13, [x19, #640] ldr x13, [x19, #704] ldr x13, [x19, #768] ldr x13, [x19, #832] ldr x13, [x19, #896] ldr x13, [x19, #960] ldrb w6, [x19, x6, LSL 0] ldrb w9, [x19, x9, LSL 0] ldrb w14, [x19, x14, LSL 0] ldrb w15, [x19, x15, LSL 0] ubfx x7, x10, #0, #8 eor w6, w6, w9, lsl 8 ubfx x9, x11, #40, #8 eor w6, w6, w14, lsl 16 ubfx x14, x11, #16, #8 eor w6, w6, w15, lsl 24 ubfx x15, x10, #56, #8 lsl w7, w7, #2 lsl w9, w9, #2 lsl w14, w14, #2 lsl w15, w15, #2 ldrb w7, [x19, x7, LSL 0] ldrb w9, [x19, x9, LSL 0] ldrb w14, [x19, x14, LSL 0] ldrb w15, [x19, x15, LSL 0] ubfx x8, x10, #32, #8 eor w7, w7, w9, lsl 8 ubfx x9, x10, #8, #8 eor w7, w7, w14, lsl 16 ubfx x14, x11, #48, #8 eor w7, w7, w15, lsl 24 ubfx x15, x11, #24, #8 bfi x6, x7, #32, #32 lsl w8, w8, #2 lsl w9, w9, #2 lsl w14, w14, #2 lsl w15, w15, #2 ldrb w8, [x19, x8, LSL 0] ldrb w9, [x19, x9, LSL 0] ldrb w14, [x19, x14, LSL 0] ldrb w15, [x19, x15, LSL 0] ubfx x13, x11, #56, #8 eor w8, w8, w9, lsl 8 ubfx x9, x11, #0, #8 eor w8, w8, w14, lsl 16 ubfx x14, x10, #40, #8 eor w7, w8, w15, lsl 24 ubfx x15, x10, #16, #8 lsl w13, w13, #2 lsl w9, w9, #2 lsl w14, w14, #2 lsl w15, w15, #2 ldrb w13, [x19, x13, LSL 0] ldrb w9, [x19, x9, LSL 0] ldrb w14, [x19, x14, LSL 0] ldrb w15, [x19, x15, LSL 0] eor w14, w14, w13, lsl 16 ldp x10, x11, [x21] eor w9, w9, w14, lsl 8 eor w9, w9, w15, lsl 16 bfi x7, x9, #32, #32 # XOR in Key Schedule eor x6, x6, x10 eor x7, x7, x11 rev32 x6, x6 rev32 x7, x7 ldr x10, [x0] ldr x11, [x0, #8] eor x6, x6, x10 eor x7, x7, x11 str x6, [x1] str x7, [x1, #8] subs x2, x2, #16 add x0, x0, #16 add x1, x1, #16 bne L_AES_GCM_encrypt_loop_block rev32 x16, x16 rev32 x17, x17 stp x16, x17, [x5] ldp x17, x19, [x29, #16] ldp x20, x21, [x29, #32] ldp x29, x30, [sp], #48 ret #ifndef __APPLE__ .size AES_GCM_encrypt,.-AES_GCM_encrypt #endif /* __APPLE__ */ #endif /* HAVE_AESGCM */ #ifdef WOLFSSL_AES_XTS #ifndef __APPLE__ .text .globl AES_XTS_encrypt .type AES_XTS_encrypt,@function .align 2 AES_XTS_encrypt: #else .section __TEXT,__text .globl _AES_XTS_encrypt .p2align 2 _AES_XTS_encrypt: #endif /* __APPLE__ */ stp x29, x30, [sp, #-96]! add x29, sp, #0 stp x17, x19, [x29, #24] stp x20, x21, [x29, #40] stp x22, x23, [x29, #56] stp x24, x25, [x29, #72] str x26, [x29, #88] #ifndef __APPLE__ adrp x8, L_AES_ARM64_te add x8, x8, :lo12:L_AES_ARM64_te #else adrp x8, L_AES_ARM64_te@PAGE add x8, x8, L_AES_ARM64_te@PAGEOFF #endif /* __APPLE__ */ mov x9, #0x87 mov x26, x5 ldp x21, x22, [x3] ldp x14, x15, [x26], #16 rev32 x21, x21 rev32 x22, x22 # Round: 0 - XOR in key schedule eor x21, x21, x14 eor x22, x22, x15 sub w25, w7, #2 L_AES_XTS_encrypt_loop_nr_tweak: ubfx x14, x21, #48, #8 ubfx x17, x21, #24, #8 ubfx x19, x22, #8, #8 ubfx x20, x22, #32, #8 ldr x23, [x8] ldr x23, [x8, #64] ldr x23, [x8, #128] ldr x23, [x8, #192] ldr x23, [x8, #256] ldr x23, [x8, #320] ldr x23, [x8, #384] ldr x23, [x8, #448] ldr x23, [x8, #512] ldr x23, [x8, #576] ldr x23, [x8, #640] ldr x23, [x8, #704] ldr x23, [x8, #768] ldr x23, [x8, #832] ldr x23, [x8, #896] ldr x23, [x8, #960] ldr w14, [x8, x14, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x15, x22, #16, #8 eor w14, w14, w17, ror 24 ubfx x17, x21, #56, #8 eor w14, w14, w19, ror 8 ubfx x19, x22, #40, #8 eor w14, w14, w20, ror 16 ubfx x20, x21, #0, #8 ldr w15, [x8, x15, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x16, x22, #48, #8 eor w15, w15, w17, ror 24 ubfx x17, x22, #24, #8 eor w15, w15, w19, ror 8 ubfx x19, x21, #8, #8 eor w15, w15, w20, ror 16 ubfx x20, x21, #32, #8 bfi x14, x15, #32, #32 ldr w16, [x8, x16, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x23, x22, #0, #8 eor w16, w16, w17, ror 24 ubfx x17, x21, #16, #8 eor w16, w16, w19, ror 8 ubfx x19, x22, #56, #8 eor w15, w16, w20, ror 16 ubfx x20, x21, #40, #8 ldr w23, [x8, x23, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w20, [x8, x20, LSL 2] eor w19, w19, w23, ror 24 ldp x21, x22, [x26], #16 eor w17, w17, w19, ror 24 eor w17, w17, w20, ror 8 bfi x15, x17, #32, #32 # XOR in Key Schedule eor x14, x14, x21 eor x15, x15, x22 ubfx x21, x14, #48, #8 ubfx x24, x14, #24, #8 ubfx x19, x15, #8, #8 ubfx x20, x15, #32, #8 ldr x16, [x8] ldr x16, [x8, #64] ldr x16, [x8, #128] ldr x16, [x8, #192] ldr x16, [x8, #256] ldr x16, [x8, #320] ldr x16, [x8, #384] ldr x16, [x8, #448] ldr x16, [x8, #512] ldr x16, [x8, #576] ldr x16, [x8, #640] ldr x16, [x8, #704] ldr x16, [x8, #768] ldr x16, [x8, #832] ldr x16, [x8, #896] ldr x16, [x8, #960] ldr w21, [x8, x21, LSL 2] ldr w24, [x8, x24, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x22, x15, #16, #8 eor w21, w21, w24, ror 24 ubfx x24, x14, #56, #8 eor w21, w21, w19, ror 8 ubfx x19, x15, #40, #8 eor w21, w21, w20, ror 16 ubfx x20, x14, #0, #8 ldr w22, [x8, x22, LSL 2] ldr w24, [x8, x24, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x23, x15, #48, #8 eor w22, w22, w24, ror 24 ubfx x24, x15, #24, #8 eor w22, w22, w19, ror 8 ubfx x19, x14, #8, #8 eor w22, w22, w20, ror 16 ubfx x20, x14, #32, #8 bfi x21, x22, #32, #32 ldr w23, [x8, x23, LSL 2] ldr w24, [x8, x24, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x16, x15, #0, #8 eor w23, w23, w24, ror 24 ubfx x24, x14, #16, #8 eor w23, w23, w19, ror 8 ubfx x19, x15, #56, #8 eor w22, w23, w20, ror 16 ubfx x20, x14, #40, #8 ldr w16, [x8, x16, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w24, [x8, x24, LSL 2] ldr w20, [x8, x20, LSL 2] eor w19, w19, w16, ror 24 ldp x14, x15, [x26], #16 eor w24, w24, w19, ror 24 eor w24, w24, w20, ror 8 bfi x22, x24, #32, #32 # XOR in Key Schedule eor x21, x21, x14 eor x22, x22, x15 subs w25, w25, #2 bne L_AES_XTS_encrypt_loop_nr_tweak ubfx x14, x21, #48, #8 ubfx x17, x21, #24, #8 ubfx x19, x22, #8, #8 ubfx x20, x22, #32, #8 ldr x23, [x8] ldr x23, [x8, #64] ldr x23, [x8, #128] ldr x23, [x8, #192] ldr x23, [x8, #256] ldr x23, [x8, #320] ldr x23, [x8, #384] ldr x23, [x8, #448] ldr x23, [x8, #512] ldr x23, [x8, #576] ldr x23, [x8, #640] ldr x23, [x8, #704] ldr x23, [x8, #768] ldr x23, [x8, #832] ldr x23, [x8, #896] ldr x23, [x8, #960] ldr w14, [x8, x14, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x15, x22, #16, #8 eor w14, w14, w17, ror 24 ubfx x17, x21, #56, #8 eor w14, w14, w19, ror 8 ubfx x19, x22, #40, #8 eor w14, w14, w20, ror 16 ubfx x20, x21, #0, #8 ldr w15, [x8, x15, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x16, x22, #48, #8 eor w15, w15, w17, ror 24 ubfx x17, x22, #24, #8 eor w15, w15, w19, ror 8 ubfx x19, x21, #8, #8 eor w15, w15, w20, ror 16 ubfx x20, x21, #32, #8 bfi x14, x15, #32, #32 ldr w16, [x8, x16, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x23, x22, #0, #8 eor w16, w16, w17, ror 24 ubfx x17, x21, #16, #8 eor w16, w16, w19, ror 8 ubfx x19, x22, #56, #8 eor w15, w16, w20, ror 16 ubfx x20, x21, #40, #8 ldr w23, [x8, x23, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w20, [x8, x20, LSL 2] eor w19, w19, w23, ror 24 ldp x21, x22, [x26], #16 eor w17, w17, w19, ror 24 eor w17, w17, w20, ror 8 bfi x15, x17, #32, #32 # XOR in Key Schedule eor x14, x14, x21 eor x15, x15, x22 ubfx x21, x15, #32, #8 ubfx x24, x15, #8, #8 ubfx x19, x14, #48, #8 ubfx x20, x14, #24, #8 lsl w21, w21, #2 lsl w24, w24, #2 lsl w19, w19, #2 lsl w20, w20, #2 ldr x17, [x8] ldr x17, [x8, #64] ldr x17, [x8, #128] ldr x17, [x8, #192] ldr x17, [x8, #256] ldr x17, [x8, #320] ldr x17, [x8, #384] ldr x17, [x8, #448] ldr x17, [x8, #512] ldr x17, [x8, #576] ldr x17, [x8, #640] ldr x17, [x8, #704] ldr x17, [x8, #768] ldr x17, [x8, #832] ldr x17, [x8, #896] ldr x17, [x8, #960] ldrb w21, [x8, x21, LSL 0] ldrb w24, [x8, x24, LSL 0] ldrb w19, [x8, x19, LSL 0] ldrb w20, [x8, x20, LSL 0] ubfx x22, x14, #0, #8 eor w21, w21, w24, lsl 8 ubfx x24, x15, #40, #8 eor w21, w21, w19, lsl 16 ubfx x19, x15, #16, #8 eor w21, w21, w20, lsl 24 ubfx x20, x14, #56, #8 lsl w22, w22, #2 lsl w24, w24, #2 lsl w19, w19, #2 lsl w20, w20, #2 ldrb w22, [x8, x22, LSL 0] ldrb w24, [x8, x24, LSL 0] ldrb w19, [x8, x19, LSL 0] ldrb w20, [x8, x20, LSL 0] ubfx x23, x14, #32, #8 eor w22, w22, w24, lsl 8 ubfx x24, x14, #8, #8 eor w22, w22, w19, lsl 16 ubfx x19, x15, #48, #8 eor w22, w22, w20, lsl 24 ubfx x20, x15, #24, #8 bfi x21, x22, #32, #32 lsl w23, w23, #2 lsl w24, w24, #2 lsl w19, w19, #2 lsl w20, w20, #2 ldrb w23, [x8, x23, LSL 0] ldrb w24, [x8, x24, LSL 0] ldrb w19, [x8, x19, LSL 0] ldrb w20, [x8, x20, LSL 0] ubfx x17, x15, #56, #8 eor w23, w23, w24, lsl 8 ubfx x24, x15, #0, #8 eor w23, w23, w19, lsl 16 ubfx x19, x14, #40, #8 eor w22, w23, w20, lsl 24 ubfx x20, x14, #16, #8 lsl w17, w17, #2 lsl w24, w24, #2 lsl w19, w19, #2 lsl w20, w20, #2 ldrb w17, [x8, x17, LSL 0] ldrb w24, [x8, x24, LSL 0] ldrb w19, [x8, x19, LSL 0] ldrb w20, [x8, x20, LSL 0] eor w19, w19, w17, lsl 16 ldp x14, x15, [x26] eor w24, w24, w19, lsl 8 eor w24, w24, w20, lsl 16 bfi x22, x24, #32, #32 # XOR in Key Schedule eor x21, x21, x14 eor x22, x22, x15 rev32 x21, x21 rev32 x22, x22 L_AES_XTS_encrypt_loop_block: mov x26, x4 ldp x10, x11, [x0] ldp x14, x15, [x26], #16 eor x10, x10, x21 eor x11, x11, x22 rev32 x10, x10 rev32 x11, x11 # Round: 0 - XOR in key schedule eor x10, x10, x14 eor x11, x11, x15 sub w25, w7, #2 L_AES_XTS_encrypt_loop_nr: ubfx x14, x10, #48, #8 ubfx x17, x10, #24, #8 ubfx x19, x11, #8, #8 ubfx x20, x11, #32, #8 ldr x12, [x8] ldr x12, [x8, #64] ldr x12, [x8, #128] ldr x12, [x8, #192] ldr x12, [x8, #256] ldr x12, [x8, #320] ldr x12, [x8, #384] ldr x12, [x8, #448] ldr x12, [x8, #512] ldr x12, [x8, #576] ldr x12, [x8, #640] ldr x12, [x8, #704] ldr x12, [x8, #768] ldr x12, [x8, #832] ldr x12, [x8, #896] ldr x12, [x8, #960] ldr w14, [x8, x14, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x15, x11, #16, #8 eor w14, w14, w17, ror 24 ubfx x17, x10, #56, #8 eor w14, w14, w19, ror 8 ubfx x19, x11, #40, #8 eor w14, w14, w20, ror 16 ubfx x20, x10, #0, #8 ldr w15, [x8, x15, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x16, x11, #48, #8 eor w15, w15, w17, ror 24 ubfx x17, x11, #24, #8 eor w15, w15, w19, ror 8 ubfx x19, x10, #8, #8 eor w15, w15, w20, ror 16 ubfx x20, x10, #32, #8 bfi x14, x15, #32, #32 ldr w16, [x8, x16, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x12, x11, #0, #8 eor w16, w16, w17, ror 24 ubfx x17, x10, #16, #8 eor w16, w16, w19, ror 8 ubfx x19, x11, #56, #8 eor w15, w16, w20, ror 16 ubfx x20, x10, #40, #8 ldr w12, [x8, x12, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w20, [x8, x20, LSL 2] eor w19, w19, w12, ror 24 ldp x10, x11, [x26], #16 eor w17, w17, w19, ror 24 eor w17, w17, w20, ror 8 bfi x15, x17, #32, #32 # XOR in Key Schedule eor x14, x14, x10 eor x15, x15, x11 ubfx x10, x14, #48, #8 ubfx x13, x14, #24, #8 ubfx x19, x15, #8, #8 ubfx x20, x15, #32, #8 ldr x16, [x8] ldr x16, [x8, #64] ldr x16, [x8, #128] ldr x16, [x8, #192] ldr x16, [x8, #256] ldr x16, [x8, #320] ldr x16, [x8, #384] ldr x16, [x8, #448] ldr x16, [x8, #512] ldr x16, [x8, #576] ldr x16, [x8, #640] ldr x16, [x8, #704] ldr x16, [x8, #768] ldr x16, [x8, #832] ldr x16, [x8, #896] ldr x16, [x8, #960] ldr w10, [x8, x10, LSL 2] ldr w13, [x8, x13, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x11, x15, #16, #8 eor w10, w10, w13, ror 24 ubfx x13, x14, #56, #8 eor w10, w10, w19, ror 8 ubfx x19, x15, #40, #8 eor w10, w10, w20, ror 16 ubfx x20, x14, #0, #8 ldr w11, [x8, x11, LSL 2] ldr w13, [x8, x13, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x12, x15, #48, #8 eor w11, w11, w13, ror 24 ubfx x13, x15, #24, #8 eor w11, w11, w19, ror 8 ubfx x19, x14, #8, #8 eor w11, w11, w20, ror 16 ubfx x20, x14, #32, #8 bfi x10, x11, #32, #32 ldr w12, [x8, x12, LSL 2] ldr w13, [x8, x13, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x16, x15, #0, #8 eor w12, w12, w13, ror 24 ubfx x13, x14, #16, #8 eor w12, w12, w19, ror 8 ubfx x19, x15, #56, #8 eor w11, w12, w20, ror 16 ubfx x20, x14, #40, #8 ldr w16, [x8, x16, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w13, [x8, x13, LSL 2] ldr w20, [x8, x20, LSL 2] eor w19, w19, w16, ror 24 ldp x14, x15, [x26], #16 eor w13, w13, w19, ror 24 eor w13, w13, w20, ror 8 bfi x11, x13, #32, #32 # XOR in Key Schedule eor x10, x10, x14 eor x11, x11, x15 subs w25, w25, #2 bne L_AES_XTS_encrypt_loop_nr ubfx x14, x10, #48, #8 ubfx x17, x10, #24, #8 ubfx x19, x11, #8, #8 ubfx x20, x11, #32, #8 ldr x12, [x8] ldr x12, [x8, #64] ldr x12, [x8, #128] ldr x12, [x8, #192] ldr x12, [x8, #256] ldr x12, [x8, #320] ldr x12, [x8, #384] ldr x12, [x8, #448] ldr x12, [x8, #512] ldr x12, [x8, #576] ldr x12, [x8, #640] ldr x12, [x8, #704] ldr x12, [x8, #768] ldr x12, [x8, #832] ldr x12, [x8, #896] ldr x12, [x8, #960] ldr w14, [x8, x14, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x15, x11, #16, #8 eor w14, w14, w17, ror 24 ubfx x17, x10, #56, #8 eor w14, w14, w19, ror 8 ubfx x19, x11, #40, #8 eor w14, w14, w20, ror 16 ubfx x20, x10, #0, #8 ldr w15, [x8, x15, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x16, x11, #48, #8 eor w15, w15, w17, ror 24 ubfx x17, x11, #24, #8 eor w15, w15, w19, ror 8 ubfx x19, x10, #8, #8 eor w15, w15, w20, ror 16 ubfx x20, x10, #32, #8 bfi x14, x15, #32, #32 ldr w16, [x8, x16, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x12, x11, #0, #8 eor w16, w16, w17, ror 24 ubfx x17, x10, #16, #8 eor w16, w16, w19, ror 8 ubfx x19, x11, #56, #8 eor w15, w16, w20, ror 16 ubfx x20, x10, #40, #8 ldr w12, [x8, x12, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w20, [x8, x20, LSL 2] eor w19, w19, w12, ror 24 ldp x10, x11, [x26], #16 eor w17, w17, w19, ror 24 eor w17, w17, w20, ror 8 bfi x15, x17, #32, #32 # XOR in Key Schedule eor x14, x14, x10 eor x15, x15, x11 ubfx x10, x15, #32, #8 ubfx x13, x15, #8, #8 ubfx x19, x14, #48, #8 ubfx x20, x14, #24, #8 lsl w10, w10, #2 lsl w13, w13, #2 lsl w19, w19, #2 lsl w20, w20, #2 ldr x17, [x8] ldr x17, [x8, #64] ldr x17, [x8, #128] ldr x17, [x8, #192] ldr x17, [x8, #256] ldr x17, [x8, #320] ldr x17, [x8, #384] ldr x17, [x8, #448] ldr x17, [x8, #512] ldr x17, [x8, #576] ldr x17, [x8, #640] ldr x17, [x8, #704] ldr x17, [x8, #768] ldr x17, [x8, #832] ldr x17, [x8, #896] ldr x17, [x8, #960] ldrb w10, [x8, x10, LSL 0] ldrb w13, [x8, x13, LSL 0] ldrb w19, [x8, x19, LSL 0] ldrb w20, [x8, x20, LSL 0] ubfx x11, x14, #0, #8 eor w10, w10, w13, lsl 8 ubfx x13, x15, #40, #8 eor w10, w10, w19, lsl 16 ubfx x19, x15, #16, #8 eor w10, w10, w20, lsl 24 ubfx x20, x14, #56, #8 lsl w11, w11, #2 lsl w13, w13, #2 lsl w19, w19, #2 lsl w20, w20, #2 ldrb w11, [x8, x11, LSL 0] ldrb w13, [x8, x13, LSL 0] ldrb w19, [x8, x19, LSL 0] ldrb w20, [x8, x20, LSL 0] ubfx x12, x14, #32, #8 eor w11, w11, w13, lsl 8 ubfx x13, x14, #8, #8 eor w11, w11, w19, lsl 16 ubfx x19, x15, #48, #8 eor w11, w11, w20, lsl 24 ubfx x20, x15, #24, #8 bfi x10, x11, #32, #32 lsl w12, w12, #2 lsl w13, w13, #2 lsl w19, w19, #2 lsl w20, w20, #2 ldrb w12, [x8, x12, LSL 0] ldrb w13, [x8, x13, LSL 0] ldrb w19, [x8, x19, LSL 0] ldrb w20, [x8, x20, LSL 0] ubfx x17, x15, #56, #8 eor w12, w12, w13, lsl 8 ubfx x13, x15, #0, #8 eor w12, w12, w19, lsl 16 ubfx x19, x14, #40, #8 eor w11, w12, w20, lsl 24 ubfx x20, x14, #16, #8 lsl w17, w17, #2 lsl w13, w13, #2 lsl w19, w19, #2 lsl w20, w20, #2 ldrb w17, [x8, x17, LSL 0] ldrb w13, [x8, x13, LSL 0] ldrb w19, [x8, x19, LSL 0] ldrb w20, [x8, x20, LSL 0] eor w19, w19, w17, lsl 16 ldp x14, x15, [x26] eor w13, w13, w19, lsl 8 eor w13, w13, w20, lsl 16 bfi x11, x13, #32, #32 # XOR in Key Schedule eor x10, x10, x14 eor x11, x11, x15 rev32 x10, x10 rev32 x11, x11 eor x10, x10, x21 eor x11, x11, x22 stp x10, x11, [x1] and x19, x9, x22, asr 63 extr x22, x22, x21, #63 eor x21, x19, x21, lsl 1 sub w2, w2, #16 add x0, x0, #16 add x1, x1, #16 cmp w2, #16 bge L_AES_XTS_encrypt_loop_block cbz w2, L_AES_XTS_encrypt_done_data mov x26, x4 sub x1, x1, #16 ldp x10, x11, [x1], #16 stp x10, x11, [x6] mov w14, w2 L_AES_XTS_encrypt_start_byte: ldrb w19, [x6] ldrb w20, [x0], #1 strb w19, [x1], #1 strb w20, [x6], #1 subs w14, w14, #1 bgt L_AES_XTS_encrypt_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 ldp x10, x11, [x6] ldp x14, x15, [x26], #16 eor x10, x10, x21 eor x11, x11, x22 rev32 x10, x10 rev32 x11, x11 # Round: 0 - XOR in key schedule eor x10, x10, x14 eor x11, x11, x15 sub w25, w7, #2 L_AES_XTS_encrypt_loop_nr_partial: ubfx x14, x10, #48, #8 ubfx x17, x10, #24, #8 ubfx x19, x11, #8, #8 ubfx x20, x11, #32, #8 ldr x12, [x8] ldr x12, [x8, #64] ldr x12, [x8, #128] ldr x12, [x8, #192] ldr x12, [x8, #256] ldr x12, [x8, #320] ldr x12, [x8, #384] ldr x12, [x8, #448] ldr x12, [x8, #512] ldr x12, [x8, #576] ldr x12, [x8, #640] ldr x12, [x8, #704] ldr x12, [x8, #768] ldr x12, [x8, #832] ldr x12, [x8, #896] ldr x12, [x8, #960] ldr w14, [x8, x14, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x15, x11, #16, #8 eor w14, w14, w17, ror 24 ubfx x17, x10, #56, #8 eor w14, w14, w19, ror 8 ubfx x19, x11, #40, #8 eor w14, w14, w20, ror 16 ubfx x20, x10, #0, #8 ldr w15, [x8, x15, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x16, x11, #48, #8 eor w15, w15, w17, ror 24 ubfx x17, x11, #24, #8 eor w15, w15, w19, ror 8 ubfx x19, x10, #8, #8 eor w15, w15, w20, ror 16 ubfx x20, x10, #32, #8 bfi x14, x15, #32, #32 ldr w16, [x8, x16, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x12, x11, #0, #8 eor w16, w16, w17, ror 24 ubfx x17, x10, #16, #8 eor w16, w16, w19, ror 8 ubfx x19, x11, #56, #8 eor w15, w16, w20, ror 16 ubfx x20, x10, #40, #8 ldr w12, [x8, x12, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w20, [x8, x20, LSL 2] eor w19, w19, w12, ror 24 ldp x10, x11, [x26], #16 eor w17, w17, w19, ror 24 eor w17, w17, w20, ror 8 bfi x15, x17, #32, #32 # XOR in Key Schedule eor x14, x14, x10 eor x15, x15, x11 ubfx x10, x14, #48, #8 ubfx x13, x14, #24, #8 ubfx x19, x15, #8, #8 ubfx x20, x15, #32, #8 ldr x16, [x8] ldr x16, [x8, #64] ldr x16, [x8, #128] ldr x16, [x8, #192] ldr x16, [x8, #256] ldr x16, [x8, #320] ldr x16, [x8, #384] ldr x16, [x8, #448] ldr x16, [x8, #512] ldr x16, [x8, #576] ldr x16, [x8, #640] ldr x16, [x8, #704] ldr x16, [x8, #768] ldr x16, [x8, #832] ldr x16, [x8, #896] ldr x16, [x8, #960] ldr w10, [x8, x10, LSL 2] ldr w13, [x8, x13, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x11, x15, #16, #8 eor w10, w10, w13, ror 24 ubfx x13, x14, #56, #8 eor w10, w10, w19, ror 8 ubfx x19, x15, #40, #8 eor w10, w10, w20, ror 16 ubfx x20, x14, #0, #8 ldr w11, [x8, x11, LSL 2] ldr w13, [x8, x13, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x12, x15, #48, #8 eor w11, w11, w13, ror 24 ubfx x13, x15, #24, #8 eor w11, w11, w19, ror 8 ubfx x19, x14, #8, #8 eor w11, w11, w20, ror 16 ubfx x20, x14, #32, #8 bfi x10, x11, #32, #32 ldr w12, [x8, x12, LSL 2] ldr w13, [x8, x13, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x16, x15, #0, #8 eor w12, w12, w13, ror 24 ubfx x13, x14, #16, #8 eor w12, w12, w19, ror 8 ubfx x19, x15, #56, #8 eor w11, w12, w20, ror 16 ubfx x20, x14, #40, #8 ldr w16, [x8, x16, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w13, [x8, x13, LSL 2] ldr w20, [x8, x20, LSL 2] eor w19, w19, w16, ror 24 ldp x14, x15, [x26], #16 eor w13, w13, w19, ror 24 eor w13, w13, w20, ror 8 bfi x11, x13, #32, #32 # XOR in Key Schedule eor x10, x10, x14 eor x11, x11, x15 subs w25, w25, #2 bne L_AES_XTS_encrypt_loop_nr_partial ubfx x14, x10, #48, #8 ubfx x17, x10, #24, #8 ubfx x19, x11, #8, #8 ubfx x20, x11, #32, #8 ldr x12, [x8] ldr x12, [x8, #64] ldr x12, [x8, #128] ldr x12, [x8, #192] ldr x12, [x8, #256] ldr x12, [x8, #320] ldr x12, [x8, #384] ldr x12, [x8, #448] ldr x12, [x8, #512] ldr x12, [x8, #576] ldr x12, [x8, #640] ldr x12, [x8, #704] ldr x12, [x8, #768] ldr x12, [x8, #832] ldr x12, [x8, #896] ldr x12, [x8, #960] ldr w14, [x8, x14, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x15, x11, #16, #8 eor w14, w14, w17, ror 24 ubfx x17, x10, #56, #8 eor w14, w14, w19, ror 8 ubfx x19, x11, #40, #8 eor w14, w14, w20, ror 16 ubfx x20, x10, #0, #8 ldr w15, [x8, x15, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x16, x11, #48, #8 eor w15, w15, w17, ror 24 ubfx x17, x11, #24, #8 eor w15, w15, w19, ror 8 ubfx x19, x10, #8, #8 eor w15, w15, w20, ror 16 ubfx x20, x10, #32, #8 bfi x14, x15, #32, #32 ldr w16, [x8, x16, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ubfx x12, x11, #0, #8 eor w16, w16, w17, ror 24 ubfx x17, x10, #16, #8 eor w16, w16, w19, ror 8 ubfx x19, x11, #56, #8 eor w15, w16, w20, ror 16 ubfx x20, x10, #40, #8 ldr w12, [x8, x12, LSL 2] ldr w19, [x8, x19, LSL 2] ldr w17, [x8, x17, LSL 2] ldr w20, [x8, x20, LSL 2] eor w19, w19, w12, ror 24 ldp x10, x11, [x26], #16 eor w17, w17, w19, ror 24 eor w17, w17, w20, ror 8 bfi x15, x17, #32, #32 # XOR in Key Schedule eor x14, x14, x10 eor x15, x15, x11 ubfx x10, x15, #32, #8 ubfx x13, x15, #8, #8 ubfx x19, x14, #48, #8 ubfx x20, x14, #24, #8 lsl w10, w10, #2 lsl w13, w13, #2 lsl w19, w19, #2 lsl w20, w20, #2 ldr x17, [x8] ldr x17, [x8, #64] ldr x17, [x8, #128] ldr x17, [x8, #192] ldr x17, [x8, #256] ldr x17, [x8, #320] ldr x17, [x8, #384] ldr x17, [x8, #448] ldr x17, [x8, #512] ldr x17, [x8, #576] ldr x17, [x8, #640] ldr x17, [x8, #704] ldr x17, [x8, #768] ldr x17, [x8, #832] ldr x17, [x8, #896] ldr x17, [x8, #960] ldrb w10, [x8, x10, LSL 0] ldrb w13, [x8, x13, LSL 0] ldrb w19, [x8, x19, LSL 0] ldrb w20, [x8, x20, LSL 0] ubfx x11, x14, #0, #8 eor w10, w10, w13, lsl 8 ubfx x13, x15, #40, #8 eor w10, w10, w19, lsl 16 ubfx x19, x15, #16, #8 eor w10, w10, w20, lsl 24 ubfx x20, x14, #56, #8 lsl w11, w11, #2 lsl w13, w13, #2 lsl w19, w19, #2 lsl w20, w20, #2 ldrb w11, [x8, x11, LSL 0] ldrb w13, [x8, x13, LSL 0] ldrb w19, [x8, x19, LSL 0] ldrb w20, [x8, x20, LSL 0] ubfx x12, x14, #32, #8 eor w11, w11, w13, lsl 8 ubfx x13, x14, #8, #8 eor w11, w11, w19, lsl 16 ubfx x19, x15, #48, #8 eor w11, w11, w20, lsl 24 ubfx x20, x15, #24, #8 bfi x10, x11, #32, #32 lsl w12, w12, #2 lsl w13, w13, #2 lsl w19, w19, #2 lsl w20, w20, #2 ldrb w12, [x8, x12, LSL 0] ldrb w13, [x8, x13, LSL 0] ldrb w19, [x8, x19, LSL 0] ldrb w20, [x8, x20, LSL 0] ubfx x17, x15, #56, #8 eor w12, w12, w13, lsl 8 ubfx x13, x15, #0, #8 eor w12, w12, w19, lsl 16 ubfx x19, x14, #40, #8 eor w11, w12, w20, lsl 24 ubfx x20, x14, #16, #8 lsl w17, w17, #2 lsl w13, w13, #2 lsl w19, w19, #2 lsl w20, w20, #2 ldrb w17, [x8, x17, LSL 0] ldrb w13, [x8, x13, LSL 0] ldrb w19, [x8, x19, LSL 0] ldrb w20, [x8, x20, LSL 0] eor w19, w19, w17, lsl 16 ldp x14, x15, [x26] eor w13, w13, w19, lsl 8 eor w13, w13, w20, lsl 16 bfi x11, x13, #32, #32 # XOR in Key Schedule eor x10, x10, x14 eor x11, x11, x15 rev32 x10, x10 rev32 x11, x11 eor x10, x10, x21 eor x11, x11, x22 stp x10, x11, [x1] L_AES_XTS_encrypt_done_data: ldp x17, x19, [x29, #24] ldp x20, x21, [x29, #40] ldp x22, x23, [x29, #56] ldp x24, x25, [x29, #72] ldr x26, [x29, #88] ldp x29, x30, [sp], #0x60 ret #ifndef __APPLE__ .size AES_XTS_encrypt,.-AES_XTS_encrypt #endif /* __APPLE__ */ #ifdef HAVE_AES_DECRYPT #ifndef __APPLE__ .text .globl AES_XTS_decrypt .type AES_XTS_decrypt,@function .align 2 AES_XTS_decrypt: #else .section __TEXT,__text .globl _AES_XTS_decrypt .p2align 2 _AES_XTS_decrypt: #endif /* __APPLE__ */ stp x29, x30, [sp, #-112]! add x29, sp, #0 stp x17, x19, [x29, #24] stp x20, x21, [x29, #40] stp x22, x23, [x29, #56] stp x24, x25, [x29, #72] stp x26, x27, [x29, #88] str x28, [x29, #104] #ifndef __APPLE__ adrp x8, L_AES_ARM64_td add x8, x8, :lo12:L_AES_ARM64_td #else adrp x8, L_AES_ARM64_td@PAGE add x8, x8, L_AES_ARM64_td@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x9, L_AES_ARM64_td4 add x9, x9, :lo12:L_AES_ARM64_td4 #else adrp x9, L_AES_ARM64_td4@PAGE add x9, x9, L_AES_ARM64_td4@PAGEOFF #endif /* __APPLE__ */ #ifndef __APPLE__ adrp x10, L_AES_ARM64_te add x10, x10, :lo12:L_AES_ARM64_te #else adrp x10, L_AES_ARM64_te@PAGE add x10, x10, L_AES_ARM64_te@PAGEOFF #endif /* __APPLE__ */ ands w11, w2, #15 cset w11, ne lsl w11, w11, #4 sub w2, w2, w11 mov x11, #0x87 mov x28, x5 ldp x23, x24, [x3] ldp x16, x17, [x28], #16 rev32 x23, x23 rev32 x24, x24 # Round: 0 - XOR in key schedule eor x23, x23, x16 eor x24, x24, x17 sub w27, w7, #2 L_AES_XTS_decrypt_loop_nr_tweak: ubfx x16, x23, #48, #8 ubfx x20, x23, #24, #8 ubfx x21, x24, #8, #8 ubfx x22, x24, #32, #8 ldr x25, [x10] ldr x25, [x10, #64] ldr x25, [x10, #128] ldr x25, [x10, #192] ldr x25, [x10, #256] ldr x25, [x10, #320] ldr x25, [x10, #384] ldr x25, [x10, #448] ldr x25, [x10, #512] ldr x25, [x10, #576] ldr x25, [x10, #640] ldr x25, [x10, #704] ldr x25, [x10, #768] ldr x25, [x10, #832] ldr x25, [x10, #896] ldr x25, [x10, #960] ldr w16, [x10, x16, LSL 2] ldr w20, [x10, x20, LSL 2] ldr w21, [x10, x21, LSL 2] ldr w22, [x10, x22, LSL 2] ubfx x17, x24, #16, #8 eor w16, w16, w20, ror 24 ubfx x20, x23, #56, #8 eor w16, w16, w21, ror 8 ubfx x21, x24, #40, #8 eor w16, w16, w22, ror 16 ubfx x22, x23, #0, #8 ldr w17, [x10, x17, LSL 2] ldr w20, [x10, x20, LSL 2] ldr w21, [x10, x21, LSL 2] ldr w22, [x10, x22, LSL 2] ubfx x19, x24, #48, #8 eor w17, w17, w20, ror 24 ubfx x20, x24, #24, #8 eor w17, w17, w21, ror 8 ubfx x21, x23, #8, #8 eor w17, w17, w22, ror 16 ubfx x22, x23, #32, #8 bfi x16, x17, #32, #32 ldr w19, [x10, x19, LSL 2] ldr w20, [x10, x20, LSL 2] ldr w21, [x10, x21, LSL 2] ldr w22, [x10, x22, LSL 2] ubfx x25, x24, #0, #8 eor w19, w19, w20, ror 24 ubfx x20, x23, #16, #8 eor w19, w19, w21, ror 8 ubfx x21, x24, #56, #8 eor w17, w19, w22, ror 16 ubfx x22, x23, #40, #8 ldr w25, [x10, x25, LSL 2] ldr w21, [x10, x21, LSL 2] ldr w20, [x10, x20, LSL 2] ldr w22, [x10, x22, LSL 2] eor w21, w21, w25, ror 24 ldp x23, x24, [x28], #16 eor w20, w20, w21, ror 24 eor w20, w20, w22, ror 8 bfi x17, x20, #32, #32 # XOR in Key Schedule eor x16, x16, x23 eor x17, x17, x24 ubfx x23, x16, #48, #8 ubfx x26, x16, #24, #8 ubfx x21, x17, #8, #8 ubfx x22, x17, #32, #8 ldr x19, [x10] ldr x19, [x10, #64] ldr x19, [x10, #128] ldr x19, [x10, #192] ldr x19, [x10, #256] ldr x19, [x10, #320] ldr x19, [x10, #384] ldr x19, [x10, #448] ldr x19, [x10, #512] ldr x19, [x10, #576] ldr x19, [x10, #640] ldr x19, [x10, #704] ldr x19, [x10, #768] ldr x19, [x10, #832] ldr x19, [x10, #896] ldr x19, [x10, #960] ldr w23, [x10, x23, LSL 2] ldr w26, [x10, x26, LSL 2] ldr w21, [x10, x21, LSL 2] ldr w22, [x10, x22, LSL 2] ubfx x24, x17, #16, #8 eor w23, w23, w26, ror 24 ubfx x26, x16, #56, #8 eor w23, w23, w21, ror 8 ubfx x21, x17, #40, #8 eor w23, w23, w22, ror 16 ubfx x22, x16, #0, #8 ldr w24, [x10, x24, LSL 2] ldr w26, [x10, x26, LSL 2] ldr w21, [x10, x21, LSL 2] ldr w22, [x10, x22, LSL 2] ubfx x25, x17, #48, #8 eor w24, w24, w26, ror 24 ubfx x26, x17, #24, #8 eor w24, w24, w21, ror 8 ubfx x21, x16, #8, #8 eor w24, w24, w22, ror 16 ubfx x22, x16, #32, #8 bfi x23, x24, #32, #32 ldr w25, [x10, x25, LSL 2] ldr w26, [x10, x26, LSL 2] ldr w21, [x10, x21, LSL 2] ldr w22, [x10, x22, LSL 2] ubfx x19, x17, #0, #8 eor w25, w25, w26, ror 24 ubfx x26, x16, #16, #8 eor w25, w25, w21, ror 8 ubfx x21, x17, #56, #8 eor w24, w25, w22, ror 16 ubfx x22, x16, #40, #8 ldr w19, [x10, x19, LSL 2] ldr w21, [x10, x21, LSL 2] ldr w26, [x10, x26, LSL 2] ldr w22, [x10, x22, LSL 2] eor w21, w21, w19, ror 24 ldp x16, x17, [x28], #16 eor w26, w26, w21, ror 24 eor w26, w26, w22, ror 8 bfi x24, x26, #32, #32 # XOR in Key Schedule eor x23, x23, x16 eor x24, x24, x17 subs w27, w27, #2 bne L_AES_XTS_decrypt_loop_nr_tweak ubfx x16, x23, #48, #8 ubfx x20, x23, #24, #8 ubfx x21, x24, #8, #8 ubfx x22, x24, #32, #8 ldr x25, [x10] ldr x25, [x10, #64] ldr x25, [x10, #128] ldr x25, [x10, #192] ldr x25, [x10, #256] ldr x25, [x10, #320] ldr x25, [x10, #384] ldr x25, [x10, #448] ldr x25, [x10, #512] ldr x25, [x10, #576] ldr x25, [x10, #640] ldr x25, [x10, #704] ldr x25, [x10, #768] ldr x25, [x10, #832] ldr x25, [x10, #896] ldr x25, [x10, #960] ldr w16, [x10, x16, LSL 2] ldr w20, [x10, x20, LSL 2] ldr w21, [x10, x21, LSL 2] ldr w22, [x10, x22, LSL 2] ubfx x17, x24, #16, #8 eor w16, w16, w20, ror 24 ubfx x20, x23, #56, #8 eor w16, w16, w21, ror 8 ubfx x21, x24, #40, #8 eor w16, w16, w22, ror 16 ubfx x22, x23, #0, #8 ldr w17, [x10, x17, LSL 2] ldr w20, [x10, x20, LSL 2] ldr w21, [x10, x21, LSL 2] ldr w22, [x10, x22, LSL 2] ubfx x19, x24, #48, #8 eor w17, w17, w20, ror 24 ubfx x20, x24, #24, #8 eor w17, w17, w21, ror 8 ubfx x21, x23, #8, #8 eor w17, w17, w22, ror 16 ubfx x22, x23, #32, #8 bfi x16, x17, #32, #32 ldr w19, [x10, x19, LSL 2] ldr w20, [x10, x20, LSL 2] ldr w21, [x10, x21, LSL 2] ldr w22, [x10, x22, LSL 2] ubfx x25, x24, #0, #8 eor w19, w19, w20, ror 24 ubfx x20, x23, #16, #8 eor w19, w19, w21, ror 8 ubfx x21, x24, #56, #8 eor w17, w19, w22, ror 16 ubfx x22, x23, #40, #8 ldr w25, [x10, x25, LSL 2] ldr w21, [x10, x21, LSL 2] ldr w20, [x10, x20, LSL 2] ldr w22, [x10, x22, LSL 2] eor w21, w21, w25, ror 24 ldp x23, x24, [x28], #16 eor w20, w20, w21, ror 24 eor w20, w20, w22, ror 8 bfi x17, x20, #32, #32 # XOR in Key Schedule eor x16, x16, x23 eor x17, x17, x24 ubfx x23, x17, #32, #8 ubfx x26, x17, #8, #8 ubfx x21, x16, #48, #8 ubfx x22, x16, #24, #8 lsl w23, w23, #2 lsl w26, w26, #2 lsl w21, w21, #2 lsl w22, w22, #2 ldr x20, [x10] ldr x20, [x10, #64] ldr x20, [x10, #128] ldr x20, [x10, #192] ldr x20, [x10, #256] ldr x20, [x10, #320] ldr x20, [x10, #384] ldr x20, [x10, #448] ldr x20, [x10, #512] ldr x20, [x10, #576] ldr x20, [x10, #640] ldr x20, [x10, #704] ldr x20, [x10, #768] ldr x20, [x10, #832] ldr x20, [x10, #896] ldr x20, [x10, #960] ldrb w23, [x10, x23, LSL 0] ldrb w26, [x10, x26, LSL 0] ldrb w21, [x10, x21, LSL 0] ldrb w22, [x10, x22, LSL 0] ubfx x24, x16, #0, #8 eor w23, w23, w26, lsl 8 ubfx x26, x17, #40, #8 eor w23, w23, w21, lsl 16 ubfx x21, x17, #16, #8 eor w23, w23, w22, lsl 24 ubfx x22, x16, #56, #8 lsl w24, w24, #2 lsl w26, w26, #2 lsl w21, w21, #2 lsl w22, w22, #2 ldrb w24, [x10, x24, LSL 0] ldrb w26, [x10, x26, LSL 0] ldrb w21, [x10, x21, LSL 0] ldrb w22, [x10, x22, LSL 0] ubfx x25, x16, #32, #8 eor w24, w24, w26, lsl 8 ubfx x26, x16, #8, #8 eor w24, w24, w21, lsl 16 ubfx x21, x17, #48, #8 eor w24, w24, w22, lsl 24 ubfx x22, x17, #24, #8 bfi x23, x24, #32, #32 lsl w25, w25, #2 lsl w26, w26, #2 lsl w21, w21, #2 lsl w22, w22, #2 ldrb w25, [x10, x25, LSL 0] ldrb w26, [x10, x26, LSL 0] ldrb w21, [x10, x21, LSL 0] ldrb w22, [x10, x22, LSL 0] ubfx x20, x17, #56, #8 eor w25, w25, w26, lsl 8 ubfx x26, x17, #0, #8 eor w25, w25, w21, lsl 16 ubfx x21, x16, #40, #8 eor w24, w25, w22, lsl 24 ubfx x22, x16, #16, #8 lsl w20, w20, #2 lsl w26, w26, #2 lsl w21, w21, #2 lsl w22, w22, #2 ldrb w20, [x10, x20, LSL 0] ldrb w26, [x10, x26, LSL 0] ldrb w21, [x10, x21, LSL 0] ldrb w22, [x10, x22, LSL 0] eor w21, w21, w20, lsl 16 ldp x16, x17, [x28] eor w26, w26, w21, lsl 8 eor w26, w26, w22, lsl 16 bfi x24, x26, #32, #32 # XOR in Key Schedule eor x23, x23, x16 eor x24, x24, x17 rev32 x23, x23 rev32 x24, x24 cmp w2, #16 blt L_AES_XTS_decrypt_start_partail L_AES_XTS_decrypt_loop_block: mov x28, x4 ldp x12, x13, [x0] ldp x16, x17, [x28], #16 eor x12, x12, x23 eor x13, x13, x24 rev32 x12, x12 rev32 x13, x13 # Round: 0 - XOR in key schedule eor x12, x12, x16 eor x13, x13, x17 sub w27, w7, #2 L_AES_XTS_decrypt_loop_nr: ubfx x16, x13, #48, #8 ubfx x20, x12, #24, #8 ubfx x21, x13, #8, #8 ubfx x22, x12, #32, #8 ldr x14, [x8] ldr x14, [x8, #64] ldr x14, [x8, #128] ldr x14, [x8, #192] ldr x14, [x8, #256] ldr x14, [x8, #320] ldr x14, [x8, #384] ldr x14, [x8, #448] ldr x14, [x8, #512] ldr x14, [x8, #576] ldr x14, [x8, #640] ldr x14, [x8, #704] ldr x14, [x8, #768] ldr x14, [x8, #832] ldr x14, [x8, #896] ldr x14, [x8, #960] ldr w16, [x8, x16, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x17, x12, #16, #8 eor w16, w16, w20, ror 24 ubfx x20, x12, #56, #8 eor w16, w16, w21, ror 8 ubfx x21, x13, #40, #8 eor w16, w16, w22, ror 16 ubfx x22, x13, #0, #8 ldr w17, [x8, x17, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x19, x12, #48, #8 eor w17, w17, w20, ror 24 ubfx x20, x13, #24, #8 eor w17, w17, w21, ror 8 ubfx x21, x12, #8, #8 eor w17, w17, w22, ror 16 ubfx x22, x13, #32, #8 bfi x16, x17, #32, #32 ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x14, x12, #0, #8 eor w19, w19, w20, ror 24 ubfx x20, x13, #16, #8 eor w19, w19, w21, ror 8 ubfx x21, x13, #56, #8 eor w17, w19, w22, ror 16 ubfx x22, x12, #40, #8 ldr w14, [x8, x14, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w22, [x8, x22, LSL 2] eor w21, w21, w14, ror 24 ldp x12, x13, [x28], #16 eor w20, w20, w22, ror 8 eor w20, w20, w21, ror 24 bfi x17, x20, #32, #32 # XOR in Key Schedule eor x16, x16, x12 eor x17, x17, x13 ubfx x12, x17, #48, #8 ubfx x15, x16, #24, #8 ubfx x21, x17, #8, #8 ubfx x22, x16, #32, #8 ldr x19, [x8] ldr x19, [x8, #64] ldr x19, [x8, #128] ldr x19, [x8, #192] ldr x19, [x8, #256] ldr x19, [x8, #320] ldr x19, [x8, #384] ldr x19, [x8, #448] ldr x19, [x8, #512] ldr x19, [x8, #576] ldr x19, [x8, #640] ldr x19, [x8, #704] ldr x19, [x8, #768] ldr x19, [x8, #832] ldr x19, [x8, #896] ldr x19, [x8, #960] ldr w12, [x8, x12, LSL 2] ldr w15, [x8, x15, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x13, x16, #16, #8 eor w12, w12, w15, ror 24 ubfx x15, x16, #56, #8 eor w12, w12, w21, ror 8 ubfx x21, x17, #40, #8 eor w12, w12, w22, ror 16 ubfx x22, x17, #0, #8 ldr w13, [x8, x13, LSL 2] ldr w15, [x8, x15, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x14, x16, #48, #8 eor w13, w13, w15, ror 24 ubfx x15, x17, #24, #8 eor w13, w13, w21, ror 8 ubfx x21, x16, #8, #8 eor w13, w13, w22, ror 16 ubfx x22, x17, #32, #8 bfi x12, x13, #32, #32 ldr w14, [x8, x14, LSL 2] ldr w15, [x8, x15, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x19, x16, #0, #8 eor w14, w14, w15, ror 24 ubfx x15, x17, #16, #8 eor w14, w14, w21, ror 8 ubfx x21, x17, #56, #8 eor w13, w14, w22, ror 16 ubfx x22, x16, #40, #8 ldr w19, [x8, x19, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w15, [x8, x15, LSL 2] ldr w22, [x8, x22, LSL 2] eor w21, w21, w19, ror 24 ldp x16, x17, [x28], #16 eor w15, w15, w22, ror 8 eor w15, w15, w21, ror 24 bfi x13, x15, #32, #32 # XOR in Key Schedule eor x12, x12, x16 eor x13, x13, x17 subs w27, w27, #2 bne L_AES_XTS_decrypt_loop_nr ubfx x16, x13, #48, #8 ubfx x20, x12, #24, #8 ubfx x21, x13, #8, #8 ubfx x22, x12, #32, #8 ldr x14, [x8] ldr x14, [x8, #64] ldr x14, [x8, #128] ldr x14, [x8, #192] ldr x14, [x8, #256] ldr x14, [x8, #320] ldr x14, [x8, #384] ldr x14, [x8, #448] ldr x14, [x8, #512] ldr x14, [x8, #576] ldr x14, [x8, #640] ldr x14, [x8, #704] ldr x14, [x8, #768] ldr x14, [x8, #832] ldr x14, [x8, #896] ldr x14, [x8, #960] ldr w16, [x8, x16, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x17, x12, #16, #8 eor w16, w16, w20, ror 24 ubfx x20, x12, #56, #8 eor w16, w16, w21, ror 8 ubfx x21, x13, #40, #8 eor w16, w16, w22, ror 16 ubfx x22, x13, #0, #8 ldr w17, [x8, x17, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x19, x12, #48, #8 eor w17, w17, w20, ror 24 ubfx x20, x13, #24, #8 eor w17, w17, w21, ror 8 ubfx x21, x12, #8, #8 eor w17, w17, w22, ror 16 ubfx x22, x13, #32, #8 bfi x16, x17, #32, #32 ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x14, x12, #0, #8 eor w19, w19, w20, ror 24 ubfx x20, x13, #16, #8 eor w19, w19, w21, ror 8 ubfx x21, x13, #56, #8 eor w17, w19, w22, ror 16 ubfx x22, x12, #40, #8 ldr w14, [x8, x14, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w22, [x8, x22, LSL 2] eor w21, w21, w14, ror 24 ldp x12, x13, [x28], #16 eor w20, w20, w22, ror 8 eor w20, w20, w21, ror 24 bfi x17, x20, #32, #32 # XOR in Key Schedule eor x16, x16, x12 eor x17, x17, x13 ubfx x12, x16, #32, #8 ubfx x15, x17, #8, #8 ubfx x21, x17, #48, #8 ubfx x22, x16, #24, #8 ldr x20, [x9] ldr x20, [x9, #64] ldr x20, [x9, #128] ldr x20, [x9, #192] ldrb w12, [x9, x12, LSL 0] ldrb w15, [x9, x15, LSL 0] ldrb w21, [x9, x21, LSL 0] ldrb w22, [x9, x22, LSL 0] ubfx x13, x17, #0, #8 eor w12, w12, w15, lsl 8 ubfx x15, x17, #40, #8 eor w12, w12, w21, lsl 16 ubfx x21, x16, #16, #8 eor w12, w12, w22, lsl 24 ubfx x22, x16, #56, #8 ldrb w15, [x9, x15, LSL 0] ldrb w22, [x9, x22, LSL 0] ldrb w13, [x9, x13, LSL 0] ldrb w21, [x9, x21, LSL 0] ubfx x14, x17, #32, #8 eor w13, w13, w15, lsl 8 ubfx x15, x16, #8, #8 eor w13, w13, w21, lsl 16 ubfx x21, x16, #48, #8 eor w13, w13, w22, lsl 24 ubfx x22, x17, #24, #8 bfi x12, x13, #32, #32 ldrb w15, [x9, x15, LSL 0] ldrb w22, [x9, x22, LSL 0] ldrb w14, [x9, x14, LSL 0] ldrb w21, [x9, x21, LSL 0] ubfx x20, x17, #56, #8 eor w14, w14, w15, lsl 8 ubfx x15, x16, #0, #8 eor w14, w14, w21, lsl 16 ubfx x21, x16, #40, #8 eor w13, w14, w22, lsl 24 ubfx x22, x17, #16, #8 ldrb w20, [x9, x20, LSL 0] ldrb w21, [x9, x21, LSL 0] ldrb w15, [x9, x15, LSL 0] ldrb w22, [x9, x22, LSL 0] eor w21, w21, w20, lsl 16 ldp x16, x17, [x28] eor w15, w15, w21, lsl 8 eor w15, w15, w22, lsl 16 bfi x13, x15, #32, #32 # XOR in Key Schedule eor x12, x12, x16 eor x13, x13, x17 rev32 x12, x12 rev32 x13, x13 eor x12, x12, x23 eor x13, x13, x24 stp x12, x13, [x1] and x21, x11, x24, asr 63 extr x24, x24, x23, #63 eor x23, x21, x23, lsl 1 sub w2, w2, #16 add x0, x0, #16 add x1, x1, #16 cmp w2, #16 bge L_AES_XTS_decrypt_loop_block cbz w2, L_AES_XTS_decrypt_done_data L_AES_XTS_decrypt_start_partail: and x21, x11, x24, asr 63 extr x26, x24, x23, #63 eor x25, x21, x23, lsl 1 mov x28, x4 ldp x12, x13, [x0], #16 ldp x16, x17, [x28], #16 eor x12, x12, x25 eor x13, x13, x26 rev32 x12, x12 rev32 x13, x13 # Round: 0 - XOR in key schedule eor x12, x12, x16 eor x13, x13, x17 sub w27, w7, #2 L_AES_XTS_decrypt_loop_nr_partial_1: ubfx x16, x13, #48, #8 ubfx x20, x12, #24, #8 ubfx x21, x13, #8, #8 ubfx x22, x12, #32, #8 ldr x14, [x8] ldr x14, [x8, #64] ldr x14, [x8, #128] ldr x14, [x8, #192] ldr x14, [x8, #256] ldr x14, [x8, #320] ldr x14, [x8, #384] ldr x14, [x8, #448] ldr x14, [x8, #512] ldr x14, [x8, #576] ldr x14, [x8, #640] ldr x14, [x8, #704] ldr x14, [x8, #768] ldr x14, [x8, #832] ldr x14, [x8, #896] ldr x14, [x8, #960] ldr w16, [x8, x16, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x17, x12, #16, #8 eor w16, w16, w20, ror 24 ubfx x20, x12, #56, #8 eor w16, w16, w21, ror 8 ubfx x21, x13, #40, #8 eor w16, w16, w22, ror 16 ubfx x22, x13, #0, #8 ldr w17, [x8, x17, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x19, x12, #48, #8 eor w17, w17, w20, ror 24 ubfx x20, x13, #24, #8 eor w17, w17, w21, ror 8 ubfx x21, x12, #8, #8 eor w17, w17, w22, ror 16 ubfx x22, x13, #32, #8 bfi x16, x17, #32, #32 ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x14, x12, #0, #8 eor w19, w19, w20, ror 24 ubfx x20, x13, #16, #8 eor w19, w19, w21, ror 8 ubfx x21, x13, #56, #8 eor w17, w19, w22, ror 16 ubfx x22, x12, #40, #8 ldr w14, [x8, x14, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w22, [x8, x22, LSL 2] eor w21, w21, w14, ror 24 ldp x12, x13, [x28], #16 eor w20, w20, w22, ror 8 eor w20, w20, w21, ror 24 bfi x17, x20, #32, #32 # XOR in Key Schedule eor x16, x16, x12 eor x17, x17, x13 ubfx x12, x17, #48, #8 ubfx x15, x16, #24, #8 ubfx x21, x17, #8, #8 ubfx x22, x16, #32, #8 ldr x19, [x8] ldr x19, [x8, #64] ldr x19, [x8, #128] ldr x19, [x8, #192] ldr x19, [x8, #256] ldr x19, [x8, #320] ldr x19, [x8, #384] ldr x19, [x8, #448] ldr x19, [x8, #512] ldr x19, [x8, #576] ldr x19, [x8, #640] ldr x19, [x8, #704] ldr x19, [x8, #768] ldr x19, [x8, #832] ldr x19, [x8, #896] ldr x19, [x8, #960] ldr w12, [x8, x12, LSL 2] ldr w15, [x8, x15, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x13, x16, #16, #8 eor w12, w12, w15, ror 24 ubfx x15, x16, #56, #8 eor w12, w12, w21, ror 8 ubfx x21, x17, #40, #8 eor w12, w12, w22, ror 16 ubfx x22, x17, #0, #8 ldr w13, [x8, x13, LSL 2] ldr w15, [x8, x15, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x14, x16, #48, #8 eor w13, w13, w15, ror 24 ubfx x15, x17, #24, #8 eor w13, w13, w21, ror 8 ubfx x21, x16, #8, #8 eor w13, w13, w22, ror 16 ubfx x22, x17, #32, #8 bfi x12, x13, #32, #32 ldr w14, [x8, x14, LSL 2] ldr w15, [x8, x15, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x19, x16, #0, #8 eor w14, w14, w15, ror 24 ubfx x15, x17, #16, #8 eor w14, w14, w21, ror 8 ubfx x21, x17, #56, #8 eor w13, w14, w22, ror 16 ubfx x22, x16, #40, #8 ldr w19, [x8, x19, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w15, [x8, x15, LSL 2] ldr w22, [x8, x22, LSL 2] eor w21, w21, w19, ror 24 ldp x16, x17, [x28], #16 eor w15, w15, w22, ror 8 eor w15, w15, w21, ror 24 bfi x13, x15, #32, #32 # XOR in Key Schedule eor x12, x12, x16 eor x13, x13, x17 subs w27, w27, #2 bne L_AES_XTS_decrypt_loop_nr_partial_1 ubfx x16, x13, #48, #8 ubfx x20, x12, #24, #8 ubfx x21, x13, #8, #8 ubfx x22, x12, #32, #8 ldr x14, [x8] ldr x14, [x8, #64] ldr x14, [x8, #128] ldr x14, [x8, #192] ldr x14, [x8, #256] ldr x14, [x8, #320] ldr x14, [x8, #384] ldr x14, [x8, #448] ldr x14, [x8, #512] ldr x14, [x8, #576] ldr x14, [x8, #640] ldr x14, [x8, #704] ldr x14, [x8, #768] ldr x14, [x8, #832] ldr x14, [x8, #896] ldr x14, [x8, #960] ldr w16, [x8, x16, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x17, x12, #16, #8 eor w16, w16, w20, ror 24 ubfx x20, x12, #56, #8 eor w16, w16, w21, ror 8 ubfx x21, x13, #40, #8 eor w16, w16, w22, ror 16 ubfx x22, x13, #0, #8 ldr w17, [x8, x17, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x19, x12, #48, #8 eor w17, w17, w20, ror 24 ubfx x20, x13, #24, #8 eor w17, w17, w21, ror 8 ubfx x21, x12, #8, #8 eor w17, w17, w22, ror 16 ubfx x22, x13, #32, #8 bfi x16, x17, #32, #32 ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x14, x12, #0, #8 eor w19, w19, w20, ror 24 ubfx x20, x13, #16, #8 eor w19, w19, w21, ror 8 ubfx x21, x13, #56, #8 eor w17, w19, w22, ror 16 ubfx x22, x12, #40, #8 ldr w14, [x8, x14, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w22, [x8, x22, LSL 2] eor w21, w21, w14, ror 24 ldp x12, x13, [x28], #16 eor w20, w20, w22, ror 8 eor w20, w20, w21, ror 24 bfi x17, x20, #32, #32 # XOR in Key Schedule eor x16, x16, x12 eor x17, x17, x13 ubfx x12, x16, #32, #8 ubfx x15, x17, #8, #8 ubfx x21, x17, #48, #8 ubfx x22, x16, #24, #8 ldr x20, [x9] ldr x20, [x9, #64] ldr x20, [x9, #128] ldr x20, [x9, #192] ldrb w12, [x9, x12, LSL 0] ldrb w15, [x9, x15, LSL 0] ldrb w21, [x9, x21, LSL 0] ldrb w22, [x9, x22, LSL 0] ubfx x13, x17, #0, #8 eor w12, w12, w15, lsl 8 ubfx x15, x17, #40, #8 eor w12, w12, w21, lsl 16 ubfx x21, x16, #16, #8 eor w12, w12, w22, lsl 24 ubfx x22, x16, #56, #8 ldrb w15, [x9, x15, LSL 0] ldrb w22, [x9, x22, LSL 0] ldrb w13, [x9, x13, LSL 0] ldrb w21, [x9, x21, LSL 0] ubfx x14, x17, #32, #8 eor w13, w13, w15, lsl 8 ubfx x15, x16, #8, #8 eor w13, w13, w21, lsl 16 ubfx x21, x16, #48, #8 eor w13, w13, w22, lsl 24 ubfx x22, x17, #24, #8 bfi x12, x13, #32, #32 ldrb w15, [x9, x15, LSL 0] ldrb w22, [x9, x22, LSL 0] ldrb w14, [x9, x14, LSL 0] ldrb w21, [x9, x21, LSL 0] ubfx x20, x17, #56, #8 eor w14, w14, w15, lsl 8 ubfx x15, x16, #0, #8 eor w14, w14, w21, lsl 16 ubfx x21, x16, #40, #8 eor w13, w14, w22, lsl 24 ubfx x22, x17, #16, #8 ldrb w20, [x9, x20, LSL 0] ldrb w21, [x9, x21, LSL 0] ldrb w15, [x9, x15, LSL 0] ldrb w22, [x9, x22, LSL 0] eor w21, w21, w20, lsl 16 ldp x16, x17, [x28] eor w15, w15, w21, lsl 8 eor w15, w15, w22, lsl 16 bfi x13, x15, #32, #32 # XOR in Key Schedule eor x12, x12, x16 eor x13, x13, x17 rev32 x12, x12 rev32 x13, x13 eor x12, x12, x25 eor x13, x13, x26 stp x12, x13, [x6] add x1, x1, #16 mov w16, w2 L_AES_XTS_decrypt_start_byte: ldrb w21, [x6] ldrb w22, [x0], #1 strb w21, [x1], #1 strb w22, [x6], #1 subs w16, w16, #1 bgt L_AES_XTS_decrypt_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 mov x28, x4 ldp x12, x13, [x6] ldp x16, x17, [x28], #16 eor x12, x12, x23 eor x13, x13, x24 rev32 x12, x12 rev32 x13, x13 # Round: 0 - XOR in key schedule eor x12, x12, x16 eor x13, x13, x17 sub w27, w7, #2 L_AES_XTS_decrypt_loop_nr_partial_2: ubfx x16, x13, #48, #8 ubfx x20, x12, #24, #8 ubfx x21, x13, #8, #8 ubfx x22, x12, #32, #8 ldr x14, [x8] ldr x14, [x8, #64] ldr x14, [x8, #128] ldr x14, [x8, #192] ldr x14, [x8, #256] ldr x14, [x8, #320] ldr x14, [x8, #384] ldr x14, [x8, #448] ldr x14, [x8, #512] ldr x14, [x8, #576] ldr x14, [x8, #640] ldr x14, [x8, #704] ldr x14, [x8, #768] ldr x14, [x8, #832] ldr x14, [x8, #896] ldr x14, [x8, #960] ldr w16, [x8, x16, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x17, x12, #16, #8 eor w16, w16, w20, ror 24 ubfx x20, x12, #56, #8 eor w16, w16, w21, ror 8 ubfx x21, x13, #40, #8 eor w16, w16, w22, ror 16 ubfx x22, x13, #0, #8 ldr w17, [x8, x17, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x19, x12, #48, #8 eor w17, w17, w20, ror 24 ubfx x20, x13, #24, #8 eor w17, w17, w21, ror 8 ubfx x21, x12, #8, #8 eor w17, w17, w22, ror 16 ubfx x22, x13, #32, #8 bfi x16, x17, #32, #32 ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x14, x12, #0, #8 eor w19, w19, w20, ror 24 ubfx x20, x13, #16, #8 eor w19, w19, w21, ror 8 ubfx x21, x13, #56, #8 eor w17, w19, w22, ror 16 ubfx x22, x12, #40, #8 ldr w14, [x8, x14, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w22, [x8, x22, LSL 2] eor w21, w21, w14, ror 24 ldp x12, x13, [x28], #16 eor w20, w20, w22, ror 8 eor w20, w20, w21, ror 24 bfi x17, x20, #32, #32 # XOR in Key Schedule eor x16, x16, x12 eor x17, x17, x13 ubfx x12, x17, #48, #8 ubfx x15, x16, #24, #8 ubfx x21, x17, #8, #8 ubfx x22, x16, #32, #8 ldr x19, [x8] ldr x19, [x8, #64] ldr x19, [x8, #128] ldr x19, [x8, #192] ldr x19, [x8, #256] ldr x19, [x8, #320] ldr x19, [x8, #384] ldr x19, [x8, #448] ldr x19, [x8, #512] ldr x19, [x8, #576] ldr x19, [x8, #640] ldr x19, [x8, #704] ldr x19, [x8, #768] ldr x19, [x8, #832] ldr x19, [x8, #896] ldr x19, [x8, #960] ldr w12, [x8, x12, LSL 2] ldr w15, [x8, x15, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x13, x16, #16, #8 eor w12, w12, w15, ror 24 ubfx x15, x16, #56, #8 eor w12, w12, w21, ror 8 ubfx x21, x17, #40, #8 eor w12, w12, w22, ror 16 ubfx x22, x17, #0, #8 ldr w13, [x8, x13, LSL 2] ldr w15, [x8, x15, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x14, x16, #48, #8 eor w13, w13, w15, ror 24 ubfx x15, x17, #24, #8 eor w13, w13, w21, ror 8 ubfx x21, x16, #8, #8 eor w13, w13, w22, ror 16 ubfx x22, x17, #32, #8 bfi x12, x13, #32, #32 ldr w14, [x8, x14, LSL 2] ldr w15, [x8, x15, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x19, x16, #0, #8 eor w14, w14, w15, ror 24 ubfx x15, x17, #16, #8 eor w14, w14, w21, ror 8 ubfx x21, x17, #56, #8 eor w13, w14, w22, ror 16 ubfx x22, x16, #40, #8 ldr w19, [x8, x19, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w15, [x8, x15, LSL 2] ldr w22, [x8, x22, LSL 2] eor w21, w21, w19, ror 24 ldp x16, x17, [x28], #16 eor w15, w15, w22, ror 8 eor w15, w15, w21, ror 24 bfi x13, x15, #32, #32 # XOR in Key Schedule eor x12, x12, x16 eor x13, x13, x17 subs w27, w27, #2 bne L_AES_XTS_decrypt_loop_nr_partial_2 ubfx x16, x13, #48, #8 ubfx x20, x12, #24, #8 ubfx x21, x13, #8, #8 ubfx x22, x12, #32, #8 ldr x14, [x8] ldr x14, [x8, #64] ldr x14, [x8, #128] ldr x14, [x8, #192] ldr x14, [x8, #256] ldr x14, [x8, #320] ldr x14, [x8, #384] ldr x14, [x8, #448] ldr x14, [x8, #512] ldr x14, [x8, #576] ldr x14, [x8, #640] ldr x14, [x8, #704] ldr x14, [x8, #768] ldr x14, [x8, #832] ldr x14, [x8, #896] ldr x14, [x8, #960] ldr w16, [x8, x16, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x17, x12, #16, #8 eor w16, w16, w20, ror 24 ubfx x20, x12, #56, #8 eor w16, w16, w21, ror 8 ubfx x21, x13, #40, #8 eor w16, w16, w22, ror 16 ubfx x22, x13, #0, #8 ldr w17, [x8, x17, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x19, x12, #48, #8 eor w17, w17, w20, ror 24 ubfx x20, x13, #24, #8 eor w17, w17, w21, ror 8 ubfx x21, x12, #8, #8 eor w17, w17, w22, ror 16 ubfx x22, x13, #32, #8 bfi x16, x17, #32, #32 ldr w19, [x8, x19, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w22, [x8, x22, LSL 2] ubfx x14, x12, #0, #8 eor w19, w19, w20, ror 24 ubfx x20, x13, #16, #8 eor w19, w19, w21, ror 8 ubfx x21, x13, #56, #8 eor w17, w19, w22, ror 16 ubfx x22, x12, #40, #8 ldr w14, [x8, x14, LSL 2] ldr w21, [x8, x21, LSL 2] ldr w20, [x8, x20, LSL 2] ldr w22, [x8, x22, LSL 2] eor w21, w21, w14, ror 24 ldp x12, x13, [x28], #16 eor w20, w20, w22, ror 8 eor w20, w20, w21, ror 24 bfi x17, x20, #32, #32 # XOR in Key Schedule eor x16, x16, x12 eor x17, x17, x13 ubfx x12, x16, #32, #8 ubfx x15, x17, #8, #8 ubfx x21, x17, #48, #8 ubfx x22, x16, #24, #8 ldr x20, [x9] ldr x20, [x9, #64] ldr x20, [x9, #128] ldr x20, [x9, #192] ldrb w12, [x9, x12, LSL 0] ldrb w15, [x9, x15, LSL 0] ldrb w21, [x9, x21, LSL 0] ldrb w22, [x9, x22, LSL 0] ubfx x13, x17, #0, #8 eor w12, w12, w15, lsl 8 ubfx x15, x17, #40, #8 eor w12, w12, w21, lsl 16 ubfx x21, x16, #16, #8 eor w12, w12, w22, lsl 24 ubfx x22, x16, #56, #8 ldrb w15, [x9, x15, LSL 0] ldrb w22, [x9, x22, LSL 0] ldrb w13, [x9, x13, LSL 0] ldrb w21, [x9, x21, LSL 0] ubfx x14, x17, #32, #8 eor w13, w13, w15, lsl 8 ubfx x15, x16, #8, #8 eor w13, w13, w21, lsl 16 ubfx x21, x16, #48, #8 eor w13, w13, w22, lsl 24 ubfx x22, x17, #24, #8 bfi x12, x13, #32, #32 ldrb w15, [x9, x15, LSL 0] ldrb w22, [x9, x22, LSL 0] ldrb w14, [x9, x14, LSL 0] ldrb w21, [x9, x21, LSL 0] ubfx x20, x17, #56, #8 eor w14, w14, w15, lsl 8 ubfx x15, x16, #0, #8 eor w14, w14, w21, lsl 16 ubfx x21, x16, #40, #8 eor w13, w14, w22, lsl 24 ubfx x22, x17, #16, #8 ldrb w20, [x9, x20, LSL 0] ldrb w21, [x9, x21, LSL 0] ldrb w15, [x9, x15, LSL 0] ldrb w22, [x9, x22, LSL 0] eor w21, w21, w20, lsl 16 ldp x16, x17, [x28] eor w15, w15, w21, lsl 8 eor w15, w15, w22, lsl 16 bfi x13, x15, #32, #32 # XOR in Key Schedule eor x12, x12, x16 eor x13, x13, x17 rev32 x12, x12 rev32 x13, x13 eor x12, x12, x23 eor x13, x13, x24 stp x12, x13, [x1] L_AES_XTS_decrypt_done_data: ldp x17, x19, [x29, #24] ldp x20, x21, [x29, #40] ldp x22, x23, [x29, #56] ldp x24, x25, [x29, #72] ldp x26, x27, [x29, #88] ldr x28, [x29, #104] ldp x29, x30, [sp], #0x70 ret #ifndef __APPLE__ .size AES_XTS_decrypt,.-AES_XTS_decrypt #endif /* __APPLE__ */ #endif /* HAVE_AES_DECRYPT */ #endif /* WOLFSSL_AES_XTS */ #endif /* !WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP */ #endif /* !defined(NO_AES) && defined(WOLFSSL_ARMASM) */ #endif /* __aarch64__ */ #endif /* WOLFSSL_ARMASM */ #if defined(__linux__) && defined(__ELF__) .section .note.GNU-stack,"",%progbits #endif #endif /* !WOLFSSL_ARMASM_INLINE */